Coverage for src/c2puml/core/parser.py: 79%
912 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-09-14 18:23 +0000
1#!/usr/bin/env python3
2"""
3Parser module for C to PlantUML converter - Step 1: Parse C code files and generate model.json
4"""
5import logging
6from pathlib import Path
7from typing import TYPE_CHECKING, Dict, List, Optional, Set
9from ..models import Enum, EnumValue, Field, FileModel, ProjectModel, Struct
10from .parser_tokenizer import (
11 CTokenizer,
12 StructureFinder,
13 TokenType,
14 find_enum_values,
15 find_struct_fields,
16)
17from .preprocessor import PreprocessorManager
18from .parser_anonymous_processor import AnonymousTypedefProcessor
19from ..utils import detect_file_encoding
20import re
21from .parse_utils import (
22 clean_type_string,
23 clean_value_string,
24 fix_array_bracket_spacing,
25 fix_pointer_spacing,
26 collect_array_dimensions_from_tokens,
27 join_type_with_dims,
28 normalize_dim_value,
29)
31INCLUDE_FILENAME_RE = re.compile(r'[<"\']([^>\'\"]+)[>\'\"]')
33if TYPE_CHECKING:
34 from ..config import Config
35 from ..models import Alias, Enum, Field, Function, Struct, Union
38class CParser:
39 """C/C++ parser for extracting structural information from source code using tokenization"""
41 def __init__(self):
42 self.logger = logging.getLogger(__name__)
43 self.tokenizer = CTokenizer()
44 self.preprocessor = PreprocessorManager()
46 def parse_project(
47 self, source_folder: str, recursive_search: bool = True, config: "Config" = None
48 ) -> ProjectModel:
49 """Parse a C/C++ project and return a model"""
50 # Enhanced source path validation
51 if not source_folder or not isinstance(source_folder, str):
52 raise ValueError(f"Source folder must be a non-empty string, got: {type(source_folder)}")
54 if not source_folder.strip():
55 raise ValueError("Source folder cannot be empty or whitespace")
57 try:
58 source_folder_path = Path(source_folder).resolve()
59 except (OSError, RuntimeError) as e:
60 raise ValueError(f"Failed to resolve source folder path '{source_folder}': {e}")
62 if not source_folder_path.exists():
63 # Provide helpful error message with suggestions
64 error_msg = f"Source folder not found: {source_folder_path}"
66 # Check if it's a relative path issue
67 if not Path(source_folder).is_absolute():
68 current_dir = Path.cwd()
69 error_msg += f"\nCurrent working directory: {current_dir}"
70 error_msg += f"\nTried to resolve relative path: {source_folder}"
72 # Check if parent directory exists
73 parent_dir = source_folder_path.parent
74 if parent_dir.exists():
75 error_msg += f"\nParent directory exists: {parent_dir}"
76 # List contents of parent directory
77 try:
78 contents = [item.name for item in parent_dir.iterdir() if item.is_dir()]
79 if contents:
80 error_msg += f"\nAvailable directories in parent: {', '.join(contents[:10])}"
81 if len(contents) > 10:
82 error_msg += f" (and {len(contents) - 10} more)"
83 except (OSError, PermissionError):
84 error_msg += "\nCannot list parent directory contents (permission denied)"
85 else:
86 error_msg += f"\nParent directory does not exist: {parent_dir}"
88 raise ValueError(error_msg)
90 if not source_folder_path.is_dir():
91 raise ValueError(f"Source folder must be a directory, got: {source_folder_path} (is_file: {source_folder_path.is_file()})")
93 # Check if directory is readable
94 try:
95 source_folder_path.iterdir()
96 except PermissionError:
97 raise ValueError(f"Permission denied accessing source folder: {source_folder_path}")
98 except OSError as e:
99 raise ValueError(f"Error accessing source folder '{source_folder_path}': {e}")
101 self.logger.info("Parsing project: %s", source_folder_path)
103 # Find all C/C++ files in the project
104 try:
105 all_c_files = self._find_c_files(source_folder_path, recursive_search)
106 except OSError as e:
107 raise ValueError(f"Error searching for C/C++ files in '{source_folder_path}': {e}")
109 self.logger.info("Found %d C/C++ files", len(all_c_files))
111 # Apply file filtering based on configuration
112 c_files = []
113 if config:
114 for file_path in all_c_files:
115 if config._should_include_file(file_path.name):
116 c_files.append(file_path)
117 self.logger.debug(
118 "Included file after filtering: %s", file_path.name
119 )
120 else:
121 self.logger.debug(
122 "Excluded file after filtering: %s", file_path.name
123 )
124 else:
125 c_files = all_c_files
127 self.logger.info("After filtering: %d C/C++ files", len(c_files))
129 # Parse each file using filename as key for simplified tracking
130 files = {}
131 failed_files = []
133 for file_path in c_files:
134 try:
135 # Use relative path for tracking and filename as key
136 relative_path = str(file_path.relative_to(source_folder_path))
137 file_model = self.parse_file(file_path, relative_path)
139 # Use filename as key (filenames are guaranteed to be unique)
140 if file_model.name in files:
141 raise RuntimeError(
142 f"Duplicate filename detected: '{file_model.name}' from '{file_path}'. "
143 f"Already seen from '{files[file_model.name].file_path}'."
144 )
145 files[file_model.name] = file_model
147 self.logger.debug("Successfully parsed: %s", relative_path)
149 except (OSError, ValueError) as e:
150 self.logger.warning("Failed to parse %s: %s", file_path, e)
151 failed_files.append(str(file_path))
153 if failed_files:
154 error_msg = (
155 f"Failed to parse {len(failed_files)} files: {failed_files}. "
156 "Stopping model processing."
157 )
158 self.logger.error(error_msg)
159 raise RuntimeError(error_msg)
161 model = ProjectModel(
162 project_name=source_folder_path.name,
163 source_folder=str(source_folder_path),
164 files=files,
165 )
167 # Update all uses fields across the entire project
168 model.update_uses_fields()
170 self.logger.info("Parsing complete. Parsed %d files successfully.", len(files))
171 return model
173 def parse_file(self, file_path: Path, relative_path: str) -> FileModel:
174 """Parse a single C/C++ file and return a file model using tokenization"""
175 self.logger.debug("Parsing file: %s", file_path)
177 # Detect encoding
178 encoding = self._detect_encoding(file_path)
180 # Read file content
181 with open(file_path, "r", encoding=encoding) as f:
182 content = f.read()
184 # Tokenize the content
185 tokens = self.tokenizer.tokenize(content)
186 self.logger.debug("Tokenized file into %d tokens", len(tokens))
188 # Process preprocessor directives
189 self.preprocessor.add_defines_from_content(tokens)
190 processed_tokens = self.preprocessor.process_file(tokens)
191 self.logger.debug(
192 "Preprocessor processed %d tokens -> %d tokens",
193 len(tokens),
194 len(processed_tokens),
195 )
197 # Filter out whitespace and comments for structure finding
198 filtered_tokens = self.tokenizer.filter_tokens(processed_tokens)
199 structure_finder = StructureFinder(filtered_tokens)
201 # Parse different structures using tokenizer
202 structs = self._parse_structs_with_tokenizer(processed_tokens, structure_finder)
203 enums = self._parse_enums_with_tokenizer(processed_tokens, structure_finder)
204 unions = self._parse_unions_with_tokenizer(processed_tokens, structure_finder)
205 functions = self._parse_functions_with_tokenizer(
206 processed_tokens, structure_finder
207 )
208 aliases = self._parse_aliases_with_tokenizer(processed_tokens)
210 # "uses" fields will be updated when we have the full project model
212 # Map typedef names to anonymous structs/enums/unions if needed
213 # This logic will be handled by typedef_relations instead
215 file_model = FileModel(
216 file_path=str(file_path),
217 structs=structs,
218 enums=enums,
219 unions=unions,
220 functions=functions,
221 globals=self._parse_globals_with_tokenizer(processed_tokens),
222 includes=self._parse_includes_with_tokenizer(processed_tokens),
223 macros=self._parse_macros_with_tokenizer(processed_tokens),
224 aliases=aliases,
225 # Tag names are now stored in struct/enum/union objects
226 )
228 # Process anonymous typedefs after initial parsing
229 anonymous_processor = AnonymousTypedefProcessor()
230 anonymous_processor.process_file_model(file_model)
232 return file_model
234 def _parse_structs_with_tokenizer(
235 self, tokens, structure_finder
236 ) -> Dict[str, "Struct"]:
237 """Parse struct definitions using tokenizer"""
239 structs = {}
240 struct_infos = structure_finder.find_structs()
242 for start_pos, end_pos, struct_name in struct_infos:
243 # Need to map back to original token positions
244 # Find the original token positions by looking at line/column info
245 original_start = self._find_original_token_pos(
246 tokens, structure_finder.tokens, start_pos
247 )
248 original_end = self._find_original_token_pos(
249 tokens, structure_finder.tokens, end_pos
250 )
252 if original_start is not None and original_end is not None:
253 # Extract field information from original token range
254 field_tuples = find_struct_fields(tokens, original_start, original_end)
256 # Convert to Field objects
257 fields = []
258 for field_name, field_type in field_tuples:
259 try:
260 fields.append(Field(field_name, field_type))
261 except ValueError as e:
262 self.logger.warning(
263 "Error creating field %s: %s", field_name, e
264 )
266 # For anonymous structs, use a special key that can be mapped later
267 if not struct_name:
268 struct_name = "__anonymous_struct__"
270 # Extract tag name if this is a typedef struct
271 tag_name = ""
272 if struct_name and not struct_name.startswith("__anonymous"):
273 # Check if this struct has a typedef
274 tag_name = self._extract_tag_name_for_struct(tokens, struct_name)
276 # Only register non-empty struct names here; anonymous will be created by the anonymous processor
277 if struct_name:
278 structs[struct_name] = Struct(
279 struct_name, fields, tag_name=tag_name, uses=[]
280 )
281 self.logger.debug(
282 "Parsed struct: %s with %d fields", struct_name, len(fields)
283 )
285 return structs
287 def _parse_enums_with_tokenizer(
288 self, tokens, structure_finder
289 ) -> Dict[str, "Enum"]:
290 """Parse enum definitions using tokenizer"""
291 enums = {}
292 enum_infos = structure_finder.find_enums()
294 for start_pos, end_pos, enum_name in enum_infos:
295 # Need to map back to original token positions
296 original_start = self._find_original_token_pos(
297 tokens, structure_finder.tokens, start_pos
298 )
299 original_end = self._find_original_token_pos(
300 tokens, structure_finder.tokens, end_pos
301 )
303 if original_start is not None and original_end is not None:
304 # Extract enum values from original token range
305 value_strs = find_enum_values(tokens, original_start, original_end)
306 values = []
307 for v in value_strs:
308 if "=" in v:
309 name, val = v.split("=", 1)
310 name = name.strip()
311 val = val.strip()
312 if name: # Only add if name is not empty
313 values.append(EnumValue(name=name, value=val))
314 else:
315 name = v.strip()
316 if name: # Only add if name is not empty
317 values.append(EnumValue(name=name))
319 # For anonymous enums, use a special key that can be mapped later
320 if not enum_name:
321 enum_name = "__anonymous_enum__"
323 # Extract tag name if this is a typedef enum
324 tag_name = ""
325 if enum_name and not enum_name.startswith("__anonymous"):
326 # Check if this enum has a typedef
327 tag_name = self._extract_tag_name_for_enum(tokens, enum_name)
329 enums[enum_name] = Enum(enum_name, values, tag_name=tag_name)
330 self.logger.debug(
331 "Parsed enum: %s with %d values", enum_name, len(values)
332 )
334 return enums
336 def _parse_unions_with_tokenizer(
337 self, tokens, structure_finder
338 ) -> Dict[str, "Union"]:
339 """Parse union definitions using tokenizer"""
340 from ..models import Field, Union
342 unions = {}
343 union_infos = structure_finder.find_unions()
345 for start_pos, end_pos, union_name in union_infos:
346 # Need to map back to original token positions
347 original_start = self._find_original_token_pos(
348 tokens, structure_finder.tokens, start_pos
349 )
350 original_end = self._find_original_token_pos(
351 tokens, structure_finder.tokens, end_pos
352 )
354 if original_start is not None and original_end is not None:
355 # Extract field information from original token range
356 field_tuples = find_struct_fields(tokens, original_start, original_end)
358 # Convert to Field objects
359 fields = []
360 for field_name, field_type in field_tuples:
361 try:
362 fields.append(Field(field_name, field_type))
363 except ValueError as e:
364 self.logger.warning(
365 "Error creating union field %s: %s", field_name, e
366 )
368 # For anonymous unions, use a special key that can be mapped later
369 if not union_name:
370 union_name = "__anonymous_union__"
372 # Extract tag name if this is a typedef union
373 tag_name = ""
374 if union_name and not union_name.startswith("__anonymous"):
375 # Check if this union has a typedef
376 tag_name = self._extract_tag_name_for_union(tokens, union_name)
378 unions[union_name] = Union(
379 union_name, fields, tag_name=tag_name, uses=[]
380 )
381 self.logger.debug(
382 "Parsed union: %s with %d fields", union_name, len(fields)
383 )
385 return unions
387 def _parse_functions_with_tokenizer(
388 self, tokens, structure_finder
389 ) -> List["Function"]:
390 """Parse function declarations/definitions using tokenizer"""
391 from ..models import Function
393 functions = []
394 function_infos = structure_finder.find_functions()
396 for (
397 start_pos,
398 end_pos,
399 func_name,
400 return_type,
401 is_declaration,
402 is_inline,
403 ) in function_infos:
404 # Map back to original token positions to parse parameters
405 original_start = self._find_original_token_pos(
406 tokens, structure_finder.tokens, start_pos
407 )
408 original_end = self._find_original_token_pos(
409 tokens, structure_finder.tokens, end_pos
410 )
412 parameters = []
413 if original_start is not None and original_end is not None:
414 # Parse parameters from the token range
415 parameters = self._parse_function_parameters(
416 tokens, original_start, original_end, func_name
417 )
419 try:
420 # Create function with declaration flag
421 function = Function(func_name, return_type, parameters)
422 # Add custom attributes to track if this is a declaration and if it's inline
423 function.is_declaration = is_declaration
424 function.is_inline = is_inline
425 functions.append(function)
426 self.logger.debug(
427 f"Parsed function: {func_name} with {len(parameters)} parameters (declaration: {is_declaration}, inline: {is_inline})"
428 )
429 except Exception as e:
430 self.logger.warning("Error creating function %s: %s", func_name, e)
432 return functions
434 def _parse_globals_with_tokenizer(self, tokens) -> List["Field"]:
435 """Parse global variables using tokenizer"""
436 from ..models import Field
438 globals_list = []
440 i = 0
441 while i < len(tokens):
442 # Skip preprocessor directives, comments, etc.
443 if tokens[i].type in [
444 TokenType.INCLUDE,
445 TokenType.DEFINE,
446 TokenType.COMMENT,
447 TokenType.WHITESPACE,
448 TokenType.NEWLINE,
449 ]:
450 i += 1
451 continue
453 # Skip preprocessor directives but keep their content
454 if tokens[i].type == TokenType.PREPROCESSOR:
455 i = self._skip_preprocessor_directives(tokens, i)
456 continue
458 # Skip function definitions (look for parentheses)
459 if self._looks_like_function(tokens, i):
460 i = self._skip_function(tokens, i)
461 continue
463 # Skip struct/enum/union definitions
464 if tokens[i].type in [
465 TokenType.STRUCT,
466 TokenType.ENUM,
467 TokenType.UNION,
468 TokenType.TYPEDEF,
469 ]:
470 i = self._skip_structure_definition(tokens, i)
471 continue
473 # Skip if we're inside a struct definition (look for opening brace)
474 if i > 0 and tokens[i - 1].type == TokenType.LBRACE:
475 # We're inside a struct, skip until closing brace
476 brace_count = 1
477 j = i
478 while j < len(tokens) and brace_count > 0:
479 if tokens[j].type == TokenType.LBRACE:
480 brace_count += 1
481 elif tokens[j].type == TokenType.RBRACE:
482 brace_count -= 1
483 j += 1
484 i = j
485 continue
487 # Skip macros and other preprocessor content
488 if tokens[i].type == TokenType.DEFINE:
489 # Skip the entire macro content (multi-line macros are now merged)
490 i += 1
491 continue
493 # Additional check: skip if we're inside any brace block (struct, function, etc.)
494 brace_count = 0
495 j = i - 1
496 while j >= 0:
497 if tokens[j].type == TokenType.RBRACE:
498 brace_count += 1
499 elif tokens[j].type == TokenType.LBRACE:
500 brace_count -= 1
501 if brace_count < 0:
502 # We're inside a brace block, skip this token
503 i += 1
504 break
505 j -= 1
506 else:
507 # Not inside a brace block, proceed with global variable parsing
508 global_info = self._parse_global_variable(tokens, i)
509 if global_info:
510 var_name, var_type, var_value = global_info
511 # Only add if it looks like a real global variable (not a fragment)
512 if (
513 var_name
514 and var_name.strip()
515 and var_type
516 and var_type.strip()
517 and not var_name.startswith("#")
518 and len(var_type) < 200
519 and not var_type.startswith("\\")
520 and not var_name.startswith("\\")
521 and "\\" not in var_type
522 and "\\" not in var_name
523 ):
524 try:
525 # Additional validation before creating Field
526 stripped_name = var_name.strip()
527 stripped_type = var_type.strip()
528 if stripped_name and stripped_type:
529 globals_list.append(
530 Field(
531 name=stripped_name,
532 type=stripped_type,
533 value=var_value,
534 )
535 )
536 self.logger.debug(
537 f"Parsed global: {stripped_name} : {stripped_type}"
538 )
539 except Exception as e:
540 self.logger.warning(
541 f"Error creating global field {var_name}: {e}"
542 )
543 i = self._skip_to_semicolon(tokens, i)
544 else:
545 i += 1
547 return globals_list
549 def _parse_includes_with_tokenizer(self, tokens) -> List[str]:
550 """Parse #include directives using tokenizer"""
551 includes = []
553 for token in tokens:
554 if token.type == TokenType.INCLUDE:
555 match = INCLUDE_FILENAME_RE.search(token.value)
556 if match:
557 includes.append(match.group(1))
559 return includes
561 def _parse_macros_with_tokenizer(self, tokens) -> List[str]:
562 """Parse macro definitions using tokenizer"""
563 macros = []
565 for token in tokens:
566 if token.type == TokenType.DEFINE:
567 # Store the full macro definition for display flexibility
568 # e.g., "#define PI 3.14159" -> "#define PI 3.14159"
569 # e.g., "#define MIN(a, b) ((a) < (b) ? (a) : (b))" -> "#define MIN(a, b) ((a) < (b) ? (a) : (b))"
570 macro_definition = token.value.strip()
571 if macro_definition not in macros:
572 macros.append(macro_definition)
574 return macros
576 def _parse_aliases_with_tokenizer(self, tokens) -> Dict[str, "Alias"]:
577 """Parse type aliases (primitive or derived typedefs) using tokenizer"""
578 from ..models import Alias
580 aliases = {}
582 i = 0
583 while i < len(tokens):
584 if tokens[i].type == TokenType.TYPEDEF:
585 # Found typedef, parse it
586 typedef_info = self._parse_single_typedef(tokens, i)
587 if typedef_info:
588 typedef_name, original_type = typedef_info
590 # Only include if it's NOT a struct/enum/union typedef
591 if original_type not in ["struct", "enum", "union"]:
592 aliases[typedef_name] = Alias(
593 name=typedef_name, original_type=original_type, uses=[]
594 )
596 i += 1
598 return aliases
600 # _parse_typedef_relations_with_tokenizer method removed - tag names are now in struct/enum/union
602 def _extract_tag_name_for_struct(self, tokens, struct_name: str) -> str:
603 """Extract tag name for a struct if it has a typedef"""
604 i = 0
605 while i < len(tokens):
606 if tokens[i].type == TokenType.TYPEDEF:
607 typedef_info = self._parse_single_typedef(tokens, i)
608 if typedef_info:
609 typedef_name, original_type = typedef_info
610 if original_type == "struct" and typedef_name == struct_name:
611 # Extract the tag name from the typedef
612 return self._extract_tag_name_from_typedef(tokens, i)
613 i += 1
614 return ""
616 def _extract_tag_name_for_enum(self, tokens, enum_name: str) -> str:
617 """Extract tag name for an enum if it has a typedef"""
618 i = 0
619 while i < len(tokens):
620 if tokens[i].type == TokenType.TYPEDEF:
621 typedef_info = self._parse_single_typedef(tokens, i)
622 if typedef_info:
623 typedef_name, original_type = typedef_info
624 if original_type == "enum" and typedef_name == enum_name:
625 # Extract the tag name from the typedef
626 return self._extract_tag_name_from_typedef(tokens, i)
627 i += 1
628 return ""
630 def _extract_tag_name_for_union(self, tokens, union_name: str) -> str:
631 """Extract tag name for a union if it has a typedef"""
632 i = 0
633 while i < len(tokens):
634 if tokens[i].type == TokenType.TYPEDEF:
635 typedef_info = self._parse_single_typedef(tokens, i)
636 if typedef_info:
637 typedef_name, original_type = typedef_info
638 if original_type == "union" and typedef_name == union_name:
639 # Extract the tag name from the typedef
640 return self._extract_tag_name_from_typedef(tokens, i)
641 i += 1
642 return ""
644 def _extract_non_primitive_types(
645 self, type_str: str, available_types: Set[str]
646 ) -> List[str]:
647 """Extract non-primitive type names from a type string that exist in available_types"""
648 # Define primitive types
649 primitive_types = {
650 "void",
651 "char",
652 "short",
653 "int",
654 "long",
655 "float",
656 "double",
657 "signed",
658 "unsigned",
659 "const",
660 "volatile",
661 "static",
662 "extern",
663 "auto",
664 "register",
665 "inline",
666 "restrict",
667 "size_t",
668 "ptrdiff_t",
669 "int8_t",
670 "int16_t",
671 "int32_t",
672 "int64_t",
673 "uint8_t",
674 "uint16_t",
675 "uint32_t",
676 "uint64_t",
677 "intptr_t",
678 "uintptr_t",
679 "bool",
680 "true",
681 "false",
682 "NULL",
683 "nullptr",
684 }
686 # Remove common C keywords and operators
687 import re
689 # Split by common delimiters and operators
690 parts = re.split(r"[\[\]\(\)\{\}\s\*&,;]", type_str)
692 # Extract potential type names that exist in available_types
693 types = []
694 for part in parts:
695 part = part.strip()
696 if part and len(part) > 1 and part not in primitive_types:
697 # Check if it looks like a type name (starts with letter, contains letters/numbers/underscores)
698 if re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", part):
699 # Only include if it exists in available_types
700 if part in available_types:
701 types.append(part)
703 return list(set(types)) # Remove duplicates
705 def _find_c_files(
706 self, source_folder_path: Path, recursive_search: bool
707 ) -> List[Path]:
708 """Find all C/C++ files in the source folder"""
709 c_extensions = {".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", ".hxx"}
710 files = []
712 self.logger.debug("Searching for files with extensions: %s", c_extensions)
714 try:
715 if recursive_search:
716 for ext in c_extensions:
717 try:
718 files.extend(source_folder_path.rglob(f"*{ext}"))
719 except (OSError, PermissionError) as e:
720 self.logger.warning("Error during recursive search for %s files: %s", ext, e)
721 # Continue with other extensions
722 else:
723 for ext in c_extensions:
724 try:
725 files.extend(source_folder_path.glob(f"*{ext}"))
726 except (OSError, PermissionError) as e:
727 self.logger.warning("Error during search for %s files: %s", ext, e)
728 # Continue with other extensions
729 except Exception as e:
730 raise OSError(f"Failed to search for C/C++ files in '{source_folder_path}': {e}")
732 # Filter out hidden files and common exclude patterns
733 filtered_files = []
734 exclude_patterns = {".git", "__pycache__", "node_modules", ".vscode", ".idea"}
736 for file_path in files:
737 try:
738 # Skip hidden files and directories
739 if any(part.startswith(".") for part in file_path.parts):
740 continue
742 # Skip common exclude patterns
743 if any(pattern in file_path.parts for pattern in exclude_patterns):
744 continue
746 # Verify the file is actually accessible
747 if not file_path.exists():
748 self.logger.debug("Skipping non-existent file: %s", file_path)
749 continue
751 if not file_path.is_file():
752 self.logger.debug("Skipping non-file item: %s", file_path)
753 continue
755 filtered_files.append(file_path)
756 except (OSError, PermissionError) as e:
757 self.logger.warning("Error accessing file %s: %s", file_path, e)
758 # Skip files we can't access
759 continue
761 self.logger.debug("Found %d C/C++ files after filtering", len(filtered_files))
762 return sorted(filtered_files)
764 def _detect_encoding(self, file_path: Path) -> str:
765 """Detect file encoding with platform-aware fallbacks"""
766 return detect_file_encoding(file_path)
768 def _find_original_token_pos(self, all_tokens, filtered_tokens, filtered_pos):
769 """Find the position in all_tokens that corresponds to filtered_tokens[filtered_pos]"""
770 if filtered_pos >= len(filtered_tokens):
771 return None
773 target_token = filtered_tokens[filtered_pos]
775 # Search for the token in all_tokens by line and column
776 for i, token in enumerate(all_tokens):
777 if (
778 token.line == target_token.line
779 and token.column == target_token.column
780 and token.value == target_token.value
781 ):
782 return i
784 return None
786 def _parse_single_typedef(self, tokens, start_pos):
787 """Parse a single typedef starting at the given position"""
788 # Skip 'typedef' keyword
789 pos = start_pos + 1
791 # Skip whitespace and comments
792 while pos < len(tokens) and tokens[pos].type in [
793 TokenType.WHITESPACE,
794 TokenType.COMMENT,
795 ]:
796 pos += 1
798 if pos >= len(tokens):
799 return None
801 # Check if it's a struct/enum/union typedef
802 if tokens[pos].type in [TokenType.STRUCT, TokenType.ENUM, TokenType.UNION]:
803 # Look ahead to see if this complex type is immediately followed by a function-pointer declarator
804 # Pattern to detect: ... } ( * name ) ( ... )
805 look = pos
806 # Find the matching closing brace of the outer struct/union/enum
807 if tokens[look].type in [TokenType.STRUCT, TokenType.ENUM, TokenType.UNION]:
808 # Advance to the opening brace
809 while look < len(tokens) and tokens[look].type != TokenType.LBRACE:
810 look += 1
811 if look < len(tokens) and tokens[look].type == TokenType.LBRACE:
812 brace_count = 1
813 look += 1
814 while look < len(tokens) and brace_count > 0:
815 if tokens[look].type == TokenType.LBRACE:
816 brace_count += 1
817 elif tokens[look].type == TokenType.RBRACE:
818 brace_count -= 1
819 look += 1
820 # Now 'look' is token after the closing brace
821 j = look
822 # Skip whitespace/comments
823 while j < len(tokens) and tokens[j].type in [TokenType.WHITESPACE, TokenType.COMMENT, TokenType.NEWLINE]:
824 j += 1
825 # Detect function-pointer declarator: ( * IDENT ) (
826 if (
827 j + 4 < len(tokens)
828 and tokens[j].type == TokenType.LPAREN
829 and tokens[j + 1].type == TokenType.ASTERISK
830 and tokens[j + 2].type == TokenType.IDENTIFIER
831 and tokens[j + 3].type == TokenType.RPAREN
832 and tokens[j + 4].type == TokenType.LPAREN
833 ):
834 typedef_name = tokens[j + 2].value
835 # Collect the full typedef original type up to the semicolon, preserving parentheses/brackets spacing
836 k = pos
837 formatted: list[str] = []
838 while k < len(tokens) and tokens[k].type != TokenType.SEMICOLON:
839 t = tokens[k]
840 if t.type in [TokenType.LPAREN, TokenType.RPAREN, TokenType.LBRACKET, TokenType.RBRACKET]:
841 formatted.append(t.value)
842 elif formatted and formatted[-1] not in ["(", ")", "[", "]"]:
843 # Prepend space before non-bracket tokens when previous isn't a bracket
844 formatted.append(" " + t.value)
845 else:
846 formatted.append(t.value)
847 k += 1
848 original_type = "".join(formatted)
849 # Clean excessive whitespace inside type
850 original_type = self._clean_type_string(original_type)
851 return (typedef_name, original_type)
852 # Fallback to standard complex typedef parsing
853 return self._parse_complex_typedef(tokens, pos)
855 # Collect all non-whitespace/comment tokens until semicolon
856 # But handle nested structures properly
857 all_tokens = []
858 brace_count = 0
859 paren_count = 0
861 while pos < len(tokens):
862 token = tokens[pos]
864 # Track nested braces and parentheses
865 if token.type == TokenType.LBRACE:
866 brace_count += 1
867 elif token.type == TokenType.RBRACE:
868 brace_count -= 1
869 elif token.type == TokenType.LPAREN:
870 paren_count += 1
871 elif token.type == TokenType.RPAREN:
872 paren_count -= 1
873 elif token.type == TokenType.SEMICOLON:
874 # Only treat semicolon as end if we're not inside nested structures
875 # For function pointer typedefs, we need to be outside the parameter list parentheses
876 if brace_count == 0 and paren_count == 0:
877 # We're outside any nested structures and parentheses
878 break
880 if token.type not in [TokenType.WHITESPACE, TokenType.COMMENT]:
881 all_tokens.append(token)
882 pos += 1
884 if len(all_tokens) < 2:
885 return None
887 # Function pointer typedef: typedef ret (*name)(params);
888 for i in range(len(all_tokens) - 3):
889 if (
890 all_tokens[i].type
891 in [
892 TokenType.IDENTIFIER,
893 TokenType.INT,
894 TokenType.VOID,
895 TokenType.CHAR,
896 TokenType.FLOAT,
897 TokenType.DOUBLE,
898 TokenType.LONG,
899 TokenType.SHORT,
900 TokenType.UNSIGNED,
901 TokenType.SIGNED,
902 ]
903 and all_tokens[i + 1].type == TokenType.LPAREN
904 and all_tokens[i + 2].type == TokenType.ASTERISK
905 and all_tokens[i + 3].type == TokenType.IDENTIFIER
906 ):
907 # Check if this is followed by a parameter list
908 if i + 4 < len(all_tokens) and all_tokens[i + 4].type == TokenType.RPAREN:
909 if i + 5 < len(all_tokens) and all_tokens[i + 5].type == TokenType.LPAREN:
910 # This is a function pointer with parameters - skip this pattern and use the complex logic
911 break
913 # Simple function pointer typedef without complex parameters
914 typedef_name = all_tokens[i + 3].value
915 # Fix: Properly format function pointer type - preserve spaces between tokens but not around parentheses
916 formatted_tokens = []
917 for j, token in enumerate(all_tokens):
918 if token.type in [TokenType.LPAREN, TokenType.RPAREN]:
919 # Don't add spaces around parentheses
920 formatted_tokens.append(token.value)
921 elif j > 0 and all_tokens[j - 1].type not in [
922 TokenType.LPAREN,
923 TokenType.RPAREN,
924 ]:
925 # Add space before token if previous token wasn't a parenthesis
926 formatted_tokens.append(" " + token.value)
927 else:
928 # No space before token
929 formatted_tokens.append(token.value)
930 original_type = "".join(formatted_tokens)
931 return (typedef_name, original_type)
933 # Complex function pointer typedef: typedef ret (*name)(complex_params);
934 # This handles cases where the function pointer has complex parameters that span multiple tokens
935 if len(all_tokens) >= 6:
936 # Look for pattern: type ( * name ) ( ... )
937 for i in range(len(all_tokens) - 5):
938 if (
939 all_tokens[i].type
940 in [
941 TokenType.IDENTIFIER,
942 TokenType.INT,
943 TokenType.VOID,
944 TokenType.CHAR,
945 TokenType.FLOAT,
946 TokenType.DOUBLE,
947 TokenType.LONG,
948 TokenType.SHORT,
949 TokenType.UNSIGNED,
950 TokenType.SIGNED,
951 ]
952 and all_tokens[i + 1].type == TokenType.LPAREN
953 and all_tokens[i + 2].type == TokenType.ASTERISK
954 and all_tokens[i + 3].type == TokenType.IDENTIFIER
955 and all_tokens[i + 4].type == TokenType.RPAREN
956 and all_tokens[i + 5].type == TokenType.LPAREN
957 ):
958 # Find the closing parenthesis for the parameter list
959 paren_count = 1
960 param_end = i + 6
961 while param_end < len(all_tokens) and paren_count > 0:
962 if all_tokens[param_end].type == TokenType.LPAREN:
963 paren_count += 1
964 elif all_tokens[param_end].type == TokenType.RPAREN:
965 paren_count -= 1
966 param_end += 1
968 if paren_count == 0:
969 typedef_name = all_tokens[i + 3].value
970 # Format the complete typedef properly
971 formatted_tokens = []
972 for j, token in enumerate(all_tokens):
973 if token.type in [TokenType.LPAREN, TokenType.RPAREN]:
974 # Don't add spaces around parentheses
975 formatted_tokens.append(token.value)
976 elif j > 0 and all_tokens[j - 1].type not in [
977 TokenType.LPAREN,
978 TokenType.RPAREN,
979 ]:
980 # Add space before token if previous token wasn't a parenthesis
981 formatted_tokens.append(" " + token.value)
982 else:
983 # No space before token
984 formatted_tokens.append(token.value)
985 original_type = "".join(formatted_tokens)
986 return (typedef_name, original_type)
988 # Array typedef: typedef type name[size];
989 for i in range(len(all_tokens)):
990 if (
991 all_tokens[i].type == TokenType.LBRACKET
992 and i > 0
993 and all_tokens[i - 1].type == TokenType.IDENTIFIER
994 ):
995 typedef_name = all_tokens[i - 1].value
996 # Fix: Properly format array type - preserve spaces between tokens but not around brackets
997 formatted_tokens = []
998 for j, token in enumerate(all_tokens):
999 if token.type in [TokenType.LBRACKET, TokenType.RBRACKET]:
1000 # Don't add spaces around brackets
1001 formatted_tokens.append(token.value)
1002 elif j > 0 and all_tokens[j - 1].type not in [
1003 TokenType.LBRACKET,
1004 TokenType.RBRACKET,
1005 ]:
1006 # Add space before token if previous token wasn't a bracket
1007 formatted_tokens.append(" " + token.value)
1008 else:
1009 # No space before token
1010 formatted_tokens.append(token.value)
1011 original_type = "".join(formatted_tokens)
1012 return (typedef_name, original_type)
1014 # Pointer typedef: typedef type * name;
1015 for i in range(len(all_tokens) - 2):
1016 if (
1017 all_tokens[i].type == TokenType.ASTERISK
1018 and all_tokens[i + 1].type == TokenType.IDENTIFIER
1019 ):
1020 typedef_name = all_tokens[i + 1].value
1021 # Fix: Properly format pointer type - preserve spaces between tokens
1022 formatted_tokens = []
1023 for j, token in enumerate(all_tokens):
1024 if j > 0:
1025 # Add space before token
1026 formatted_tokens.append(" " + token.value)
1027 else:
1028 # No space before first token
1029 formatted_tokens.append(token.value)
1030 original_type = "".join(formatted_tokens)
1031 return (typedef_name, original_type)
1033 # Basic typedef: the last token is the typedef name, everything else is the type
1034 typedef_name = all_tokens[-1].value
1035 type_tokens = all_tokens[:-1]
1036 original_type = " ".join(t.value for t in type_tokens)
1037 original_type = self._clean_type_string(original_type)
1038 original_type = self._fix_array_bracket_spacing(original_type)
1039 return (typedef_name, original_type)
1041 def _parse_complex_typedef(self, tokens, start_pos):
1042 """Parse complex typedef (struct/enum/union)"""
1043 # Parse complex typedefs with proper structure detection
1045 # Find the typedef name by looking for the pattern after the closing brace
1046 brace_count = 0
1047 pos = start_pos
1049 # Find opening brace
1050 while pos < len(tokens) and tokens[pos].type != TokenType.LBRACE:
1051 pos += 1
1053 if pos >= len(tokens):
1054 return None
1056 # Skip to closing brace
1057 brace_count = 1
1058 pos += 1
1060 while pos < len(tokens) and brace_count > 0:
1061 if tokens[pos].type == TokenType.LBRACE:
1062 brace_count += 1
1063 elif tokens[pos].type == TokenType.RBRACE:
1064 brace_count -= 1
1065 pos += 1
1067 if brace_count > 0:
1068 return None
1070 # Find typedef name after closing brace
1071 while pos < len(tokens) and tokens[pos].type in [
1072 TokenType.WHITESPACE,
1073 TokenType.COMMENT,
1074 ]:
1075 pos += 1
1077 if pos < len(tokens) and tokens[pos].type == TokenType.IDENTIFIER:
1078 typedef_name = tokens[pos].value
1079 struct_type = tokens[start_pos].value # struct/enum/union
1080 return (typedef_name, struct_type)
1082 return None
1084 def _extract_tag_name_from_typedef(self, tokens, start_pos):
1085 """Extract the tag name from a typedef like 'typedef struct TagName { ... } TypedefName;'"""
1086 # Skip 'typedef' keyword
1087 pos = start_pos + 1
1089 # Skip whitespace and comments
1090 while pos < len(tokens) and tokens[pos].type in [
1091 TokenType.WHITESPACE,
1092 TokenType.COMMENT,
1093 ]:
1094 pos += 1
1096 if pos >= len(tokens):
1097 return ""
1099 # Check if it's a struct/enum/union
1100 if tokens[pos].type not in [TokenType.STRUCT, TokenType.ENUM, TokenType.UNION]:
1101 return ""
1103 # Skip struct/enum/union keyword
1104 pos += 1
1106 # Skip whitespace and comments
1107 while pos < len(tokens) and tokens[pos].type in [
1108 TokenType.WHITESPACE,
1109 TokenType.COMMENT,
1110 ]:
1111 pos += 1
1113 # Look for tag name (identifier before opening brace)
1114 if pos < len(tokens) and tokens[pos].type == TokenType.IDENTIFIER:
1115 tag_name = tokens[pos].value
1116 return tag_name
1118 return ""
1120 def _looks_like_function(self, tokens, start_pos):
1121 """Check if the token sequence starting at start_pos looks like a function"""
1122 # Look ahead for parentheses within a reasonable distance
1123 for i in range(start_pos, min(start_pos + 10, len(tokens))):
1124 if tokens[i].type == TokenType.LPAREN:
1125 return True
1126 if tokens[i].type in [
1127 TokenType.SEMICOLON,
1128 TokenType.LBRACE,
1129 TokenType.RBRACE,
1130 ]:
1131 return False
1132 return False
1134 def _skip_function(self, tokens, start_pos):
1135 """Skip over a function definition or declaration"""
1136 # Find the end (either semicolon for declaration or closing brace for definition)
1137 i = start_pos
1138 brace_count = 0
1139 paren_count = 0
1141 while i < len(tokens):
1142 if tokens[i].type == TokenType.LPAREN:
1143 paren_count += 1
1144 elif tokens[i].type == TokenType.RPAREN:
1145 paren_count -= 1
1146 elif tokens[i].type == TokenType.LBRACE:
1147 brace_count += 1
1148 elif tokens[i].type == TokenType.RBRACE:
1149 brace_count -= 1
1150 if brace_count == 0 and paren_count == 0:
1151 return i + 1
1152 elif (
1153 tokens[i].type == TokenType.SEMICOLON
1154 and paren_count == 0
1155 and brace_count == 0
1156 ):
1157 return i + 1
1158 i += 1
1160 return i
1162 def _skip_structure_definition(self, tokens, start_pos):
1163 """Skip over struct/enum/union/typedef definition"""
1164 i = start_pos
1165 brace_count = 0
1167 while i < len(tokens):
1168 if tokens[i].type == TokenType.LBRACE:
1169 brace_count += 1
1170 elif tokens[i].type == TokenType.RBRACE:
1171 brace_count -= 1
1172 if brace_count == 0:
1173 # Continue until semicolon
1174 while i < len(tokens) and tokens[i].type != TokenType.SEMICOLON:
1175 i += 1
1176 return i + 1 if i < len(tokens) else i
1177 elif tokens[i].type == TokenType.SEMICOLON and brace_count == 0:
1178 return i + 1
1179 i += 1
1181 return i
1183 def _parse_global_variable(self, tokens, start_pos):
1184 """Parse a global variable declaration starting at start_pos"""
1185 # Look for pattern: [static/extern] type name [= value];
1186 i = start_pos
1187 collected_tokens = []
1189 # Collect tokens until semicolon
1190 while i < len(tokens) and tokens[i].type != TokenType.SEMICOLON:
1191 if tokens[i].type not in [TokenType.WHITESPACE, TokenType.COMMENT]:
1192 collected_tokens.append(tokens[i])
1193 i += 1
1195 if len(collected_tokens) < 2:
1196 return None
1198 # Skip modifiers
1199 start_idx = 0
1200 while start_idx < len(collected_tokens) and collected_tokens[
1201 start_idx
1202 ].type in [TokenType.STATIC, TokenType.EXTERN, TokenType.CONST]:
1203 start_idx += 1
1205 # Check if there's an assignment
1206 assign_idx = None
1207 for j in range(start_idx, len(collected_tokens)):
1208 if collected_tokens[j].type == TokenType.ASSIGN:
1209 assign_idx = j
1210 break
1212 # Extract variable name and type
1213 if assign_idx is not None:
1214 # Has assignment: type name = value or type name[size] = value
1215 if assign_idx > start_idx + 1:
1216 # Check if this is an array declaration with assignment
1217 bracket_idx = None
1218 for j in range(assign_idx - 1, start_idx, -1):
1219 if collected_tokens[j].type == TokenType.RBRACKET:
1220 bracket_idx = j
1221 break
1223 if bracket_idx is not None:
1224 # Array declaration with assignment: find the identifier before the first '['
1225 # First, find the matching '[' for this last bracket
1226 for j in range(bracket_idx - 1, start_idx, -1):
1227 if collected_tokens[j].type == TokenType.LBRACKET:
1228 # Found the first '[' of the trailing bracket groups; now find the identifier before it
1229 for k in range(j - 1, start_idx, -1):
1230 if collected_tokens[k].type == TokenType.IDENTIFIER:
1231 var_name = collected_tokens[k].value
1232 type_tokens = collected_tokens[start_idx:k]
1233 # Format base type preserving spaces
1234 formatted_type = []
1235 for idx, token in enumerate(type_tokens):
1236 if idx > 0:
1237 formatted_type.append(" " + token.value)
1238 else:
1239 formatted_type.append(token.value)
1240 base_type = "".join(formatted_type)
1241 # Collect all trailing [size] groups between name and '=' using shared helper
1242 dims, _n = collect_array_dimensions_from_tokens(collected_tokens[:assign_idx], k + 1)
1243 dims = [normalize_dim_value(d) for d in dims]
1244 var_type = join_type_with_dims(base_type, dims)
1245 var_type = self._clean_type_string(var_type)
1246 value_tokens = collected_tokens[assign_idx + 1 :]
1247 var_value = " ".join(t.value for t in value_tokens)
1248 var_value = self._clean_value_string(var_value)
1249 return (var_name, var_type, var_value)
1250 break
1251 else:
1252 # Regular assignment: type name = value
1253 var_name = collected_tokens[assign_idx - 1].value
1254 type_tokens = collected_tokens[start_idx : assign_idx - 1]
1255 value_tokens = collected_tokens[assign_idx + 1 :]
1256 var_type = " ".join(t.value for t in type_tokens)
1257 var_type = self._clean_type_string(var_type)
1258 var_type = self._fix_array_bracket_spacing(var_type)
1259 var_value = " ".join(t.value for t in value_tokens)
1260 # Clean the value string to remove excessive whitespace and newlines
1261 var_value = self._clean_value_string(var_value)
1262 return (var_name, var_type, var_value)
1263 else:
1264 # No assignment: type name or type name[size][size]...
1265 if len(collected_tokens) > start_idx + 1:
1266 # Check if this is an array declaration
1267 bracket_idx = None
1268 for j in range(len(collected_tokens) - 1, start_idx, -1):
1269 if collected_tokens[j].type == TokenType.RBRACKET:
1270 bracket_idx = j
1271 break
1273 if bracket_idx is not None:
1274 # Array declaration: find the identifier before the first '[' and collect all dims
1275 for j in range(bracket_idx - 1, start_idx, -1):
1276 if collected_tokens[j].type == TokenType.LBRACKET:
1277 # Found the first '[' of the trailing bracket groups; look for identifier before it
1278 for k in range(j - 1, start_idx, -1):
1279 if collected_tokens[k].type == TokenType.IDENTIFIER:
1280 var_name = collected_tokens[k].value
1281 type_tokens = collected_tokens[start_idx:k]
1282 # Format base type preserving spaces
1283 formatted_type = []
1284 for idx2, token in enumerate(type_tokens):
1285 if idx2 > 0:
1286 formatted_type.append(" " + token.value)
1287 else:
1288 formatted_type.append(token.value)
1289 base_type = "".join(formatted_type)
1290 # Collect all trailing [size] groups after the name using shared helper
1291 dims, _n = collect_array_dimensions_from_tokens(collected_tokens, k + 1)
1292 dims = [normalize_dim_value(d) for d in dims]
1293 var_type = join_type_with_dims(base_type, dims)
1294 var_type = self._clean_type_string(var_type)
1295 return (var_name, var_type, None)
1296 break
1297 else:
1298 # Regular variable: last token is the name
1299 var_name = collected_tokens[-1].value
1300 type_tokens = collected_tokens[start_idx:-1]
1301 var_type = " ".join(t.value for t in type_tokens)
1302 var_type = self._clean_type_string(var_type)
1303 var_type = self._fix_array_bracket_spacing(var_type)
1304 return (var_name, var_type, None)
1306 return None
1308 def _skip_to_semicolon(self, tokens, start_pos):
1309 """Skip to the next semicolon"""
1310 i = start_pos
1311 while i < len(tokens) and tokens[i].type != TokenType.SEMICOLON:
1312 i += 1
1313 return i + 1 if i < len(tokens) else i
1315 def _skip_preprocessor_directives(self, tokens, start_pos):
1316 """Skip preprocessor directives but keep their content for parsing"""
1317 # This method is deprecated - use the PreprocessorManager instead
1318 i = start_pos
1319 while i < len(tokens) and tokens[i].type == TokenType.PREPROCESSOR:
1320 # Skip the preprocessor directive itself
1321 i += 1
1322 return i
1324 def _parse_function_parameters(self, tokens, start_pos, end_pos, func_name):
1325 """Parse function parameters from token range"""
1327 parameters = []
1329 # Find the opening parenthesis for the function
1330 paren_start = None
1331 paren_end = None
1333 for i in range(start_pos, min(end_pos + 1, len(tokens))):
1334 if tokens[i].type == TokenType.IDENTIFIER and tokens[i].value == func_name:
1335 # Look for opening parenthesis after function name
1336 for j in range(i + 1, min(end_pos + 1, len(tokens))):
1337 if tokens[j].type == TokenType.LPAREN:
1338 paren_start = j
1339 break
1340 elif tokens[j].type not in [
1341 TokenType.WHITESPACE,
1342 TokenType.COMMENT,
1343 ]:
1344 break
1345 break
1347 if paren_start is None:
1348 return parameters
1350 # Find matching closing parenthesis
1351 paren_depth = 1
1352 for i in range(paren_start + 1, min(end_pos + 1, len(tokens))):
1353 if tokens[i].type == TokenType.LPAREN:
1354 paren_depth += 1
1355 elif tokens[i].type == TokenType.RPAREN:
1356 paren_depth -= 1
1357 if paren_depth == 0:
1358 paren_end = i
1359 break
1361 if paren_end is None:
1362 return parameters
1364 # Parse parameter tokens between parentheses
1365 param_tokens = []
1366 for i in range(paren_start + 1, paren_end):
1367 if tokens[i].type not in [TokenType.WHITESPACE, TokenType.COMMENT, TokenType.NEWLINE]:
1368 param_tokens.append(tokens[i])
1370 # If no parameters or just "void", return empty list
1371 if not param_tokens or (
1372 len(param_tokens) == 1 and param_tokens[0].value == "void"
1373 ):
1374 return parameters
1376 # Split parameters by commas, but handle function pointers correctly
1377 current_param = []
1378 paren_depth = 0
1379 for token in param_tokens:
1380 if token.type == TokenType.LPAREN:
1381 paren_depth += 1
1382 elif token.type == TokenType.RPAREN:
1383 paren_depth -= 1
1384 elif token.type == TokenType.COMMA and paren_depth == 0:
1385 # Only split on commas that are not inside parentheses
1386 if current_param:
1387 param = self._parse_single_parameter(current_param)
1388 if param:
1389 parameters.append(param)
1390 current_param = []
1391 continue
1393 current_param.append(token)
1395 # Handle last parameter
1396 if current_param:
1397 param = self._parse_single_parameter(current_param)
1398 if param:
1399 parameters.append(param)
1401 return parameters
1403 def _parse_single_parameter(self, param_tokens):
1404 """Parse a single function parameter from tokens"""
1405 from ..models import Field
1407 if not param_tokens:
1408 return None
1410 # Handle variadic parameters (three consecutive dots)
1411 if len(param_tokens) == 3 and all(t.value == "." for t in param_tokens):
1412 return Field(name="...", type="...")
1414 # Handle variadic parameters (single ... token)
1415 if len(param_tokens) == 1 and param_tokens[0].value == "...":
1416 return Field(name="...", type="...")
1418 # Handle function pointer parameters: type (*name)(params)
1419 if len(param_tokens) >= 5:
1420 # Look for pattern: type ( * name ) ( params )
1421 for i in range(len(param_tokens) - 4):
1422 if (
1423 param_tokens[i].type == TokenType.LPAREN
1424 and param_tokens[i + 1].type == TokenType.ASTERISK
1425 and param_tokens[i + 2].type == TokenType.IDENTIFIER
1426 and param_tokens[i + 3].type == TokenType.RPAREN
1427 and param_tokens[i + 4].type == TokenType.LPAREN
1428 ):
1429 # Found function pointer pattern
1430 func_name = param_tokens[i + 2].value
1432 # Find the closing parenthesis for the parameter list
1433 paren_count = 1
1434 param_end = i + 5
1435 while param_end < len(param_tokens) and paren_count > 0:
1436 if param_tokens[param_end].type == TokenType.LPAREN:
1437 paren_count += 1
1438 elif param_tokens[param_end].type == TokenType.RPAREN:
1439 paren_count -= 1
1440 param_end += 1
1442 if paren_count == 0:
1443 # Extract the type (everything before the function pointer)
1444 type_tokens = param_tokens[:i]
1445 param_type = " ".join(t.value for t in type_tokens)
1447 # Extract the function pointer part
1448 func_ptr_tokens = param_tokens[i:param_end]
1449 func_ptr_type = " ".join(t.value for t in func_ptr_tokens)
1451 # Combine type and function pointer
1452 full_type = (param_type + " " + func_ptr_type).strip()
1454 # Fix array bracket spacing
1455 full_type = self._fix_array_bracket_spacing(full_type)
1457 return Field(name=func_name, type=full_type)
1458 else:
1459 # Incomplete function pointer - try to reconstruct
1460 type_tokens = param_tokens[:i]
1461 param_type = " ".join(t.value for t in type_tokens)
1462 func_ptr_tokens = param_tokens[i:]
1463 func_ptr_type = " ".join(t.value for t in func_ptr_tokens)
1464 full_type = (param_type + " " + func_ptr_type).strip()
1465 full_type = self._fix_array_bracket_spacing(full_type)
1466 return Field(name=func_name, type=full_type)
1468 # Also look for pattern: type ( * name ) ( params ) with spaces
1469 for i in range(len(param_tokens) - 4):
1470 if (
1471 param_tokens[i].type == TokenType.LPAREN
1472 and param_tokens[i + 1].type == TokenType.ASTERISK
1473 and param_tokens[i + 2].type == TokenType.IDENTIFIER
1474 and param_tokens[i + 3].type == TokenType.RPAREN
1475 and param_tokens[i + 4].type == TokenType.LPAREN
1476 ):
1477 # Found function pointer pattern
1478 func_name = param_tokens[i + 2].value
1480 # Find the closing parenthesis for the parameter list
1481 paren_count = 1
1482 param_end = i + 5
1483 while param_end < len(param_tokens) and paren_count > 0:
1484 if param_tokens[param_end].type == TokenType.LPAREN:
1485 paren_count += 1
1486 elif param_tokens[param_end].type == TokenType.RPAREN:
1487 paren_count -= 1
1488 param_end += 1
1490 if paren_count == 0:
1491 # Extract the type (everything before the function pointer)
1492 type_tokens = param_tokens[:i]
1493 param_type = " ".join(t.value for t in type_tokens)
1495 # Extract the function pointer part
1496 func_ptr_tokens = param_tokens[i:param_end]
1497 func_ptr_type = " ".join(t.value for t in func_ptr_tokens)
1499 # Combine type and function pointer
1500 full_type = (param_type + " " + func_ptr_type).strip()
1502 # Fix array bracket spacing
1503 full_type = self._fix_array_bracket_spacing(full_type)
1505 return Field(name=func_name, type=full_type)
1506 else:
1507 # Incomplete function pointer - try to reconstruct
1508 type_tokens = param_tokens[:i]
1509 param_type = " ".join(t.value for t in type_tokens)
1510 func_ptr_tokens = param_tokens[i:]
1511 func_ptr_type = " ".join(t.value for t in func_ptr_tokens)
1512 full_type = (param_type + " " + func_ptr_type).strip()
1513 full_type = self._fix_array_bracket_spacing(full_type)
1514 return Field(name=func_name, type=full_type)
1516 # For parameters like "int x" or "const char *name" or "char* argv[]"
1517 if len(param_tokens) >= 2:
1518 # Check if the last token is a closing bracket (array parameter)
1519 if param_tokens[-1].type == TokenType.RBRACKET:
1520 # Find the opening bracket to get the array size
1521 bracket_start = None
1522 for i in range(len(param_tokens) - 1, -1, -1):
1523 if param_tokens[i].type == TokenType.LBRACKET:
1524 bracket_start = i
1525 break
1527 if bracket_start is not None:
1528 # Extract the parameter name (last identifier before the opening bracket)
1529 param_name = None
1530 for i in range(bracket_start - 1, -1, -1):
1531 if param_tokens[i].type == TokenType.IDENTIFIER:
1532 param_name = param_tokens[i].value
1533 break
1535 if param_name:
1536 # Extract the type (everything before the parameter name)
1537 type_tokens = param_tokens[:i]
1538 param_type = " ".join(t.value for t in type_tokens)
1540 # Add the array brackets to the type
1541 array_size = ""
1542 if bracket_start + 1 < len(param_tokens) - 1:
1543 # There's content between brackets
1544 array_content = param_tokens[bracket_start + 1:-1]
1545 array_size = " ".join(t.value for t in array_content)
1547 param_type = param_type + "[" + array_size + "]"
1549 # Fix array bracket spacing
1550 param_type = self._fix_array_bracket_spacing(param_type)
1552 return Field(name=param_name, type=param_type)
1553 else:
1554 # Regular parameter: last token is the parameter name
1555 param_name = param_tokens[-1].value
1556 type_tokens = param_tokens[:-1]
1557 param_type = " ".join(t.value for t in type_tokens)
1559 # Fix array bracket spacing and pointer spacing
1560 param_type = self._fix_array_bracket_spacing(param_type)
1561 param_type = self._fix_pointer_spacing(param_type)
1563 # Handle unnamed parameters (just type)
1564 if param_name in [
1565 "void",
1566 "int",
1567 "char",
1568 "float",
1569 "double",
1570 "long",
1571 "short",
1572 "unsigned",
1573 "signed",
1574 ]:
1575 # This is just a type without a name
1576 return Field(name="unnamed", type=param_type + " " + param_name)
1578 # Additional validation before creating Field
1579 if param_name and param_name.strip() and param_type and param_type.strip():
1580 return Field(name=param_name.strip(), type=param_type.strip())
1581 else:
1582 # Fallback for invalid parameters - try to reconstruct the full parameter
1583 full_param = " ".join(t.value for t in param_tokens)
1584 full_param = self._fix_array_bracket_spacing(full_param)
1585 if full_param.strip():
1586 return Field(name="unnamed", type=full_param.strip())
1587 else:
1588 return Field(name="unnamed", type="unknown")
1589 elif len(param_tokens) == 1:
1590 # Single token - might be just type (like "void") or name
1591 token_value = param_tokens[0].value
1592 if token_value in [
1593 "void",
1594 "int",
1595 "char",
1596 "float",
1597 "double",
1598 "long",
1599 "short",
1600 "unsigned",
1601 "signed",
1602 ]:
1603 return Field(name="unnamed", type=token_value)
1604 else:
1605 # If we can't determine the type, use the token value as type
1606 if token_value and token_value.strip():
1607 return Field(name="unnamed", type=token_value.strip())
1608 else:
1609 return Field(name="unnamed", type="unknown")
1611 return None
1613 def _fix_array_bracket_spacing(self, type_str: str) -> str:
1614 """Fix spacing around array brackets in type strings"""
1615 return fix_array_bracket_spacing(type_str)
1617 def _fix_pointer_spacing(self, type_str: str) -> str:
1618 """Fix spacing around pointer asterisks in type strings"""
1619 return fix_pointer_spacing(type_str)
1621 def _clean_type_string(self, type_str: str) -> str:
1622 """Clean type string by removing newlines and normalizing whitespace"""
1623 return clean_type_string(type_str)
1625 def _clean_value_string(self, value_str: str) -> str:
1626 """Clean value string by removing excessive whitespace and newlines"""
1627 return clean_value_string(value_str)
1629 # _get_timestamp helper removed as unused to reduce surface area
1632class Parser:
1633 """Main parser class for Step 1: Parse C code files and generate model.json"""
1635 def __init__(self):
1636 self.c_parser = CParser()
1637 self.logger = logging.getLogger(__name__)
1639 def parse(
1640 self,
1641 source_folders: "List[str]",
1642 output_file: str = "model.json",
1643 recursive_search: bool = True,
1644 config: "Config" = None,
1645 ) -> str:
1646 """Parse C/C++ projects and generate model.json
1648 Args:
1649 source_folders: List of source folder directories within the project
1650 output_file: Path to the output model.json file
1651 recursive_search: Whether to search subdirectories recursively
1652 config: Configuration object for filtering and processing
1654 Returns:
1655 Path to the generated model.json file
1656 """
1657 # Enhanced validation for source_folders
1658 if not isinstance(source_folders, list):
1659 raise TypeError(f"source_folders must be a list of strings, got: {type(source_folders)}")
1661 if not source_folders:
1662 raise ValueError("At least one source folder must be provided")
1664 # Validate all items are strings and not empty
1665 for i, folder in enumerate(source_folders):
1666 if not isinstance(folder, str):
1667 raise TypeError(f"All source folders must be strings, got {type(folder)} at index {i}: {folder}")
1668 if not folder.strip():
1669 raise ValueError(f"Source folder at index {i} cannot be empty or whitespace: {repr(folder)}")
1671 self.logger.info(
1672 f"Step 1: Parsing C/C++ project with {len(source_folders)} source folders"
1673 )
1675 # Get project name from config or use default
1676 project_name = (
1677 getattr(config, "project_name", "C_Project") if config else "C_Project"
1678 )
1680 # Parse each source folder and combine results
1681 all_files = {}
1682 total_structs = 0
1683 total_enums = 0
1684 total_functions = 0
1685 failed_folders = []
1687 for i, source_folder in enumerate(source_folders):
1688 self.logger.info(
1689 f"Parsing source folder {i+1}/{len(source_folders)}: {source_folder}"
1690 )
1692 try:
1693 # Parse the individual source folder
1694 model = self.c_parser.parse_project(
1695 source_folder, recursive_search, config
1696 )
1698 all_files.update(model.files)
1700 # Update totals
1701 total_structs += sum(len(f.structs) for f in model.files.values())
1702 total_enums += sum(len(f.enums) for f in model.files.values())
1703 total_functions += sum(len(f.functions) for f in model.files.values())
1705 self.logger.info(
1706 f"Successfully parsed source folder {source_folder}: {len(model.files)} files"
1707 )
1709 except Exception as e:
1710 self.logger.error(
1711 "Failed to parse source folder %s: %s", source_folder, e
1712 )
1713 failed_folders.append((source_folder, str(e)))
1715 # If this is the only source folder, re-raise the error
1716 if len(source_folders) == 1:
1717 raise
1719 # For multiple source folders, continue with others but log the failure
1720 self.logger.warning(
1721 "Continuing with other source folders despite failure in %s", source_folder
1722 )
1724 # If all source folders failed, raise an error
1725 if failed_folders and len(failed_folders) == len(source_folders):
1726 error_msg = "All source folders failed to parse:\n"
1727 for folder, error in failed_folders:
1728 error_msg += f" - {folder}: {error}\n"
1729 raise RuntimeError(error_msg)
1731 # If some folders failed, log a warning
1732 if failed_folders:
1733 self.logger.warning(
1734 f"Failed to parse {len(failed_folders)} out of {len(source_folders)} source folders"
1735 )
1737 # Create combined project model
1738 combined_model = ProjectModel(
1739 project_name=project_name,
1740 source_folder=(
1741 ",".join(source_folders)
1742 if len(source_folders) > 1
1743 else source_folders[0]
1744 ),
1745 files=all_files,
1746 )
1748 # Update all uses fields across the entire combined project
1749 combined_model.update_uses_fields()
1751 # Save combined model to JSON file
1752 try:
1753 combined_model.save(output_file)
1754 except Exception as e:
1755 raise RuntimeError(f"Failed to save model to {output_file}: {e}") from e
1757 # Step 1.5: Verify model sanity
1758 self.logger.info("Step 1.5: Verifying model sanity...")
1759 from .verifier import ModelVerifier
1761 verifier = ModelVerifier()
1762 is_valid, issues = verifier.verify_model(combined_model)
1764 if not is_valid:
1765 self.logger.warning(
1766 f"Model verification found {len(issues)} issues - model may contain parsing errors"
1767 )
1768 # Continue processing but warn about potential issues
1769 else:
1770 self.logger.info("Model verification passed - all values look sane")
1772 self.logger.info("Step 1 complete! Model saved to: %s", output_file)
1773 self.logger.info(
1774 f"Found {len(all_files)} total files across {len(source_folders)} source folder(s)"
1775 )
1777 # Print summary
1778 self.logger.info(
1779 f"Summary: {total_structs} structs, {total_enums} enums, "
1780 f"{total_functions} functions"
1781 )
1783 return output_file