Coverage for src/c2puml/core/parser_tokenizer.py: 88%
876 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-09-14 18:23 +0000
1#!/usr/bin/env python3
2"""
3Tokenizer module for C to PlantUML converter - Helper library for tokenizing C/C++ code
4"""
6import logging
7import re
8from dataclasses import dataclass
9from enum import Enum
10from typing import List, Optional, Tuple
13class TokenType(Enum):
14 """Token types for C/C++ lexical analysis"""
16 # Keywords
17 STRUCT = "STRUCT"
18 ENUM = "ENUM"
19 UNION = "UNION"
20 TYPEDEF = "TYPEDEF"
21 STATIC = "STATIC"
22 EXTERN = "EXTERN"
23 INLINE = "INLINE"
24 LOCAL_INLINE = "LOCAL_INLINE"
25 CONST = "CONST"
26 VOID = "VOID"
28 # Data types
29 CHAR = "CHAR"
30 INT = "INT"
31 FLOAT = "FLOAT"
32 DOUBLE = "DOUBLE"
33 LONG = "LONG"
34 SHORT = "SHORT"
35 UNSIGNED = "UNSIGNED"
36 SIGNED = "SIGNED"
38 # Operators and punctuation
39 LBRACE = "LBRACE" # {
40 RBRACE = "RBRACE" # }
41 LPAREN = "LPAREN" # (
42 RPAREN = "RPAREN" # )
43 LBRACKET = "LBRACKET" # [
44 RBRACKET = "RBRACKET" # ]
45 SEMICOLON = "SEMICOLON" # ;
46 COMMA = "COMMA" # ,
47 ASSIGN = "ASSIGN" # =
48 ASTERISK = "ASTERISK" # *
49 AMPERSAND = "AMPERSAND" # &
50 ARROW = "ARROW" # ->
52 # Literals and identifiers
53 IDENTIFIER = "IDENTIFIER"
54 NUMBER = "NUMBER"
55 STRING = "STRING"
56 CHAR_LITERAL = "CHAR_LITERAL"
58 # Preprocessor
59 INCLUDE = "INCLUDE"
60 DEFINE = "DEFINE"
61 PREPROCESSOR = "PREPROCESSOR"
63 # Special
64 COMMENT = "COMMENT"
65 WHITESPACE = "WHITESPACE"
66 NEWLINE = "NEWLINE"
67 EOF = "EOF"
68 UNKNOWN = "UNKNOWN"
71@dataclass
72class Token:
73 """Represents a single token in C/C++ code"""
75 type: TokenType
76 value: str
77 line: int
78 column: int
80 def __repr__(self) -> str:
81 return f"Token({self.type.name}, '{self.value}', {self.line}:{self.column})"
84class CTokenizer:
85 """Tokenizer for C/C++ source code"""
87 # Keywords mapping
88 KEYWORDS = {
89 "struct": TokenType.STRUCT,
90 "enum": TokenType.ENUM,
91 "union": TokenType.UNION,
92 "typedef": TokenType.TYPEDEF,
93 "static": TokenType.STATIC,
94 "extern": TokenType.EXTERN,
95 "inline": TokenType.INLINE,
96 "local_inline": TokenType.LOCAL_INLINE,
97 "const": TokenType.CONST,
98 "void": TokenType.VOID,
99 "char": TokenType.CHAR,
100 "int": TokenType.INT,
101 "float": TokenType.FLOAT,
102 "double": TokenType.DOUBLE,
103 "long": TokenType.LONG,
104 "short": TokenType.SHORT,
105 "unsigned": TokenType.UNSIGNED,
106 "signed": TokenType.SIGNED,
107 }
109 # Single character tokens
110 SINGLE_CHAR_TOKENS = {
111 "{": TokenType.LBRACE,
112 "}": TokenType.RBRACE,
113 "(": TokenType.LPAREN,
114 ")": TokenType.RPAREN,
115 "[": TokenType.LBRACKET,
116 "]": TokenType.RBRACKET,
117 ";": TokenType.SEMICOLON,
118 ",": TokenType.COMMA,
119 "=": TokenType.ASSIGN,
120 "*": TokenType.ASTERISK,
121 "&": TokenType.AMPERSAND,
122 }
124 # Two character tokens
125 TWO_CHAR_TOKENS = {
126 "->": TokenType.ARROW,
127 }
129 def __init__(self):
130 self.logger = logging.getLogger(__name__)
132 # Compiled regex patterns for efficiency
133 self.patterns = {
134 "identifier": re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*"),
135 "number": re.compile(
136 r"0[xX][0-9a-fA-F]+[uUlL]*|0[bB][01]+[uUlL]*|0[0-7]+[uUlL]*|"
137 r"\d+\.\d*([eE][+-]?\d+)?[fFlL]*|\d+([eE][+-]?\d+)?[fFlL]*|\d+[uUlL]*"
138 ),
139 "string": re.compile(r'"([^"\\]|\\.)*"'),
140 "char": re.compile(r"'([^'\\]|\\.)'"),
141 "comment_single": re.compile(r"//.*"),
142 "comment_multi": re.compile(r"/\*.*?\*/", re.DOTALL),
143 "preprocessor": re.compile(
144 r"#(include|define|ifdef|ifndef|if|endif|elif|else|pragma|error|warning)\b.*"
145 ),
146 "whitespace": re.compile(r"[ \t]+"),
147 "newline": re.compile(r"\n"),
148 }
150 def tokenize(self, content: str) -> List[Token]:
151 """Tokenize C/C++ source code content"""
152 tokens = []
153 lines = content.splitlines()
154 total_lines = len(lines)
155 line_num = 1
156 in_multiline_string = False
157 multiline_string_value = ""
158 multiline_string_start_line = 0
159 multiline_string_start_col = 0
160 in_multiline_comment = False
161 multiline_comment_value = ""
162 multiline_comment_start_line = 0
163 multiline_comment_start_col = 0
165 for idx, line in enumerate(lines):
166 if in_multiline_string:
167 multiline_string_value += "\n" + line
168 if '"' in line:
169 # End of multiline string
170 in_multiline_string = False
171 tokens.append(
172 Token(
173 TokenType.STRING,
174 multiline_string_value,
175 multiline_string_start_line,
176 multiline_string_start_col,
177 )
178 )
179 elif in_multiline_comment:
180 # Continue multi-line comment
181 multiline_comment_value += "\n" + line
182 comment_end = line.find("*/")
183 if comment_end != -1:
184 # End of multi-line comment
185 in_multiline_comment = False
186 multiline_comment_value = multiline_comment_value[
187 : multiline_comment_value.rfind("*/") + 2
188 ]
189 tokens.append(
190 Token(
191 TokenType.COMMENT,
192 multiline_comment_value,
193 multiline_comment_start_line,
194 multiline_comment_start_col,
195 )
196 )
197 else:
198 line_tokens = self._tokenize_line(line, line_num)
199 # Check if a string starts but does not end on this line
200 if (
201 line_tokens
202 and line_tokens[-1].type == TokenType.STRING
203 and not line_tokens[-1].value.endswith('"')
204 ):
205 in_multiline_string = True
206 multiline_string_value = line_tokens[-1].value
207 multiline_string_start_line = line_tokens[-1].line
208 multiline_string_start_col = line_tokens[-1].column
209 tokens.extend(line_tokens[:-1])
210 # Check if a multi-line comment starts but does not end on this line
211 elif (
212 line_tokens
213 and line_tokens[-1].type == TokenType.COMMENT
214 and line_tokens[-1].value.startswith("/*")
215 and not line_tokens[-1].value.endswith("*/")
216 ):
217 in_multiline_comment = True
218 multiline_comment_value = line_tokens[-1].value
219 multiline_comment_start_line = line_tokens[-1].line
220 multiline_comment_start_col = line_tokens[-1].column
221 tokens.extend(line_tokens[:-1])
222 else:
223 tokens.extend(line_tokens)
225 if line_num < total_lines:
226 tokens.append(Token(TokenType.NEWLINE, "\n", line_num, len(line)))
227 line_num += 1
229 if in_multiline_string:
230 tokens.append(
231 Token(
232 TokenType.STRING,
233 multiline_string_value,
234 multiline_string_start_line,
235 multiline_string_start_col,
236 )
237 )
238 if in_multiline_comment:
239 tokens.append(
240 Token(
241 TokenType.COMMENT,
242 multiline_comment_value,
243 multiline_comment_start_line,
244 multiline_comment_start_col,
245 )
246 )
248 # Post-process tokens to merge multi-line macros
249 tokens = self._merge_multiline_macros(tokens, lines)
251 tokens.append(
252 Token(TokenType.EOF, "", total_lines, len(lines[-1]) if lines else 0)
253 )
255 return tokens
257 def _tokenize_line(self, line: str, line_num: int) -> List[Token]:
258 """Tokenize a single line of code"""
259 tokens = []
260 pos = 0
262 while pos < len(line):
263 # Skip whitespace but track it
264 if match := self.patterns["whitespace"].match(line, pos):
265 tokens.append(Token(TokenType.WHITESPACE, match.group(), line_num, pos))
266 pos = match.end()
267 continue
269 # Comments
270 if match := self.patterns["comment_single"].match(line, pos):
271 tokens.append(Token(TokenType.COMMENT, match.group(), line_num, pos))
272 pos = len(line) # Rest of line is comment
273 continue
275 # Multi-line comments - check for /* at start of line or after whitespace
276 if line[pos:].startswith("/*"):
277 # Find the end of the comment
278 comment_end = line.find("*/", pos)
279 if comment_end != -1:
280 # Comment ends on this line
281 comment_text = line[pos : comment_end + 2]
282 tokens.append(Token(TokenType.COMMENT, comment_text, line_num, pos))
283 pos = comment_end + 2
284 continue
285 else:
286 # Comment continues to next line - create a partial comment token
287 comment_text = line[pos:]
288 tokens.append(Token(TokenType.COMMENT, comment_text, line_num, pos))
289 pos = len(line)
290 continue
292 # Preprocessor directives
293 if match := self.patterns["preprocessor"].match(line, pos):
294 value = match.group()
295 if value.startswith("#include"):
296 tokens.append(Token(TokenType.INCLUDE, value, line_num, pos))
297 elif value.startswith("#define"):
298 tokens.append(Token(TokenType.DEFINE, value, line_num, pos))
299 else:
300 tokens.append(Token(TokenType.PREPROCESSOR, value, line_num, pos))
301 pos = len(line) # Rest of line is preprocessor
302 continue
304 # String literals
305 if (
306 line[pos] == '"'
307 or (
308 pos > 0
309 and line[pos - 1] in ["L", "u", "U", "R"]
310 and line[pos] == '"'
311 )
312 or (pos > 1 and line[pos - 2 : pos] == "u8" and line[pos] == '"')
313 ):
314 # Handle string literals with possible prefixes
315 string_start = pos
316 if line[pos - 2 : pos] == "u8":
317 string_start -= 2
318 elif line[pos - 1] in ["L", "u", "U", "R"]:
319 string_start -= 1
320 pos += 1 # Skip opening quote
321 while pos < len(line):
322 if line[pos] == '"':
323 # Found closing quote
324 string_text = line[string_start : pos + 1]
325 tokens.append(
326 Token(TokenType.STRING, string_text, line_num, string_start)
327 )
328 pos += 1
329 break
330 elif line[pos] == "\\":
331 pos += 2
332 else:
333 pos += 1
334 else:
335 string_text = line[string_start:]
336 tokens.append(
337 Token(TokenType.STRING, string_text, line_num, string_start)
338 )
339 pos = len(line)
340 continue
342 # Character literals
343 if match := self.patterns["char"].match(line, pos):
344 tokens.append(
345 Token(TokenType.CHAR_LITERAL, match.group(), line_num, pos)
346 )
347 pos = match.end()
348 continue
350 # Numbers
351 if match := self.patterns["number"].match(line, pos):
352 tokens.append(Token(TokenType.NUMBER, match.group(), line_num, pos))
353 pos = match.end()
354 continue
356 # Single character tokens
357 if line[pos] in self.SINGLE_CHAR_TOKENS:
358 token_type = self.SINGLE_CHAR_TOKENS[line[pos]]
359 tokens.append(Token(token_type, line[pos], line_num, pos))
360 pos += 1
361 continue
363 # Multi-character operators (<<, >>, ->)
364 if line[pos : pos + 2] in ["<<", ">>", "->"]:
365 op = line[pos : pos + 2]
366 if op == "->":
367 tokens.append(Token(TokenType.ARROW, op, line_num, pos))
368 else:
369 tokens.append(
370 Token(
371 (
372 TokenType.OPERATOR
373 if hasattr(TokenType, "OPERATOR")
374 else TokenType.UNKNOWN
375 ),
376 op,
377 line_num,
378 pos,
379 )
380 )
381 pos += 2
382 continue
384 # Identifiers and keywords
385 if match := self.patterns["identifier"].match(line, pos):
386 value = match.group()
387 token_type = self.KEYWORDS.get(value.lower(), TokenType.IDENTIFIER)
388 tokens.append(Token(token_type, value, line_num, pos))
389 pos = match.end()
390 continue
392 # Unknown character (always one at a time)
393 tokens.append(Token(TokenType.UNKNOWN, line[pos], line_num, pos))
394 pos += 1
396 return tokens
398 def filter_tokens(
399 self, tokens: List[Token], exclude_types: Optional[List[TokenType]] = None
400 ) -> List[Token]:
401 """Filter tokens by type"""
402 if exclude_types is None:
403 exclude_types = [
404 TokenType.WHITESPACE,
405 TokenType.COMMENT,
406 TokenType.NEWLINE,
407 TokenType.EOF,
408 ]
410 return [token for token in tokens if token.type not in exclude_types]
412 def _merge_multiline_macros(
413 self, tokens: List[Token], lines: List[str]
414 ) -> List[Token]:
415 """Merge multi-line macro tokens that span multiple lines with backslashes"""
416 merged_tokens = []
417 i = 0
419 while i < len(tokens):
420 token = tokens[i]
422 if token.type == TokenType.DEFINE and token.value.rstrip().endswith("\\"):
423 # Found a multi-line macro, merge with subsequent lines
424 macro_content = token.value
425 current_line = token.line
427 # Continue merging lines until we find one that doesn't end with backslash
428 while macro_content.rstrip().endswith("\\"):
429 # Remove the backslash and add a newline
430 macro_content = macro_content.rstrip()[:-1] + "\n"
431 current_line += 1
433 # Find the next line content
434 if current_line <= len(lines):
435 next_line = lines[current_line - 1]
436 macro_content += next_line
437 else:
438 break
440 # Create a new token with the merged content
441 merged_tokens.append(
442 Token(TokenType.DEFINE, macro_content, token.line, token.column)
443 )
444 else:
445 merged_tokens.append(token)
447 i += 1
449 return merged_tokens
452class StructureFinder:
453 """Helper class to find C/C++ structures in token streams"""
455 def __init__(self, tokens: List[Token]):
456 self.tokens = tokens
457 self.pos = 0
458 self.logger = logging.getLogger(__name__)
460 def _skip_whitespace(self) -> None:
461 """Advance self.pos over consecutive WHITESPACE tokens."""
462 while self.pos < len(self.tokens) and self._current_token_is(TokenType.WHITESPACE):
463 self.pos += 1
465 def find_structs(self) -> List[Tuple[int, int, str]]:
466 """Find struct definitions in token stream
468 Returns:
469 List of tuples (start_pos, end_pos, struct_name)
470 """
471 structs = []
472 self.pos = 0
474 while self.pos < len(self.tokens):
475 if self._current_token_is(TokenType.STRUCT):
476 struct_info = self._parse_struct()
477 if struct_info:
478 structs.append(struct_info)
479 elif self._current_token_is(TokenType.TYPEDEF):
480 typedef_struct = self._parse_typedef_struct()
481 if typedef_struct:
482 structs.append(typedef_struct)
483 else:
484 self.pos += 1
486 return structs
488 def find_enums(self) -> List[Tuple[int, int, str]]:
489 """Find enum definitions in token stream"""
490 enums = []
491 self.pos = 0
493 while self.pos < len(self.tokens):
494 if self._current_token_is(TokenType.ENUM):
495 enum_info = self._parse_enum()
496 if enum_info:
497 enums.append(enum_info)
498 elif self._current_token_is(TokenType.TYPEDEF):
499 typedef_enum = self._parse_typedef_enum()
500 if typedef_enum:
501 enums.append(typedef_enum)
502 else:
503 self.pos += 1
505 return enums
507 def find_functions(self) -> List[Tuple[int, int, str, str, bool, bool]]:
508 """Find all function declarations and definitions in the token stream
510 Returns:
511 List of tuples (start_pos, end_pos, func_name, return_type, is_declaration, is_inline)
512 """
513 functions = []
514 self.pos = 0
516 while self.pos < len(self.tokens):
517 result = self._parse_function()
518 if result:
519 functions.append(result)
521 return functions
523 def find_unions(self) -> List[Tuple[int, int, str]]:
524 """Find union definitions in token stream"""
525 unions = []
526 self.pos = 0
528 while self.pos < len(self.tokens):
529 if self._current_token_is(TokenType.UNION):
530 union_info = self._parse_union()
531 if union_info:
532 unions.append(union_info)
533 elif self._current_token_is(TokenType.TYPEDEF):
534 typedef_union = self._parse_typedef_union()
535 if typedef_union:
536 unions.append(typedef_union)
537 else:
538 self.pos += 1
540 return unions
542 def _current_token_is(self, token_type: TokenType) -> bool:
543 """Check if current token is of specified type"""
544 return self.pos < len(self.tokens) and self.tokens[self.pos].type == token_type
546 def _peek_token(self, offset: int = 1) -> Optional[Token]:
547 """Peek at token at current position + offset"""
548 peek_pos = self.pos + offset
549 return self.tokens[peek_pos] if peek_pos < len(self.tokens) else None
551 def _advance(self) -> Optional[Token]:
552 """Advance to next token and return current"""
553 if self.pos < len(self.tokens):
554 token = self.tokens[self.pos]
555 self.pos += 1
556 return token
557 return None
559 def _find_matching_brace(self, start_pos: int) -> Optional[int]:
560 """Find matching closing brace starting from open brace position.
562 Delegates to shared parse_utils implementation to ensure one source of
563 truth for balanced scanning across components.
564 """
565 from .parse_utils import find_matching_brace as _find_brace
567 return _find_brace(self.tokens, start_pos)
569 def _parse_struct(self) -> Optional[Tuple[int, int, str]]:
570 """Parse struct definition starting at current position"""
571 start_pos = self.pos
573 # Consume 'struct' keyword
574 if not self._current_token_is(TokenType.STRUCT):
575 return None
576 self._advance()
578 # Check if this struct is inside a cast expression by looking backwards
579 check_pos = start_pos - 1
580 while check_pos >= 0:
581 if self.tokens[check_pos].type == TokenType.LPAREN:
582 # Found opening parenthesis before struct - this is likely a cast expression
583 return None
584 elif self.tokens[check_pos].type in [TokenType.STRUCT, TokenType.TYPEDEF]:
585 # Found another struct or typedef - this is not a cast expression
586 break
587 elif self.tokens[check_pos].type not in [TokenType.WHITESPACE, TokenType.COMMENT, TokenType.NEWLINE]:
588 # Found some other token - this is not a cast expression
589 break
590 check_pos -= 1
592 # Skip whitespace
593 self._skip_whitespace()
595 # Check if this is a cast expression: (struct type*)
596 if self._current_token_is(TokenType.LPAREN):
597 # Look ahead to see if this is a cast expression
598 check_pos = self.pos + 1
599 while check_pos < len(self.tokens):
600 if self.tokens[check_pos].type == TokenType.RPAREN:
601 # Found closing parenthesis - this is likely a cast expression
602 return None
603 elif self.tokens[check_pos].type == TokenType.LBRACE:
604 # Found opening brace - this is a struct definition
605 break
606 elif self.tokens[check_pos].type == TokenType.SEMICOLON:
607 # Found semicolon - this is a variable declaration
608 return None
609 check_pos += 1
611 # Get struct tag name (optional for anonymous structs)
612 struct_tag = ""
613 if self._current_token_is(TokenType.IDENTIFIER):
614 struct_tag = self._advance().value
616 # Look for opening brace or semicolon
617 while self.pos < len(self.tokens):
618 if self._current_token_is(TokenType.LBRACE):
619 # Found opening brace - this is a struct definition
620 break
621 elif self._current_token_is(TokenType.SEMICOLON):
622 # Found semicolon before opening brace - this is a variable declaration
623 return None
624 self.pos += 1
626 if not self._current_token_is(TokenType.LBRACE):
627 # This is a variable declaration
628 return None
630 # Find matching closing brace
631 brace_pos = self.pos
632 end_brace_pos = self._find_matching_brace(brace_pos)
634 if end_brace_pos is None:
635 return None
637 # Look for struct name after closing brace
638 name_pos = end_brace_pos + 1
639 struct_name = struct_tag # Default to tag name
641 # Check if this is a typedef struct by looking backwards
642 is_typedef = False
643 check_pos = start_pos - 1
644 while check_pos >= 0:
645 if self.tokens[check_pos].type == TokenType.TYPEDEF:
646 is_typedef = True
647 break
648 elif self.tokens[check_pos].type in [
649 TokenType.STRUCT,
650 TokenType.LBRACE,
651 TokenType.RBRACE,
652 ]:
653 break
654 check_pos -= 1
656 if is_typedef:
657 # For typedef struct, look for the typedef name after the closing brace
658 while name_pos < len(self.tokens):
659 if self.tokens[name_pos].type == TokenType.IDENTIFIER:
660 struct_name = self.tokens[name_pos].value
661 break
662 elif self.tokens[name_pos].type == TokenType.SEMICOLON:
663 break
664 name_pos += 1
665 else:
666 # Check if there's a variable name after the brace
667 while name_pos < len(self.tokens):
668 if self.tokens[name_pos].type == TokenType.IDENTIFIER:
669 # This is a variable name
670 struct_name = ""
671 break
672 elif self.tokens[name_pos].type == TokenType.SEMICOLON:
673 break
674 name_pos += 1
676 # Find semicolon (for struct definitions)
677 self.pos = end_brace_pos + 1
678 while self.pos < len(self.tokens) and not self._current_token_is(
679 TokenType.SEMICOLON
680 ):
681 self.pos += 1
683 end_pos = self.pos
684 return (start_pos, end_pos, struct_name)
686 def _parse_typedef_struct(self) -> Optional[Tuple[int, int, str]]:
687 """Parse typedef struct definition"""
688 start_pos = self.pos
690 # Consume 'typedef'
691 if not self._current_token_is(TokenType.TYPEDEF):
692 return None
693 self._advance()
695 # Look for 'struct'
696 if not self._current_token_is(TokenType.STRUCT):
697 # Not a typedef struct, reset position
698 self.pos = start_pos + 1
699 return None
701 # Skip 'struct'
702 self._advance()
704 # Skip whitespace
705 self._skip_whitespace()
707 # Get struct tag name (optional)
708 struct_tag = ""
709 if self._current_token_is(TokenType.IDENTIFIER):
710 struct_tag = self._advance().value
712 # Skip whitespace
713 self._skip_whitespace()
715 # Check if this is a forward declaration (no braces)
716 if not self._current_token_is(TokenType.LBRACE):
717 # This is a forward declaration, skip it
718 self.pos = start_pos + 1
719 return None
721 # Find matching closing brace
722 end_brace_pos = self._find_matching_brace(self.pos)
723 if end_brace_pos is None:
724 self.pos = start_pos + 1
725 return None
727 # Look for typedef name after closing brace
728 typedef_name = ""
729 name_pos = end_brace_pos + 1
730 while name_pos < len(self.tokens):
731 if self.tokens[name_pos].type == TokenType.IDENTIFIER:
732 typedef_name = self.tokens[name_pos].value
733 break
734 elif self.tokens[name_pos].type == TokenType.SEMICOLON:
735 break
736 name_pos += 1
738 # Find semicolon
739 while (
740 name_pos < len(self.tokens)
741 and not self.tokens[name_pos].type == TokenType.SEMICOLON
742 ):
743 name_pos += 1
745 end_pos = name_pos
746 return (start_pos, end_pos, typedef_name)
748 def _parse_enum(self) -> Optional[Tuple[int, int, str]]:
749 """Parse enum definition starting at current position"""
750 start_pos = self.pos
752 # Consume 'enum' keyword
753 if not self._current_token_is(TokenType.ENUM):
754 return None
755 self._advance()
757 # Skip whitespace
758 self._skip_whitespace()
760 # Get enum tag name (optional for anonymous enums)
761 enum_tag = ""
762 if self._current_token_is(TokenType.IDENTIFIER):
763 enum_tag = self._advance().value
765 # Find opening brace
766 while self.pos < len(self.tokens) and not self._current_token_is(
767 TokenType.LBRACE
768 ):
769 self.pos += 1
771 if not self._current_token_is(TokenType.LBRACE):
772 return None
774 # Find matching closing brace
775 brace_pos = self.pos
776 end_brace_pos = self._find_matching_brace(brace_pos)
778 if end_brace_pos is None:
779 return None
781 # Look for enum name after closing brace
782 name_pos = end_brace_pos + 1
783 enum_name = enum_tag # Default to tag name
785 # Check if this is a typedef enum by looking backwards
786 is_typedef = False
787 check_pos = start_pos - 1
788 while check_pos >= 0:
789 if self.tokens[check_pos].type == TokenType.TYPEDEF:
790 is_typedef = True
791 break
792 elif self.tokens[check_pos].type in [
793 TokenType.ENUM,
794 TokenType.LBRACE,
795 TokenType.RBRACE,
796 ]:
797 break
798 check_pos -= 1
800 if is_typedef:
801 # For typedef enum, look for the typedef name after the closing brace
802 while name_pos < len(self.tokens):
803 if self.tokens[name_pos].type == TokenType.IDENTIFIER:
804 enum_name = self.tokens[name_pos].value
805 break
806 elif self.tokens[name_pos].type == TokenType.SEMICOLON:
807 break
808 name_pos += 1
809 elif not enum_tag:
810 # Anonymous enum - check if there's a variable name after the brace
811 while name_pos < len(self.tokens):
812 if self.tokens[name_pos].type == TokenType.IDENTIFIER:
813 # This is a variable name
814 enum_name = ""
815 break
816 elif self.tokens[name_pos].type == TokenType.SEMICOLON:
817 break
818 name_pos += 1
820 # Find semicolon
821 self.pos = end_brace_pos + 1
822 while self.pos < len(self.tokens) and not self._current_token_is(
823 TokenType.SEMICOLON
824 ):
825 self.pos += 1
827 end_pos = self.pos
828 return (start_pos, end_pos, enum_name)
830 def _parse_typedef_enum(self) -> Optional[Tuple[int, int, str]]:
831 """Parse typedef enum definition"""
832 start_pos = self.pos
834 # Consume 'typedef'
835 if not self._current_token_is(TokenType.TYPEDEF):
836 return None
837 self._advance()
839 # Look for 'enum'
840 if not self._current_token_is(TokenType.ENUM):
841 # Not a typedef enum, reset position
842 self.pos = start_pos + 1
843 return None
845 # Parse the enum part - this will return the tag name (e.g., StatusEnum_tag)
846 enum_info = self._parse_enum()
847 if not enum_info:
848 self.pos = start_pos + 1
849 return None
851 # For typedef enums, we want to return the tag name, not the typedef name
852 # The typedef name will be handled separately in the parser
853 return enum_info
855 def _parse_function(self) -> Optional[Tuple[int, int, str, str, bool, bool]]:
856 """Parse function declaration/definition
858 Returns:
859 Tuple of (start_pos, end_pos, func_name, return_type, is_declaration, is_inline)
860 """
861 start_pos = self.pos
863 # Look for function pattern: [modifiers] return_type function_name (params)
864 while self.pos < len(self.tokens):
865 token = self.tokens[self.pos]
867 # If we hit a parenthesis, check if this is a function
868 if token.type == TokenType.LPAREN:
869 # Look backwards for function name
870 if (
871 self.pos > 0
872 and self.tokens[self.pos - 1].type == TokenType.IDENTIFIER
873 ):
874 func_name = self.tokens[self.pos - 1].value
875 func_name_pos = self.pos - 1
877 # Look backwards from function name to find return type
878 # Start from just before the function name
879 return_type_end = func_name_pos - 1
880 return_type_start = return_type_end
882 # Skip backwards over whitespace and comments
883 while return_type_start >= 0:
884 token_type = self.tokens[return_type_start].type
885 if token_type in [
886 TokenType.WHITESPACE,
887 TokenType.COMMENT,
888 TokenType.NEWLINE,
889 ]:
890 return_type_start -= 1
891 else:
892 break
894 # If we found a non-whitespace token, that's the end of the return type
895 # Find the start by looking backwards from there
896 if return_type_start >= 0:
897 return_type_end = return_type_start
898 return_type_start = return_type_end
900 # Define modifiers set (used in token type checking below)
902 # Collect all tokens that are part of the return type (including modifiers)
903 return_type_tokens = []
905 # Look back at most 10 tokens to capture multi-token return types
906 max_lookback = max(0, func_name_pos - 10)
907 current_pos = return_type_start
909 # Collect tokens backwards until we hit a limit or non-return-type token
910 while current_pos >= max_lookback:
911 token_type = self.tokens[current_pos].type
912 if token_type in [
913 TokenType.IDENTIFIER,
914 TokenType.INT,
915 TokenType.VOID,
916 TokenType.CHAR,
917 TokenType.FLOAT,
918 TokenType.DOUBLE,
919 TokenType.LONG,
920 TokenType.SHORT,
921 TokenType.UNSIGNED,
922 TokenType.SIGNED,
923 TokenType.ASTERISK,
924 TokenType.CONST,
925 TokenType.STATIC,
926 TokenType.EXTERN,
927 TokenType.INLINE,
928 TokenType.LOCAL_INLINE,
929 ]:
930 return_type_tokens.insert(0, self.tokens[current_pos])
931 current_pos -= 1
932 elif token_type in [
933 TokenType.WHITESPACE,
934 TokenType.COMMENT,
935 TokenType.NEWLINE,
936 ]:
937 # Skip whitespace and continue looking
938 current_pos -= 1
939 else:
940 break
942 # Extract return type
943 if return_type_tokens:
944 return_type = " ".join(
945 t.value for t in return_type_tokens
946 ).strip()
948 # Check if function is inline
949 is_inline = any(
950 token.type in [TokenType.INLINE, TokenType.LOCAL_INLINE]
951 for token in return_type_tokens
952 )
953 # Fallback: detect inline by textual prefix in return_type
954 # This covers cases where macros like LOCAL_INLINE were not token-mapped earlier
955 if not is_inline:
956 rt_upper = return_type.upper()
957 if rt_upper.startswith("LOCAL_INLINE ") or rt_upper.startswith("INLINE "):
958 is_inline = True
960 # Find end of function (either ; for declaration or { for definition)
961 end_pos = self._find_function_end(self.pos)
962 if end_pos:
963 # Determine if this is a declaration or definition
964 is_declaration = self._is_function_declaration(end_pos)
965 self.pos = end_pos + 1
966 return (
967 start_pos,
968 end_pos,
969 func_name,
970 return_type,
971 is_declaration,
972 is_inline,
973 )
975 self.pos += 1
977 # Prevent infinite loops - if we've gone too far, this isn't a function
978 if self.pos - start_pos > 50:
979 break
981 # Reset position if no function found
982 self.pos = start_pos + 1
983 return None
985 def _is_function_declaration(self, end_pos: int) -> bool:
986 """Check if the function at end_pos is a declaration (ends with ;) or definition (ends with })"""
987 if end_pos >= len(self.tokens):
988 return False
990 # Look backwards from end_pos to find the last significant token
991 pos = end_pos
992 while pos >= 0:
993 token_type = self.tokens[pos].type
994 if token_type not in [
995 TokenType.WHITESPACE,
996 TokenType.COMMENT,
997 TokenType.NEWLINE,
998 ]:
999 return token_type == TokenType.SEMICOLON
1000 pos -= 1
1002 return False
1004 def _find_function_end(self, start_pos: int) -> Optional[int]:
1005 """Find end of function declaration or definition"""
1006 pos = start_pos
1008 # Find matching closing parenthesis
1009 if pos >= len(self.tokens) or self.tokens[pos].type != TokenType.LPAREN:
1010 return None
1012 depth = 1
1013 pos += 1
1015 while pos < len(self.tokens) and depth > 0:
1016 if self.tokens[pos].type == TokenType.LPAREN:
1017 depth += 1
1018 elif self.tokens[pos].type == TokenType.RPAREN:
1019 depth -= 1
1020 pos += 1
1022 if depth > 0:
1023 return None
1025 # Look for either ; (declaration) or { (definition)
1026 while pos < len(self.tokens):
1027 if self.tokens[pos].type == TokenType.SEMICOLON:
1028 return pos
1029 elif self.tokens[pos].type == TokenType.LBRACE:
1030 # Function definition - find matching brace (delegate)
1031 end_brace = self._find_matching_brace(pos)
1032 return end_brace if end_brace else pos
1033 pos += 1
1035 return None
1037 def _parse_union(self) -> Optional[Tuple[int, int, str]]:
1038 """Parse union definition"""
1039 if not self._current_token_is(TokenType.UNION):
1040 return None
1042 start_pos = self.pos
1043 self._advance() # Consumes 'union'
1045 # Skip whitespace
1046 while self.pos < len(self.tokens) and self._current_token_is(
1047 TokenType.WHITESPACE
1048 ):
1049 self.pos += 1
1051 # Get union tag name (optional for anonymous unions)
1052 union_tag = ""
1053 if self._current_token_is(TokenType.IDENTIFIER):
1054 union_tag = self._advance().value
1056 # Find opening brace
1057 while self.pos < len(self.tokens) and not self._current_token_is(
1058 TokenType.LBRACE
1059 ):
1060 self.pos += 1
1062 if self.pos >= len(self.tokens):
1063 return None
1065 # Find matching closing brace
1066 end_pos = self._find_matching_brace(self.pos)
1067 if end_pos is None:
1068 return None
1070 # Look for union name after closing brace (for typedefs or named unions)
1071 union_name = union_tag # Default to tag name
1073 # Skip to semicolon
1074 self.pos = end_pos + 1
1075 while self.pos < len(self.tokens) and not self._current_token_is(
1076 TokenType.SEMICOLON
1077 ):
1078 if self._current_token_is(TokenType.IDENTIFIER):
1079 union_name = self._advance().value
1080 break
1081 self.pos += 1
1083 return (start_pos, end_pos, union_name)
1085 def _parse_typedef_union(self) -> Optional[Tuple[int, int, str]]:
1086 """Parse typedef union definition"""
1087 if not self._current_token_is(TokenType.TYPEDEF):
1088 return None
1090 start_pos = self.pos
1091 self._advance() # Consumes 'typedef'
1093 # Skip whitespace
1094 while self.pos < len(self.tokens) and self._current_token_is(
1095 TokenType.WHITESPACE
1096 ):
1097 self.pos += 1
1099 # Check if next token is 'union'
1100 if not self._current_token_is(TokenType.UNION):
1101 return None
1103 self._advance() # Consumes 'union'
1105 # Skip whitespace
1106 while self.pos < len(self.tokens) and self._current_token_is(
1107 TokenType.WHITESPACE
1108 ):
1109 self.pos += 1
1111 # Get union tag name (optional)
1112 union_tag = ""
1113 if self._current_token_is(TokenType.IDENTIFIER):
1114 union_tag = self._advance().value
1116 # Find opening brace
1117 while self.pos < len(self.tokens) and not self._current_token_is(
1118 TokenType.LBRACE
1119 ):
1120 self.pos += 1
1122 if self.pos >= len(self.tokens):
1123 return None
1125 # Find matching closing brace
1126 end_pos = self._find_matching_brace(self.pos)
1127 if end_pos is None:
1128 return None
1130 # Look for typedef name after closing brace
1131 typedef_name = ""
1132 self.pos = end_pos + 1
1133 while self.pos < len(self.tokens) and not self._current_token_is(
1134 TokenType.SEMICOLON
1135 ):
1136 if self._current_token_is(TokenType.IDENTIFIER):
1137 typedef_name = self._advance().value
1138 break
1139 self.pos += 1
1141 return (start_pos, end_pos, typedef_name)
1144def extract_token_range(tokens: List[Token], start: int, end: int) -> str:
1145 """Extract raw text from token range, excluding whitespace, comments, and newlines"""
1146 if start >= len(tokens) or end >= len(tokens) or start > end:
1147 return ""
1148 return " ".join(
1149 token.value
1150 for token in tokens[start : end + 1]
1151 if token.type
1152 not in [TokenType.WHITESPACE, TokenType.COMMENT, TokenType.NEWLINE]
1153 )
1156def find_struct_fields(
1157 tokens: List[Token], struct_start: int, struct_end: int
1158) -> List[Tuple[str, str]]:
1159 """Extract field information from struct token range
1160 Returns:
1161 List of tuples (field_name, field_type)
1162 """
1163 fields = []
1164 pos = struct_start
1165 while pos <= struct_end and tokens[pos].type != TokenType.LBRACE:
1166 pos += 1
1167 if pos > struct_end:
1168 return fields
1169 pos += 1 # Skip opening brace
1171 # Find the closing brace position of the main struct body
1172 closing_brace_pos = pos
1173 brace_count = 1 # Start at 1 because we're already past the opening brace
1174 while closing_brace_pos <= struct_end:
1175 if tokens[closing_brace_pos].type == TokenType.LBRACE:
1176 brace_count += 1
1177 elif tokens[closing_brace_pos].type == TokenType.RBRACE:
1178 brace_count -= 1
1179 if brace_count == 0:
1180 # This is the closing brace of the main struct body
1181 break
1182 closing_brace_pos += 1
1184 # Only parse fields up to the closing brace
1185 while pos < closing_brace_pos and tokens[pos].type != TokenType.RBRACE:
1186 field_tokens = []
1187 # Collect tokens until we find the semicolon that ends this field
1188 # For nested structures, we need to handle braces properly
1189 brace_count = 0
1190 field_start_pos = pos
1192 # First pass: collect tokens until we find the semicolon outside of braces
1193 while pos < closing_brace_pos:
1194 if tokens[pos].type == TokenType.LBRACE:
1195 brace_count += 1
1196 elif tokens[pos].type == TokenType.RBRACE:
1197 brace_count -= 1
1198 # Only stop if we're at the main closing brace
1199 if pos == closing_brace_pos:
1200 break
1201 elif tokens[pos].type == TokenType.SEMICOLON and brace_count == 0:
1202 # This is the semicolon that ends the field
1203 break
1205 if tokens[pos].type not in [TokenType.WHITESPACE, TokenType.COMMENT, TokenType.NEWLINE]:
1206 field_tokens.append(tokens[pos])
1207 pos += 1
1209 # For nested structures, we need to continue collecting tokens until we find the field name
1210 # and the semicolon that ends the entire field
1211 if (len(field_tokens) >= 3 and
1212 field_tokens[0].type in [TokenType.STRUCT, TokenType.UNION] and
1213 field_tokens[1].type == TokenType.LBRACE):
1214 # This might be a nested structure, continue collecting until we find the field name
1215 temp_pos = pos
1216 brace_count = 0 # Track nested braces to find the correct field boundary
1217 while temp_pos < len(tokens):
1218 if tokens[temp_pos].type == TokenType.LBRACE:
1219 brace_count += 1
1220 elif tokens[temp_pos].type == TokenType.RBRACE:
1221 brace_count -= 1
1222 elif tokens[temp_pos].type == TokenType.SEMICOLON and brace_count == 0:
1223 # Found the semicolon that ends the field (not inside nested braces)
1224 break
1226 if tokens[temp_pos].type not in [TokenType.WHITESPACE, TokenType.COMMENT, TokenType.NEWLINE]:
1227 field_tokens.append(tokens[temp_pos])
1228 temp_pos += 1
1229 pos = temp_pos
1231 # Parse field from collected tokens
1232 if len(field_tokens) >= 2:
1233 # Check if this is a nested struct field
1234 if (
1235 len(field_tokens) >= 3
1236 and field_tokens[0].type == TokenType.STRUCT
1237 and field_tokens[1].type == TokenType.LBRACE
1238 ):
1239 # This is a nested struct - find the field name after the closing brace
1240 # Look for the pattern: struct { ... } field_name;
1241 field_name = None
1242 # Find the LAST closing brace and then the field name
1243 # This handles deeply nested structures correctly
1244 for i in range(len(field_tokens) - 1, -1, -1):
1245 if field_tokens[i].type == TokenType.RBRACE and i + 1 < len(field_tokens):
1246 # The field name should be the next identifier after the closing brace
1247 for j in range(i + 1, len(field_tokens)):
1248 if field_tokens[j].type == TokenType.IDENTIFIER:
1249 field_name = field_tokens[j].value
1250 break
1251 if field_name:
1252 break
1254 if field_name:
1255 # Extract the content between braces for anonymous processor using special format
1256 content = _extract_brace_content(field_tokens)
1257 if content:
1258 # Preserve content for anonymous processor using special format
1259 import base64
1260 encoded_content = base64.b64encode(content.encode()).decode()
1261 field_type = f"struct {{ /*ANON:{encoded_content}:{field_name}*/ ... }}"
1262 else:
1263 field_type = "struct { ... }"
1265 if field_name not in ["[", "]", ";", "}"]:
1266 fields.append((field_name, field_type))
1267 # Skip parsing the nested struct's fields as separate fields
1268 # Let the normal flow handle semicolon advancement
1269 else:
1270 # Anonymous nested struct without a field name
1271 content = _extract_brace_content(field_tokens)
1272 if content:
1273 import base64
1274 encoded_content = base64.b64encode(content.encode()).decode()
1275 # Use generic field name for anonymous struct
1276 generic_name = "__anonymous_struct__"
1277 field_type = f"struct {{ /*ANON:{encoded_content}:{generic_name}*/ ... }}"
1278 else:
1279 generic_name = "__anonymous_struct__"
1280 field_type = "struct { ... }"
1281 fields.append((generic_name, field_type))
1282 # Check if this is a nested union field
1283 elif (
1284 len(field_tokens) >= 3
1285 and field_tokens[0].type == TokenType.UNION
1286 and field_tokens[1].type == TokenType.LBRACE
1287 ):
1288 # This is a nested union - find the field name after the closing brace
1289 # Look for the pattern: union { ... } field_name;
1290 field_name = None
1291 # Find the LAST closing brace and then the field name
1292 # This handles deeply nested structures correctly
1293 for i in range(len(field_tokens) - 1, -1, -1):
1294 if field_tokens[i].type == TokenType.RBRACE and i + 1 < len(field_tokens):
1295 # The field name should be the next identifier after the closing brace
1296 for j in range(i + 1, len(field_tokens)):
1297 if field_tokens[j].type == TokenType.IDENTIFIER:
1298 field_name = field_tokens[j].value
1299 break
1300 if field_name:
1301 break
1303 if field_name:
1304 # Extract the content between braces for anonymous processor
1305 content = _extract_brace_content(field_tokens)
1306 if content:
1307 # Preserve content for anonymous processor using special format
1308 import base64
1309 encoded_content = base64.b64encode(content.encode()).decode()
1310 field_type = f"union {{ /*ANON:{encoded_content}:{field_name}*/ ... }}"
1311 else:
1312 field_type = "union { ... }"
1314 if field_name not in ["[", "]", ";", "}"]:
1315 fields.append((field_name, field_type))
1316 # Skip parsing the nested union's fields as separate fields
1317 # Let the normal flow handle semicolon advancement
1318 else:
1319 # Anonymous nested union without a field name
1320 content = _extract_brace_content(field_tokens)
1321 if content:
1322 import base64
1323 encoded_content = base64.b64encode(content.encode()).decode()
1324 generic_name = "__anonymous_union__"
1325 field_type = f"union {{ /*ANON:{encoded_content}:{generic_name}*/ ... }}"
1326 else:
1327 generic_name = "__anonymous_union__"
1328 field_type = "union { ... }"
1329 fields.append((generic_name, field_type))
1330 # Function pointer array field: type (*name[size])(params)
1331 elif (
1332 len(field_tokens) >= 8
1333 and field_tokens[1].type == TokenType.LPAREN
1334 and field_tokens[2].type == TokenType.ASTERISK
1335 and any(t.type == TokenType.LBRACKET for t in field_tokens)
1336 and any(t.type == TokenType.RBRACKET for t in field_tokens)
1337 ):
1338 # Find the function pointer name (between * and [)
1339 # Look for the identifier between * and [
1340 name_start = 3 # After the *
1341 name_end = None
1342 for i in range(name_start, len(field_tokens)):
1343 if field_tokens[i].type == TokenType.LBRACKET:
1344 name_end = i
1345 break
1347 if name_end is not None:
1348 field_name = " ".join(
1349 t.value for t in field_tokens[name_start:name_end]
1350 )
1352 # Format the type properly - preserve spaces between tokens but not around brackets/parentheses
1353 formatted_tokens = []
1354 for j, token in enumerate(field_tokens):
1355 if token.type in [
1356 TokenType.LPAREN,
1357 TokenType.RPAREN,
1358 TokenType.LBRACKET,
1359 TokenType.RBRACKET,
1360 ]:
1361 # Don't add spaces around brackets/parentheses
1362 formatted_tokens.append(token.value)
1363 elif j > 0 and field_tokens[j - 1].type not in [
1364 TokenType.LPAREN,
1365 TokenType.RPAREN,
1366 TokenType.LBRACKET,
1367 TokenType.RBRACKET,
1368 ]:
1369 # Add space before token if previous token wasn't a bracket/parenthesis
1370 formatted_tokens.append(" " + token.value)
1371 else:
1372 # No space before token
1373 formatted_tokens.append(token.value)
1374 field_type = "".join(formatted_tokens)
1376 # Validate and add the field
1377 if (
1378 field_name
1379 and field_name.strip()
1380 and field_type.strip()
1381 and field_name not in ["[", "]", ";", "}"]
1382 ):
1383 stripped_name = field_name.strip()
1384 stripped_type = field_type.strip()
1385 if stripped_name and stripped_type:
1386 fields.append((stripped_name, stripped_type))
1387 # Function pointer field: type (*name)(params) or type (*name[size])(params)
1388 elif (
1389 len(field_tokens) >= 5
1390 and field_tokens[1].type == TokenType.LPAREN and field_tokens[2].type == TokenType.ASTERISK
1391 ):
1392 # Find the opening parenthesis and asterisk pattern
1393 func_ptr_start = None
1394 for i in range(len(field_tokens) - 1):
1395 if field_tokens[i].type == TokenType.LPAREN and field_tokens[i + 1].type == TokenType.ASTERISK:
1396 func_ptr_start = i
1397 break
1399 if func_ptr_start is not None:
1400 # Extract the type (everything before the opening parenthesis)
1401 type_tokens = field_tokens[:func_ptr_start]
1402 field_type = " ".join(t.value for t in type_tokens)
1404 # Find the closing parenthesis after the function name
1405 paren_count = 0
1406 name_end = None
1407 for i in range(func_ptr_start, len(field_tokens)):
1408 if field_tokens[i].type == TokenType.LPAREN:
1409 paren_count += 1
1410 elif field_tokens[i].type == TokenType.RPAREN:
1411 paren_count -= 1
1412 if paren_count == 0 and i > func_ptr_start + 1:
1413 name_end = i
1414 break
1416 if name_end is not None:
1417 # Extract function name (between * and closing parenthesis)
1418 name_tokens = field_tokens[func_ptr_start + 2:name_end]
1419 field_name = " ".join(t.value for t in name_tokens)
1421 # Extract the parameter list as part of the type
1422 param_tokens = field_tokens[name_end + 1:]
1423 param_type = " ".join(t.value for t in param_tokens)
1425 # Combine type and parameter list (without the function name in the type)
1426 # The function name is already extracted as field_name, so we don't include it in the type
1427 func_ptr_start_tokens = field_tokens[func_ptr_start:func_ptr_start + 2] # ( *
1428 func_ptr_end_tokens = field_tokens[name_end:name_end + 1] # )
1429 full_type = field_type + " " + " ".join(t.value for t in func_ptr_start_tokens) + " " + " ".join(t.value for t in func_ptr_end_tokens) + " " + param_type
1431 if (
1432 field_name
1433 and field_name.strip()
1434 and full_type.strip()
1435 and field_name not in ["[", "]", ";", "}"]
1436 ):
1437 stripped_name = field_name.strip()
1438 stripped_type = full_type.strip()
1439 if stripped_name and stripped_type:
1440 fields.append((stripped_name, stripped_type))
1441 # Array field(s): support multi-dimensional arrays and normalize numeric suffixes (e.g., 2U -> 2)
1442 elif (
1443 len(field_tokens) >= 4
1444 and field_tokens[-1].type == TokenType.RBRACKET
1445 and any(t.type == TokenType.LBRACKET for t in field_tokens)
1446 ):
1447 # Robust multi-dimensional array handling using shared utils
1448 from .parse_utils import (
1449 collect_array_dimensions_from_tokens,
1450 join_type_with_dims,
1451 normalize_dim_value,
1452 )
1454 # Find the first '[' scanning forward (start of dimensions)
1455 first_lbracket_idx = None
1456 for idx in range(0, len(field_tokens)):
1457 if field_tokens[idx].type == TokenType.LBRACKET:
1458 first_lbracket_idx = idx
1459 break
1460 if first_lbracket_idx is None:
1461 pass
1462 else:
1463 # Find field name just before this '['
1464 name_idx = None
1465 for j in range(first_lbracket_idx - 1, -1, -1):
1466 if field_tokens[j].type == TokenType.IDENTIFIER:
1467 name_idx = j
1468 break
1469 if name_idx is not None:
1470 field_name = field_tokens[name_idx].value
1471 base_type_tokens = field_tokens[:name_idx]
1472 base_type = " ".join(t.value for t in base_type_tokens).strip()
1474 dims, _next = collect_array_dimensions_from_tokens(field_tokens, first_lbracket_idx)
1475 # Normalize pure numeric suffixes like 5U/6UL to base number for dims
1476 normalized_dims = [normalize_dim_value(d) for d in dims]
1478 field_type = join_type_with_dims(base_type, normalized_dims)
1479 if (
1480 field_name
1481 and field_name.strip()
1482 and field_type.strip()
1483 and field_name not in ["[", "]", ";", "}"]
1484 ):
1485 stripped_name = field_name.strip()
1486 stripped_type = field_type.strip()
1487 if stripped_name and stripped_type:
1488 fields.append((stripped_name, stripped_type))
1489 else:
1490 # Regular field: type name
1491 # Check if this field declaration contains commas (multiple fields of same type)
1492 comma_positions = []
1493 paren_count = 0
1494 brace_count = 0
1496 # Find comma positions that are outside of parentheses and braces
1497 for i, token in enumerate(field_tokens):
1498 if token.type == TokenType.LPAREN:
1499 paren_count += 1
1500 elif token.type == TokenType.RPAREN:
1501 paren_count -= 1
1502 elif token.type == TokenType.LBRACE:
1503 brace_count += 1
1504 elif token.type == TokenType.RBRACE:
1505 brace_count -= 1
1506 elif token.type == TokenType.COMMA and paren_count == 0 and brace_count == 0:
1507 comma_positions.append(i)
1509 if comma_positions:
1510 # Multiple fields of the same type: "int x, y, z;"
1511 # Extract the type (everything before the first field name)
1512 first_field_start = None
1513 for i in range(len(field_tokens)):
1514 if field_tokens[i].type == TokenType.IDENTIFIER:
1515 first_field_start = i
1516 break
1518 if first_field_start is not None:
1519 type_tokens = field_tokens[:first_field_start]
1520 field_type = " ".join(t.value for t in type_tokens)
1522 # Split fields on commas
1523 field_starts = [first_field_start] + [pos + 1 for pos in comma_positions]
1524 field_ends = comma_positions + [len(field_tokens)]
1526 for start, end in zip(field_starts, field_ends):
1527 if start < end:
1528 field_name_tokens = field_tokens[start:end]
1529 field_name = " ".join(t.value for t in field_name_tokens)
1531 if (
1532 field_name
1533 and field_name.strip()
1534 and field_type.strip()
1535 and field_name not in ["[", "]", ";", "}"]
1536 ):
1537 stripped_name = field_name.strip()
1538 stripped_type = field_type.strip()
1539 if stripped_name and stripped_type:
1540 fields.append((stripped_name, stripped_type))
1541 else:
1542 # Single field: type name
1543 field_name = field_tokens[-1].value
1544 field_type = " ".join(t.value for t in field_tokens[:-1])
1545 if (
1546 field_name not in ["[", "]", ";", "}"]
1547 and field_name
1548 and field_name.strip()
1549 and field_type.strip()
1550 ):
1551 # Additional validation to ensure we don't have empty strings
1552 stripped_name = field_name.strip()
1553 stripped_type = field_type.strip()
1554 if stripped_name and stripped_type:
1555 fields.append((stripped_name, stripped_type))
1556 if pos < closing_brace_pos:
1557 pos += 1 # Skip semicolon
1558 return fields
1561def find_enum_values(tokens: List[Token], enum_start: int, enum_end: int) -> List[str]:
1562 """Extract enum values from enum token range"""
1563 values = []
1564 pos = enum_start
1565 while pos <= enum_end and tokens[pos].type != TokenType.LBRACE:
1566 pos += 1
1567 if pos > enum_end:
1568 return values
1569 pos += 1 # Skip opening brace
1570 current_value = []
1571 while pos <= enum_end and tokens[pos].type != TokenType.RBRACE:
1572 token = tokens[pos]
1573 if token.type == TokenType.COMMA:
1574 if current_value:
1575 filtered_value = [
1576 t
1577 for t in current_value
1578 if t.type not in [TokenType.WHITESPACE, TokenType.COMMENT]
1579 ]
1580 if filtered_value:
1581 value_str = " ".join(t.value for t in filtered_value).strip()
1582 if value_str:
1583 values.append(value_str)
1584 current_value = []
1585 elif token.type not in [TokenType.WHITESPACE, TokenType.COMMENT]:
1586 current_value.append(token)
1587 pos += 1
1588 if current_value:
1589 filtered_value = [
1590 t
1591 for t in current_value
1592 if t.type not in [TokenType.WHITESPACE, TokenType.COMMENT]
1593 ]
1594 if filtered_value:
1595 value_str = " ".join(t.value for t in filtered_value).strip()
1596 if value_str:
1597 values.append(value_str)
1598 return values
1601def _extract_brace_content(field_tokens: List[Token]) -> str:
1602 """Extract the content between braces from field tokens.
1604 Args:
1605 field_tokens: List of tokens representing a field with anonymous structure
1607 Returns:
1608 String content between the braces, or empty string if not found
1609 """
1610 content_tokens = []
1611 in_braces = False
1612 brace_count = 0
1614 for token in field_tokens:
1615 if token.type == TokenType.LBRACE:
1616 if not in_braces:
1617 in_braces = True
1618 brace_count = 1
1619 else:
1620 brace_count += 1
1621 content_tokens.append(token)
1622 elif token.type == TokenType.RBRACE:
1623 if in_braces:
1624 brace_count -= 1
1625 if brace_count == 0:
1626 # Found the closing brace
1627 break
1628 else:
1629 content_tokens.append(token)
1630 elif in_braces:
1631 content_tokens.append(token)
1633 # Convert tokens back to text preserving spacing
1634 if content_tokens:
1635 result = ""
1636 for i, token in enumerate(content_tokens):
1637 result += token.value
1638 # Add space after most tokens except when next token is punctuation
1639 if (i < len(content_tokens) - 1 and
1640 token.type not in [TokenType.WHITESPACE, TokenType.NEWLINE] and
1641 content_tokens[i + 1].type not in [TokenType.LBRACKET, TokenType.RBRACKET,
1642 TokenType.SEMICOLON, TokenType.COMMA,
1643 TokenType.WHITESPACE, TokenType.NEWLINE]):
1644 result += " "
1645 return result
1646 return ""