优化流程

cadfs_to_cdsl/featurescript_lexer.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import re
+
+
+@dataclass(frozen=True)
+class Token:
+    kind: str
+    value: str
+    line: int
+    column: int
+
+
+_RE = re.compile(r"(?P<ws>\s+)|(?P<comment>//[^\n]*|/\*.*?\*/)|(?P<string>\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*')|(?P<number>[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?)|(?P<ident>[A-Za-z_][A-Za-z0-9_.]*)|(?P<op>==|!=|<=|>=|=>|[{}\[\]():,;=+*/-])", re.S)
+
+
+def lex(source: str) -> list[Token]:
+    out: list[Token] = []
+    pos = 0; line = 1; column = 1
+    while pos < len(source):
+        match = _RE.match(source, pos)
+        if not match:
+            raise SyntaxError(f"unexpected character at {line}:{column}: {source[pos]!r}")
+        kind, value = match.lastgroup or "", match.group(0)
+        if kind not in {"ws", "comment"}:
+            out.append(Token(kind, value, line, column))
+        nl = value.count("\n")
+        if nl: line, column = line + nl, len(value.rsplit("\n", 1)[-1]) + 1
+        else: column += len(value)
+        pos = match.end()
+    out.append(Token("eof", "", line, column))
+    return out