34 lines
1.1 KiB
Python
34 lines
1.1 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import re
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Token:
|
|
kind: str
|
|
value: str
|
|
line: int
|
|
column: int
|
|
|
|
|
|
_RE = re.compile(r"(?P<ws>\s+)|(?P<comment>//[^\n]*|/\*.*?\*/)|(?P<string>\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*')|(?P<number>[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?)|(?P<ident>[A-Za-z_][A-Za-z0-9_.]*)|(?P<op>==|!=|<=|>=|=>|[{}\[\]():,;=+*/-])", re.S)
|
|
|
|
|
|
def lex(source: str) -> list[Token]:
|
|
out: list[Token] = []
|
|
pos = 0; line = 1; column = 1
|
|
while pos < len(source):
|
|
match = _RE.match(source, pos)
|
|
if not match:
|
|
raise SyntaxError(f"unexpected character at {line}:{column}: {source[pos]!r}")
|
|
kind, value = match.lastgroup or "", match.group(0)
|
|
if kind not in {"ws", "comment"}:
|
|
out.append(Token(kind, value, line, column))
|
|
nl = value.count("\n")
|
|
if nl: line, column = line + nl, len(value.rsplit("\n", 1)[-1]) + 1
|
|
else: column += len(value)
|
|
pos = match.end()
|
|
out.append(Token("eof", "", line, column))
|
|
return out
|