Lexical Analysis — character stream থেকে token stream

toylang/lexer.py — পরের সব lesson এর উপর দাঁড়াবে:

"""ToyLang lexer: character stream -> token stream."""
from dataclasses import dataclass
from enum import Enum, auto


class Tok(Enum):
    # literals & names
    INT = auto(); STRING = auto(); IDENT = auto()
    # keywords
    LET = auto(); IF = auto(); ELSE = auto(); WHILE = auto()
    TRUE = auto(); FALSE = auto(); PRINT = auto()
    # operators & punctuation
    PLUS = auto(); MINUS = auto(); STAR = auto(); SLASH = auto()
    PERCENT = auto(); BANG = auto()
    EQEQ = auto(); BANGEQ = auto(); LTE = auto(); GTE = auto()
    LT = auto(); GT = auto(); EQUALS = auto()
    ANDAND = auto(); OROR = auto()
    LPAREN = auto(); RPAREN = auto(); LBRACE = auto(); RBRACE = auto()
    SEMI = auto()
    EOF = auto()


KEYWORDS = {
    "let": Tok.LET, "if": Tok.IF, "else": Tok.ELSE,
    "while": Tok.WHILE, "true": Tok.TRUE, "false": Tok.FALSE,
    "print": Tok.PRINT,
}

TWO_CHAR = {"==": Tok.EQEQ, "!=": Tok.BANGEQ,
            "\<=": Tok.LTE, ">=": Tok.GTE,
            "&&": Tok.ANDAND, "||": Tok.OROR}

ONE_CHAR = {"+": Tok.PLUS, "-": Tok.MINUS, "*": Tok.STAR,
            "/": Tok.SLASH, "%": Tok.PERCENT, "!": Tok.BANG,
            "=": Tok.EQUALS, "<": Tok.LT, ">": Tok.GT,
            "(": Tok.LPAREN, ")": Tok.RPAREN,
            "{": Tok.LBRACE, "}": Tok.RBRACE, ";": Tok.SEMI}


@dataclass(frozen=True)
class Token:
    kind: Tok
    value: str
    line: int
    col: int


class LexError(Exception):
    def __init__(self, msg, line, col):
        super().__init__(f"{msg} at {line}:{col}")


class Lexer:
    def __init__(self, src: str):
        self.src = src
        self.pos = 0
        self.line = 1
        self.col = 1

    def _peek(self, off=0):
        i = self.pos + off
        return self.src[i] if i \< len(self.src) else ""

    def _advance(self) -> str:
        ch = self.src[self.pos]
        self.pos += 1
        if ch == "\n":
            self.line += 1
            self.col = 1
        else:
            self.col += 1
        return ch

    def _skip_trivia(self):
        while self.pos \< len(self.src):
            ch = self._peek()
            if ch in " \t\r\n":
                self._advance()
            elif ch == "/" and self._peek(1) == "/":
                while self.pos \< len(self.src) and self._peek() != "\n":
                    self._advance()
            else:
                return

    def _ident(self):
        chars = []
        while self._peek().isalnum() or self._peek() == "_":
            chars.append(self._advance())
        text = "".join(chars)
        return KEYWORDS.get(text, Tok.IDENT), text

    def _number(self):
        chars = []
        while self._peek().isdigit():
            chars.append(self._advance())
        if self._peek().isalpha() or self._peek() == "_":
            raise LexError("invalid suffix on numeric literal",
                           self.line, self.col)
        return Tok.INT, "".join(chars)

    def _string(self):
        self._advance()  # opening quote
        chars = []
        escapes = {"n": "\n", "t": "\t", '"': '"'}
        while True:
            if self.pos >= len(self.src) or self._peek() == "\n":
                raise LexError("unterminated string",
                               self.line, self.col)
            ch = self._advance()
            if ch == '"':
                break
            if ch == "\\":
                esc = self._advance()
                if esc not in escapes:
                    raise LexError(f"unknown escape \\{esc}",
                                   self.line, self.col)
                chars.append(escapes[esc])
            else:
                chars.append(ch)
        return Tok.STRING, "".join(chars)

    def next_token(self) -> Token:
        self._skip_trivia()
        line, col = self.line, self.col
        if self.pos >= len(self.src):
            return Token(Tok.EOF, "", line, col)
        ch = self._peek()
        if ch.isalpha() or ch == "_":
            kind, val = self._ident()
        elif ch.isdigit():
            kind, val = self._number()
        elif ch == '"':
            kind, val = self._string()
        else:
            two = self.src[self.pos:self.pos + 2]
            if two in TWO_CHAR:
                self._advance(); self._advance()
                return Token(TWO_CHAR[two], two, line, col)
            if ch in ONE_CHAR:
                self._advance()
                return Token(ONE_CHAR[ch], ch, line, col)
            raise LexError(f"unexpected character {ch!r}", line, col)
        return Token(kind, val, line, col)

    def tokens(self):
        out = []
        while True:
            tok = self.next_token()
            out.append(tok)
            if tok.kind is Tok.EOF:
                return out

Inline tests — ফাইলের শেষে:

def _test():
    L = Lexer
    assert L("let x = 10;").tokens() == [
        Token(Tok.LET, "let", 1, 1),
        Token(Tok.IDENT, "x", 1, 5),
        Token(Tok.EQUALS, "=", 1, 7),
        Token(Tok.INT, "10", 1, 9),
        Token(Tok.SEMI, ";", 1, 11),
        Token(Tok.EOF, "", 1, 12),
    ]
    assert L("if x \<= 3 {}").tokens()[2].kind is Tok.LTE
    assert L("a!=b").tokens()[1].value == "!="
    assert L("true").tokens()[0].kind is Tok.TRUE
    assert L("trueX").tokens()[0].kind is Tok.IDENT   # maximal munch
    assert L("// gone\nx").tokens()[0] == Token(Tok.IDENT, "x", 2, 1)
    assert L('"a\\nb"').tokens()[0].value == "a\nb"

    for bad in ['x = "oops', "42abc;", "x @ y;", '"a\\qb"']:
        try:
            L(bad).tokens()
            raise AssertionError(f"no error for {bad!r}")
        except LexError as e:
            assert ":" in str(e)

    print("all lexer tests passed")

if __name__ == "__main__":
    _test()

Expected:

$ python3 -m toylang.lexer
all lexer tests passed

নিজে বাড়ান:

1. Token-এ end_col — caret underline (^^^) diagnostics-এর ভিত্তি।
2. Bangla identifiers: combining marks, ZWNJ, NFC/NFD normalization — bangla-encoding লেসনের সেই theory lexer-ই প্রথম practical pain।