diagnostics

src/ast.ts

@@ -1,3 +1,5 @@
+import { Loc } from "./diagnostics.ts";
+
 export class File {
     constructor(
         public readonly id: number,
@@ -13,7 +15,7 @@ export class File {
 export class Block {
     constructor(
         public readonly id: number,
-        public readonly line: number,
+        public readonly loc: Loc,
         public readonly stmts: Stmt[],
         public readonly expr?: Expr,
     ) {}
@@ -30,7 +32,7 @@ export class Block {
 export class Stmt {
     constructor(
         public readonly id: number,
-        public readonly line: number,
+        public readonly loc: Loc,
         public readonly kind: StmtKind,
     ) {}
 
@@ -72,7 +74,7 @@ export type StmtKind =
 export class Expr {
     constructor(
         public readonly id: number,
-        public readonly line: number,
+        public readonly loc: Loc,
         public readonly kind: ExprKind,
     ) {}
 
@@ -111,7 +113,7 @@ export type ExprKind =
 export class Param {
     constructor(
         public readonly id: number,
-        public readonly line: number,
+        public readonly loc: Loc,
         public readonly pat: Pat,
         public readonly ty?: Ty,
     ) {}
@@ -128,7 +130,7 @@ export class Param {
 export class Pat {
     constructor(
         public readonly id: number,
-        public readonly line: number,
+        public readonly loc: Loc,
         public readonly kind: PatKind,
     ) {}
 
@@ -153,7 +155,7 @@ export type PatKind =
 export class Ty {
     constructor(
         public readonly id: number,
-        public readonly line: number,
+        public readonly loc: Loc,
         public readonly kind: TyKind,
     ) {}
 
@@ -205,70 +207,70 @@ export class AstBuilder {
     }
 
     block(
-        line: number,
+        loc: Loc,
         stmts: Stmt[],
         expr?: Expr,
     ): Block {
         return new Block(
             this.id++,
-            line,
+            loc,
             stmts,
             expr,
         );
     }
 
     stmt<Tag extends StmtKind["tag"]>(
-        line: number,
+        loc: Loc,
         tag: Tag,
         kind: Omit<StmtKind & { tag: Tag }, "tag">,
     ): Stmt {
         return new Stmt(
             this.id++,
-            line,
+            loc,
             { tag, ...kind } as StmtKind,
         );
     }
 
     expr<Tag extends ExprKind["tag"]>(
-        line: number,
+        loc: Loc,
         tag: Tag,
         kind: Omit<StmtKind & { tag: Tag }, "tag">,
     ): Expr {
         return new Expr(
             this.id++,
-            line,
+            loc,
             { tag, ...kind } as ExprKind,
         );
     }
 
     param(
-        line: number,
+        loc: Loc,
         pat: Pat,
         ty?: Ty,
     ): Param {
-        return new Param(this.id++, line, pat, ty);
+        return new Param(this.id++, loc, pat, ty);
     }
 
     pat<Tag extends PatKind["tag"]>(
-        line: number,
+        loc: Loc,
         tag: Tag,
         kind: Omit<PatKind & { tag: Tag }, "tag">,
     ): Pat {
         return new Pat(
             this.id++,
-            line,
+            loc,
             { tag, ...kind } as PatKind,
         );
     }
 
     ty<Tag extends TyKind["tag"]>(
-        line: number,
+        loc: Loc,
         tag: Tag,
         kind: Omit<TyKind & { tag: Tag }, "tag">,
     ): Ty {
         return new Ty(
             this.id++,
-            line,
+            loc,
             { tag, ...kind } as TyKind,
         );
     }

src/cx.ts

@@ -0,0 +1,40 @@
+import type { Tok } from "./tok.ts";
+import type * as ast from "./ast.ts";
+import type { Syms } from "./resolve.ts";
+
+export type FileInfo = {
+    id: number;
+    filename: string;
+    text: string;
+    toks?: Tok[];
+    ast?: ast.File;
+    syms?: Syms;
+};
+
+export class Cx {
+    private fileIdCounter = 0;
+    private fileInfoMap = new Map<number, FileInfo>();
+
+    async readFile(filename: string): Promise<number> {
+        const id = this.fileIdCounter++;
+        const text = await Deno.readTextFile(filename);
+        this.fileInfoMap.set(id, { id, filename, text });
+        return id;
+    }
+
+    file(id: number): Readonly<FileInfo> {
+        return this.fileInfoMap.get(id)!;
+    }
+
+    setFileToks(id: number, toks: Tok[]) {
+        this.fileInfoMap.get(id)!.toks = toks;
+    }
+
+    setFileAst(id: number, ast: ast.File) {
+        this.fileInfoMap.get(id)!.ast = ast;
+    }
+
+    setFileSyms(id: number, syms: Syms) {
+        this.fileInfoMap.get(id)!.syms = syms;
+    }
+}

src/diagnostics.ts

@@ -0,0 +1,59 @@
+import { Cx } from "./cx.ts";
+
+export type Loc = {
+    fileId: number;
+    idx: number;
+    line: number;
+    col: number;
+};
+
+export class Reporter {
+    constructor(
+        private cx: Cx,
+    ) {}
+
+    error(loc: Loc, message: string) {
+        this.printDiagnostic(loc, message, "error", "red");
+    }
+
+    info(loc: Loc, message: string) {
+        this.printDiagnostic(loc, message, "info", "blue");
+    }
+
+    private printDiagnostic(
+        { fileId, idx, line, col }: Loc,
+        message: string,
+        type: string,
+        color: string,
+    ) {
+        const { filename, text } = this.cx.file(fileId);
+        const lineNum = line.toString();
+
+        const start = text.lastIndexOf("\n", idx) + 1;
+        const end = text.includes("\n", idx)
+            ? text.indexOf("\n", idx)
+            : text.length;
+        const section = text.slice(start, end);
+
+        console.error(
+            "" +
+                `%c${type}%c: ${message}\n` +
+                ` %c--> ${filename}:${line}:${col}\n` +
+                ` ${" ".repeat(lineNum.length)}%c|\n` +
+                ` %c${lineNum}%c|%c${section}\n` +
+                ` ${" ".repeat(lineNum.length)}` +
+                `%c|${" ".repeat(col - 1)}%c^ %c${message}%c`,
+            `font-weight: bold; color: ${color}`,
+            "font-weight: bold; color: while",
+            "color: cyan",
+            "color: gray",
+            "color: light-gray",
+            "color: gray",
+            "color: light-gray",
+            "color: gray",
+            `font-weight: bold; color: ${color}`,
+            "font-weight: bold; color: while",
+            "",
+        );
+    }
+}

src/main.ts

@@ -2,16 +2,16 @@ import * as yaml from "jsr:@std/yaml";
 import { tokenize } from "./tok.ts";
 import { Parser } from "./parse.ts";
 import { Resolver } from "./resolve.ts";
+import { Cx } from "./cx.ts";
 
 async function main() {
-    const text = await Deno.readTextFile(Deno.args[0]);
+    const cx = new Cx();
-    const toks = tokenize(text);
-    console.log({ toks });
-    return;
 
-    const parser = new Parser(toks);
+    const fileId = await cx.readFile(Deno.args[0]);
-    const file = parser.parseFile();
+    tokenize(cx, fileId);
-    if (parser.errorOccured) {
+
+    const parseResult = Parser.parseFile(cx, fileId);
+    if (!parseResult.ok) {
         console.error("parsing failed");
         Deno.exit(1);
     }
@@ -19,13 +19,13 @@ async function main() {
     // console.log(yaml.stringify({ file }, { skipInvalid: true, indent: 2 }));
     // console.log(JSON.stringify({ file }, null, 2));
 
-    const resolver = new Resolver();
+    const resolveResult = Resolver.resolveFile(cx, fileId);
-    const syms = resolver.resolveFile(file);
+    if (!resolveResult.ok) {
-    if (resolver.errorOccured) {
         console.error("resolving failed");
         Deno.exit(1);
     }
 
+    const syms = cx.file(fileId).syms!;
     console.log(syms);
 
     // console.log(cx);

src/parse.ts

@@ -1,16 +1,32 @@
 import { AstBuilder, Block, Expr, File, Param, Pat, Stmt, Ty } from "./ast.ts";
+import { Cx } from "./cx.ts";
+import { Loc, Reporter } from "./diagnostics.ts";
 import { Tok } from "./tok.ts";
 
 const t = new AstBuilder();
 
+export type ParseResult = { ok: true } | { ok: false } & object;
+
 export class Parser {
     private i = 0;
     private eaten?: Tok;
+    private errorOccured = false;
 
-    constructor(
+    private constructor(
+        private reporter: Reporter,
         private toks: Tok[],
     ) {}
 
+    static parseFile(cx: Cx, fileId: number): ParseResult {
+        const parser = new Parser(
+            new Reporter(cx),
+            cx.file(fileId).toks!,
+        );
+        const ast = parser.parseFile();
+        cx.setFileAst(fileId, ast);
+        return { ok: !parser.errorOccured };
+    }
+
     parseFile(): File {
         const stmts: Stmt[] = [];
         while (!this.done) {
@@ -20,7 +36,7 @@ export class Parser {
     }
 
     parseItem(): Stmt {
-        const line = this.line();
+        const loc = this.loc();
         if (this.test("fn")) {
             return this.parseFn();
         } else if (this.test("let")) {
@@ -28,19 +44,19 @@ export class Parser {
         } else {
             this.expect("item");
             this.step();
-            return t.stmt(line, "error", {});
+            return t.stmt(loc, "error", {});
         }
     }
 
     parseBlock(): Block {
-        const line = this.line();
+        const loc = this.loc();
         this.step();
 
         const stmts: Stmt[] = [];
         let expr: Expr | undefined = undefined;
 
         while (!this.done && !this.test("}")) {
-            const line = this.line();
+            const loc = this.loc();
             if (this.test("fn")) {
                 stmts.push(this.parseFn());
             } else if (this.test("let")) {
@@ -52,10 +68,10 @@ export class Parser {
                     const rhs = this.parseExpr();
                     this.expect(";");
                     stmts.push(
-                        t.stmt(line, "assign", { place: lhs, expr: rhs }),
+                        t.stmt(loc, "assign", { place: lhs, expr: rhs }),
                     );
                 } else if (this.eat(";")) {
-                    stmts.push(t.stmt(line, "expr", { expr: lhs }));
+                    stmts.push(t.stmt(loc, "expr", { expr: lhs }));
                 } else if (this.test("}")) {
                     expr = lhs;
                     break;
@@ -66,19 +82,19 @@ export class Parser {
         }
 
         this.expect("}");
-        return t.block(line, stmts, expr);
+        return t.block(loc, stmts, expr);
     }
 
     parseFn(): Stmt {
-        const line = this.line();
+        const loc = this.loc();
         this.step();
         if (!this.expect("ident")) {
-            return t.stmt(line, "error", {});
+            return t.stmt(loc, "error", {});
         }
         const ident = this.eaten!.value!;
         const params: Param[] = [];
         if (!this.expect("(")) {
-            return t.stmt(line, "error", {});
+            return t.stmt(loc, "error", {});
         }
         if (!this.done && !this.test(")")) {
             params.push(this.parseParam());
@@ -90,7 +106,7 @@ export class Parser {
             }
         }
         if (!this.expect(")")) {
-            return t.stmt(line, "error", {});
+            return t.stmt(loc, "error", {});
         }
         let retTy: Ty | undefined = undefined;
         if (this.eat("->")) {
@@ -98,21 +114,21 @@ export class Parser {
         }
         if (!this.test("{")) {
             this.expect("{");
-            return t.stmt(line, "error", {});
+            return t.stmt(loc, "error", {});
         }
         const body = this.parseBlock();
-        return t.stmt(line, "fn", { ident, params, retTy, body });
+        return t.stmt(loc, "fn", { ident, params, retTy, body });
     }
 
     parseLet(): Stmt {
-        const line = this.line();
+        const loc = this.loc();
         this.step();
         const param = this.parseParam();
         if (!this.expect("=")) {
-            return t.stmt(line, "error", {});
+            return t.stmt(loc, "error", {});
         }
         const init = this.parseExpr();
-        return t.stmt(line, "let", { param, init });
+        return t.stmt(loc, "let", { param, init });
     }
 
     parseExpr(): Expr {
@@ -122,7 +138,7 @@ export class Parser {
     parsePostfix(): Expr {
         let expr = this.parseOp();
         while (true) {
-            const line = this.line();
+            const loc = this.loc();
             if (this.eat("(")) {
                 const args: Expr[] = [];
                 if (!this.done && !this.test(")")) {
@@ -135,9 +151,9 @@ export class Parser {
                     }
                 }
                 if (!this.expect(")")) {
-                    return t.expr(line, "error", {});
+                    return t.expr(loc, "error", {});
                 }
-                expr = t.expr(line, "call", { expr, args });
+                expr = t.expr(loc, "call", { expr, args });
             } else {
                 break;
             }
@@ -146,75 +162,75 @@ export class Parser {
     }
 
     parseOp(): Expr {
-        const line = this.line();
+        const loc = this.loc();
         if (this.eat("ident")) {
             const ident = this.eaten!.value!;
-            return t.expr(line, "ident", { ident });
+            return t.expr(loc, "ident", { ident });
         } else if (this.eat("int")) {
             const value = this.eaten!.value!;
-            return t.expr(line, "int", { value });
+            return t.expr(loc, "int", { value });
         } else if (this.eat("char")) {
             const value = this.eaten!.value!;
-            return t.expr(line, "int", { value });
+            return t.expr(loc, "int", { value });
         } else if (this.eat("str")) {
             const value = this.eaten!.value!;
-            return t.expr(line, "int", { value });
+            return t.expr(loc, "int", { value });
         } else {
             this.expect("expr");
             this.step();
-            return t.expr(line, "error", {});
+            return t.expr(loc, "error", {});
         }
     }
 
     parseParam(): Param {
-        const line = this.line();
+        const loc = this.loc();
         const pat = this.parsePat();
         let ty: Ty | undefined = undefined;
         if (this.eat(":")) {
             ty = this.parseTy();
         }
-        return t.param(line, pat, ty);
+        return t.param(loc, pat, ty);
     }
 
     parsePat(): Pat {
-        const line = this.line();
+        const loc = this.loc();
         if (this.eat("ident")) {
             const ident = this.eaten!.value!;
-            return t.pat(line, "ident", { ident });
+            return t.pat(loc, "ident", { ident });
         } else {
             this.expect("pat");
             this.step();
-            return t.pat(line, "error", {});
+            return t.pat(loc, "error", {});
         }
     }
 
     parseTy(): Ty {
-        const line = this.line();
+        const loc = this.loc();
         if (this.eat("int")) {
-            return t.ty(line, "int", {});
+            return t.ty(loc, "int", {});
         } else if (this.eat("bool")) {
-            return t.ty(line, "bool", {});
+            return t.ty(loc, "bool", {});
         } else if (this.eat("char")) {
-            return t.ty(line, "char", {});
+            return t.ty(loc, "char", {});
         } else if (this.eat("str")) {
-            return t.ty(line, "str", {});
+            return t.ty(loc, "str", {});
         } else if (this.eat("ident")) {
             const ident = this.eaten!.value!;
-            return t.ty(line, "ident", { ident });
+            return t.ty(loc, "ident", { ident });
         } else {
             this.expect("ty");
             this.step();
-            return t.ty(line, "error", {});
+            return t.ty(loc, "error", {});
         }
     }
 
     private expect(type: string): boolean {
-        const line = this.line();
+        const loc = this.loc();
         if (!this.eat(type)) {
             if (this.done) {
-                this.error(line, `expected '${type}', got 'eof'`);
+                this.error(loc, `expected '${type}', got 'eof'`);
             } else {
-                this.error(line, `expected '${type}', got '${this.tok.type}'`);
+                this.error(loc, `expected '${type}', got '${this.tok.type}'`);
             }
             return false;
         }
@@ -238,8 +254,8 @@ export class Parser {
         this.i += 1;
     }
 
-    private line(): number {
+    private loc(): Loc {
-        return this.tok.line;
+        return this.tok.loc;
     }
 
     private get tok(): Tok {
@@ -250,15 +266,8 @@ export class Parser {
         return this.i >= this.toks.length;
     }
 
-    public errorOccured = false;
+    private error(loc: Loc, message: string) {
-    private error(line: number, message: string) {
         this.errorOccured = true;
-        console.error(
+        this.reporter.error(loc, message);
-            `%cerror%c: ${message}\n %c--> line ${line}%c`,
-            "font-weight: bold; color: red",
-            "font-weight: bold; color: while",
-            "color: cyan",
-            "",
-        );
     }
 }

src/resolve.ts

@@ -1,13 +1,6 @@
-import {
+import * as ast from "./ast.ts";
-    Block,
+import { Cx } from "./cx.ts";
-    Expr,
+import { Loc, Reporter } from "./diagnostics.ts";
-    File,
-    Pat,
-    Stmt,
-    Ty,
-    Visitor,
-    VisitorBreak,
-} from "./ast.ts";
 
 class Res {
     constructor(
@@ -40,23 +33,23 @@ export class Def {
         }
     }
 
-    line(): number {
+    loc(): Loc {
         const k = this.kind;
         switch (k.tag) {
             case "fn":
-                return k.stmt.line;
+                return k.stmt.loc;
             case "param":
-                return k.pat.line;
+                return k.pat.loc;
             case "let":
-                return k.pat.line;
+                return k.pat.loc;
         }
     }
 }
 
 export type DefKind =
-    | { tag: "fn"; stmt: Stmt }
+    | { tag: "fn"; stmt: ast.Stmt }
-    | { tag: "param"; stmt: Stmt; pat: Pat }
+    | { tag: "param"; stmt: ast.Stmt; pat: ast.Pat }
-    | { tag: "let"; stmt: Stmt; pat: Pat };
+    | { tag: "let"; stmt: ast.Stmt; pat: ast.Pat };
 
 type DefineResult =
     | { ok: true }
@@ -110,42 +103,57 @@ export class Syms {
         private astNodeDefs: Map<number, Res>,
     ) {}
 
-    exprRes(expr: Expr): Res {
+    exprRes(expr: ast.Expr): Res {
         const res = this.astNodeDefs.get(expr.id);
         if (!res) throw new Error();
         return res;
     }
 
-    patRes(pat: Pat): Res {
+    patRes(pat: ast.Pat): Res {
         const res = this.astNodeDefs.get(pat.id);
         if (!res) throw new Error();
         return res;
     }
 
-    tyRes(ty: Ty): Res {
+    tyRes(ty: ast.Ty): Res {
         const res = this.astNodeDefs.get(ty.id);
         if (!res) throw new Error();
         return res;
     }
 }
 
-export class Resolver implements Visitor {
+export type ResolveResult = { ok: true } | { ok: false };
+
+export class Resolver implements ast.Visitor {
     private rib = new Rib({ tag: "root" });
     private scopeStack: Rib[] = [];
     private astNodeDefs = new Map<number, Res>();
+    private errorOccured = false;
 
-    resolveFile(file: File): Syms {
+    private constructor(
+        private reporter: Reporter,
+    ) {}
+
+    static resolveFile(cx: Cx, fileId: number): ResolveResult {
+        const file = cx.file(fileId);
+        const resolver = new Resolver(new Reporter(cx));
+        const syms = resolver.resolveFile(file.ast!);
+        cx.setFileSyms(fileId, syms);
+        return { ok: !resolver.errorOccured };
+    }
+
+    resolveFile(file: ast.File): Syms {
         file.visit({
-            visitStmt: (stmt): void | VisitorBreak => {
+            visitStmt: (stmt): void | ast.VisitorBreak => {
                 const k = stmt.kind;
                 if (k.tag === "fn") {
                     this.tryDefine(
-                        stmt.line,
+                        stmt.loc,
                         k.ident,
                         new Def({ tag: "fn", stmt }),
                     );
                 }
-                return VisitorBreak;
+                return ast.VisitorBreak;
             },
         });
 
@@ -153,36 +161,36 @@ export class Resolver implements Visitor {
         return new Syms(this.astNodeDefs);
     }
 
-    visitStmt(stmt: Stmt): void | VisitorBreak {
+    visitStmt(stmt: ast.Stmt): void | ast.VisitorBreak {
         const k = stmt.kind;
         if (k.tag === "fn") {
             this.saveScope();
             this.pushRib("fn", {});
 
             k.body.visit({
-                visitStmt: (stmt): void | VisitorBreak => {
+                visitStmt: (stmt): void | ast.VisitorBreak => {
                     const k = stmt.kind;
                     if (k.tag === "fn") {
                         this.tryDefine(
-                            stmt.line,
+                            stmt.loc,
                             k.ident,
                             new Def({ tag: "fn", stmt }),
                         );
                     }
-                    return VisitorBreak;
+                    return ast.VisitorBreak;
                 },
             });
 
             this.pushRib("param", {});
             for (const param of k.params) {
                 param.pat.visit({
-                    visitPat: (pat: Pat) => {
+                    visitPat: (pat: ast.Pat) => {
                         const k = pat.kind;
                         if (k.tag !== "ident") {
                             return;
                         }
                         this.tryDefine(
-                            pat.line,
+                            pat.loc,
                             k.ident,
                             new Def({ tag: "param", stmt, pat }),
                         );
@@ -194,17 +202,17 @@ export class Resolver implements Visitor {
             k.body.visit(this);
 
             this.restoreScope();
-            return VisitorBreak;
+            return ast.VisitorBreak;
         } else if (k.tag === "let") {
             this.pushRib("let", {});
             k.param.pat.visit({
-                visitPat: (pat: Pat) => {
+                visitPat: (pat: ast.Pat) => {
                     const k = pat.kind;
                     if (k.tag !== "ident") {
                         return;
                     }
                     this.tryDefine(
-                        pat.line,
+                        pat.loc,
                         k.ident,
                         new Def({ tag: "let", stmt, pat }),
                     );
@@ -213,30 +221,30 @@ export class Resolver implements Visitor {
         }
     }
 
-    visitBlock(block: Block): void | VisitorBreak {
+    visitBlock(block: ast.Block): void | ast.VisitorBreak {
         this.saveScope();
         this.pushRib("block", {});
         block.stmts.forEach((stmt) => stmt.visit(this));
         block.expr?.visit(this);
         this.restoreScope();
-        return VisitorBreak;
+        return ast.VisitorBreak;
     }
 
-    visitExpr(expr: Expr): void | VisitorBreak {
+    visitExpr(expr: ast.Expr): void | ast.VisitorBreak {
         const k = expr.kind;
         if (k.tag !== "ident") {
             return;
         }
-        const res = this.resolve(expr.line, k.ident);
+        const res = this.resolve(expr.loc, k.ident);
         this.astNodeDefs.set(expr.id, res);
     }
 
-    visitTy(ty: Ty): void | VisitorBreak {
+    visitTy(ty: ast.Ty): void | ast.VisitorBreak {
         const k = ty.kind;
         if (k.tag !== "ident") {
             return;
         }
-        const res = this.resolve(ty.line, k.ident);
+        const res = this.resolve(ty.loc, k.ident);
         this.astNodeDefs.set(ty.id, res);
     }
 
@@ -255,11 +263,11 @@ export class Resolver implements Visitor {
         this.rib = this.scopeStack.pop()!;
     }
 
-    private tryDefine(line: number, ident: string, def: Def) {
+    private tryDefine(loc: Loc, ident: string, def: Def) {
         const result = this.define(ident, def);
         if (!result.ok) {
-            this.error(line, `redefinition of '${ident}'`);
+            this.error(loc, `redefinition of '${ident}'`);
-            this.info(result.originalDef.line(), `'${ident}' defined here`);
+            this.info(result.originalDef.loc(), `'${ident}' defined here`);
         }
     }
 
@@ -267,33 +275,20 @@ export class Resolver implements Visitor {
         return this.rib.define(ident, def);
     }
 
-    private resolve(line: number, ident: string): Res {
+    private resolve(loc: Loc, ident: string): Res {
         const res = this.rib.resolve(ident);
         if (res.is("unresolved")) {
-            this.error(line, `unresolved ident '${ident}'`);
+            this.error(loc, `unresolved ident '${ident}'`);
         }
         return res;
     }
 
-    public errorOccured = false;
+    private error(loc: Loc, message: string) {
-    private error(line: number, message: string) {
         this.errorOccured = true;
-        console.error(
+        this.reporter.error(loc, message);
-            `%cerror%c: ${message}\n %c--> line ${line}%c`,
-            "font-weight: bold; color: red",
-            "font-weight: bold; color: while",
-            "color: cyan",
-            "",
-        );
     }
 
-    private info(line: number, message: string) {
+    private info(loc: Loc, message: string) {
-        console.error(
+        this.reporter.info(loc, message);
-            `%cinfo%c: ${message}\n %c--> line ${line}%c`,
-            "font-weight: bold; color: blue",
-            "font-weight: bold; color: while",
-            "color: cyan",
-            "",
-        );
     }
 }

src/tok.ts

@@ -1,40 +1,49 @@
+import { Cx } from "./cx.ts";
+import { Loc, Reporter } from "./diagnostics.ts";
+
 export type Tok = {
     type: string;
-    idx: number;
+    loc: Loc;
-    line: number;
+    value: string;
-    col: number;
-    length: number;
-    value?: string;
 };
-const keywords = ["true", "false", "bool", "int", "char", "str", "fn", "let"];
 
-export function tokenize(text: string): Tok[] {
+const keywords = new Set([
-    const rules: Record<string, { match: RegExp; ignore?: boolean }> = {
+    "true",
-        "whitespace": { match: /^[ \t\r]+/, ignore: true },
+    "false",
-        "newline": { match: /^\n/s, ignore: true },
+    "bool",
-        "linecomment": { match: /^\/\/[^\n]*/, ignore: true },
+    "int",
-        "blockcomment": { match: /^\/\*.*?\*\//s, ignore: true },
+    "char",
-        _keywords: {
+    "str",
-            match: new RegExp(
+    "fn",
-                `^(${keywords.map((s) => `(?:${s})`).join("|")})`,
+    "let",
-            ),
+]);
-        },
+
-        _identity: {
+const rules: Record<string, { match: RegExp; ignore?: boolean }> = {
-            match: new RegExp(
+    "whitespace": { match: /^[ \t\r]+/, ignore: true },
-                `^(?:(?:\-\>)|[${RegExp.escape("()[]{}+-*/,.;:!=<>&|?")}])`,
+    "newline": { match: /^\n/s, ignore: true },
-            ),
+    "linecomment": { match: /^\/\/[^\n]*/, ignore: true },
-        },
+    "blockcomment": { match: /^\/\*.*?\*\//s, ignore: true },
-        "ident": { match: /^[a-zA-Z_][a-zA-Z0-9_]*/ },
+    _identity: {
-        "int": { match: /^[0-9_]+/ },
+        match: new RegExp(
-        "char": { match: /^'(?:(?:\\.)|[^'\n])'/ },
+            `^(?:(?:\-\>)|[${RegExp.escape("()[]{}+-*/,.;:!=<>&|?")}])`,
-        "str": { match: /^"(?:(?:\\.)|[^"])*"/s },
+        ),
-    };
+    },
+    "ident": { match: /^[a-zA-Z_][a-zA-Z0-9_]*/ },
+    "int": { match: /^[0-9_]+/ },
+    "char": { match: /^'(?:(?:\\.)|[^'\n])'/ },
+    "str": { match: /^"(?:(?:\\.)|[^"])*"/s },
+};
+
+export function tokenize(cx: Cx, fileId: number) {
+    const { text } = cx.file(fileId);
+    const rep = new Reporter(cx);
 
     const toks: Tok[] = [];
     let idx = 0;
     let line = 1;
     let col = 1;
     while (idx < text.length) {
+        const loc: Loc = { fileId, idx, line, col };
         let found = false;
         for (const [id, rule] of Object.entries(rules)) {
             const match = text.slice(idx).match(rule.match);
@@ -53,10 +62,15 @@ export function tokenize(text: string): Tok[] {
 
             if (rule.ignore) continue;
 
-            const length = match[0].length;
+            const tok: Tok = {
-            const tok: Tok = { type: id, idx, line, col, length };
+                type: id,
-            if (id === "_keywords" || id === "_identity") {
+                loc,
+                value: match[0],
+            };
+            if (id === "_identity") {
                 tok.type = match[0];
+            } else if (id === "ident") {
+                tok.type = keywords.has(match[0]) ? match[0] : id;
             } else {
                 tok.value = match[0];
             }
@@ -64,20 +78,10 @@ export function tokenize(text: string): Tok[] {
             break;
         }
         if (!found) {
-            printError(line, `invalid character '${text[idx]}'`);
+            rep.error(loc, `invalid character '${text[idx]}'`);
             idx += 1;
         }
     }
 
-    return toks;
+    cx.setFileToks(fileId, toks);
-}
-
-function printError(line: number, message: string) {
-    console.error(
-        `%cerror%c: ${message}\n %c--> line ${line}%c`,
-        "font-weight: bold; color: red",
-        "font-weight: bold; color: while",
-        "color: cyan",
-        "",
-    );
 }