From 697e928802d797b6cd3364bda3bf32f3c0582f56 Mon Sep 17 00:00:00 2001 From: sfja Date: Fri, 12 Dec 2025 02:58:48 +0100 Subject: [PATCH] diagnostics --- src/ast.ts | 38 ++++++++------- src/cx.ts | 40 +++++++++++++++ src/diagnostics.ts | 59 ++++++++++++++++++++++ src/main.ts | 20 ++++---- src/parse.ts | 113 ++++++++++++++++++++++-------------------- src/resolve.ts | 119 ++++++++++++++++++++++----------------------- src/tok.ts | 88 +++++++++++++++++---------------- 7 files changed, 293 insertions(+), 184 deletions(-) create mode 100644 src/cx.ts create mode 100644 src/diagnostics.ts diff --git a/src/ast.ts b/src/ast.ts index 80e3d33..3c50385 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -1,3 +1,5 @@ +import { Loc } from "./diagnostics.ts"; + export class File { constructor( public readonly id: number, @@ -13,7 +15,7 @@ export class File { export class Block { constructor( public readonly id: number, - public readonly line: number, + public readonly loc: Loc, public readonly stmts: Stmt[], public readonly expr?: Expr, ) {} @@ -30,7 +32,7 @@ export class Block { export class Stmt { constructor( public readonly id: number, - public readonly line: number, + public readonly loc: Loc, public readonly kind: StmtKind, ) {} @@ -72,7 +74,7 @@ export type StmtKind = export class Expr { constructor( public readonly id: number, - public readonly line: number, + public readonly loc: Loc, public readonly kind: ExprKind, ) {} @@ -111,7 +113,7 @@ export type ExprKind = export class Param { constructor( public readonly id: number, - public readonly line: number, + public readonly loc: Loc, public readonly pat: Pat, public readonly ty?: Ty, ) {} @@ -128,7 +130,7 @@ export class Param { export class Pat { constructor( public readonly id: number, - public readonly line: number, + public readonly loc: Loc, public readonly kind: PatKind, ) {} @@ -153,7 +155,7 @@ export type PatKind = export class Ty { constructor( public readonly id: number, - public readonly line: number, + public readonly loc: Loc, public readonly kind: TyKind, ) {} @@ -205,70 +207,70 @@ export class AstBuilder { } block( - line: number, + loc: Loc, stmts: Stmt[], expr?: Expr, ): Block { return new Block( this.id++, - line, + loc, stmts, expr, ); } stmt( - line: number, + loc: Loc, tag: Tag, kind: Omit, ): Stmt { return new Stmt( this.id++, - line, + loc, { tag, ...kind } as StmtKind, ); } expr( - line: number, + loc: Loc, tag: Tag, kind: Omit, ): Expr { return new Expr( this.id++, - line, + loc, { tag, ...kind } as ExprKind, ); } param( - line: number, + loc: Loc, pat: Pat, ty?: Ty, ): Param { - return new Param(this.id++, line, pat, ty); + return new Param(this.id++, loc, pat, ty); } pat( - line: number, + loc: Loc, tag: Tag, kind: Omit, ): Pat { return new Pat( this.id++, - line, + loc, { tag, ...kind } as PatKind, ); } ty( - line: number, + loc: Loc, tag: Tag, kind: Omit, ): Ty { return new Ty( this.id++, - line, + loc, { tag, ...kind } as TyKind, ); } diff --git a/src/cx.ts b/src/cx.ts new file mode 100644 index 0000000..ac1749f --- /dev/null +++ b/src/cx.ts @@ -0,0 +1,40 @@ +import type { Tok } from "./tok.ts"; +import type * as ast from "./ast.ts"; +import type { Syms } from "./resolve.ts"; + +export type FileInfo = { + id: number; + filename: string; + text: string; + toks?: Tok[]; + ast?: ast.File; + syms?: Syms; +}; + +export class Cx { + private fileIdCounter = 0; + private fileInfoMap = new Map(); + + async readFile(filename: string): Promise { + const id = this.fileIdCounter++; + const text = await Deno.readTextFile(filename); + this.fileInfoMap.set(id, { id, filename, text }); + return id; + } + + file(id: number): Readonly { + return this.fileInfoMap.get(id)!; + } + + setFileToks(id: number, toks: Tok[]) { + this.fileInfoMap.get(id)!.toks = toks; + } + + setFileAst(id: number, ast: ast.File) { + this.fileInfoMap.get(id)!.ast = ast; + } + + setFileSyms(id: number, syms: Syms) { + this.fileInfoMap.get(id)!.syms = syms; + } +} diff --git a/src/diagnostics.ts b/src/diagnostics.ts new file mode 100644 index 0000000..cb1a074 --- /dev/null +++ b/src/diagnostics.ts @@ -0,0 +1,59 @@ +import { Cx } from "./cx.ts"; + +export type Loc = { + fileId: number; + idx: number; + line: number; + col: number; +}; + +export class Reporter { + constructor( + private cx: Cx, + ) {} + + error(loc: Loc, message: string) { + this.printDiagnostic(loc, message, "error", "red"); + } + + info(loc: Loc, message: string) { + this.printDiagnostic(loc, message, "info", "blue"); + } + + private printDiagnostic( + { fileId, idx, line, col }: Loc, + message: string, + type: string, + color: string, + ) { + const { filename, text } = this.cx.file(fileId); + const lineNum = line.toString(); + + const start = text.lastIndexOf("\n", idx) + 1; + const end = text.includes("\n", idx) + ? text.indexOf("\n", idx) + : text.length; + const section = text.slice(start, end); + + console.error( + "" + + `%c${type}%c: ${message}\n` + + ` %c--> ${filename}:${line}:${col}\n` + + ` ${" ".repeat(lineNum.length)}%c|\n` + + ` %c${lineNum}%c|%c${section}\n` + + ` ${" ".repeat(lineNum.length)}` + + `%c|${" ".repeat(col - 1)}%c^ %c${message}%c`, + `font-weight: bold; color: ${color}`, + "font-weight: bold; color: while", + "color: cyan", + "color: gray", + "color: light-gray", + "color: gray", + "color: light-gray", + "color: gray", + `font-weight: bold; color: ${color}`, + "font-weight: bold; color: while", + "", + ); + } +} diff --git a/src/main.ts b/src/main.ts index f1f959d..ff6b0bb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,16 +2,16 @@ import * as yaml from "jsr:@std/yaml"; import { tokenize } from "./tok.ts"; import { Parser } from "./parse.ts"; import { Resolver } from "./resolve.ts"; +import { Cx } from "./cx.ts"; async function main() { - const text = await Deno.readTextFile(Deno.args[0]); - const toks = tokenize(text); - console.log({ toks }); - return; + const cx = new Cx(); - const parser = new Parser(toks); - const file = parser.parseFile(); - if (parser.errorOccured) { + const fileId = await cx.readFile(Deno.args[0]); + tokenize(cx, fileId); + + const parseResult = Parser.parseFile(cx, fileId); + if (!parseResult.ok) { console.error("parsing failed"); Deno.exit(1); } @@ -19,13 +19,13 @@ async function main() { // console.log(yaml.stringify({ file }, { skipInvalid: true, indent: 2 })); // console.log(JSON.stringify({ file }, null, 2)); - const resolver = new Resolver(); - const syms = resolver.resolveFile(file); - if (resolver.errorOccured) { + const resolveResult = Resolver.resolveFile(cx, fileId); + if (!resolveResult.ok) { console.error("resolving failed"); Deno.exit(1); } + const syms = cx.file(fileId).syms!; console.log(syms); // console.log(cx); diff --git a/src/parse.ts b/src/parse.ts index c058406..3abe9ca 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -1,16 +1,32 @@ import { AstBuilder, Block, Expr, File, Param, Pat, Stmt, Ty } from "./ast.ts"; +import { Cx } from "./cx.ts"; +import { Loc, Reporter } from "./diagnostics.ts"; import { Tok } from "./tok.ts"; const t = new AstBuilder(); +export type ParseResult = { ok: true } | { ok: false } & object; + export class Parser { private i = 0; private eaten?: Tok; + private errorOccured = false; - constructor( + private constructor( + private reporter: Reporter, private toks: Tok[], ) {} + static parseFile(cx: Cx, fileId: number): ParseResult { + const parser = new Parser( + new Reporter(cx), + cx.file(fileId).toks!, + ); + const ast = parser.parseFile(); + cx.setFileAst(fileId, ast); + return { ok: !parser.errorOccured }; + } + parseFile(): File { const stmts: Stmt[] = []; while (!this.done) { @@ -20,7 +36,7 @@ export class Parser { } parseItem(): Stmt { - const line = this.line(); + const loc = this.loc(); if (this.test("fn")) { return this.parseFn(); } else if (this.test("let")) { @@ -28,19 +44,19 @@ export class Parser { } else { this.expect("item"); this.step(); - return t.stmt(line, "error", {}); + return t.stmt(loc, "error", {}); } } parseBlock(): Block { - const line = this.line(); + const loc = this.loc(); this.step(); const stmts: Stmt[] = []; let expr: Expr | undefined = undefined; while (!this.done && !this.test("}")) { - const line = this.line(); + const loc = this.loc(); if (this.test("fn")) { stmts.push(this.parseFn()); } else if (this.test("let")) { @@ -52,10 +68,10 @@ export class Parser { const rhs = this.parseExpr(); this.expect(";"); stmts.push( - t.stmt(line, "assign", { place: lhs, expr: rhs }), + t.stmt(loc, "assign", { place: lhs, expr: rhs }), ); } else if (this.eat(";")) { - stmts.push(t.stmt(line, "expr", { expr: lhs })); + stmts.push(t.stmt(loc, "expr", { expr: lhs })); } else if (this.test("}")) { expr = lhs; break; @@ -66,19 +82,19 @@ export class Parser { } this.expect("}"); - return t.block(line, stmts, expr); + return t.block(loc, stmts, expr); } parseFn(): Stmt { - const line = this.line(); + const loc = this.loc(); this.step(); if (!this.expect("ident")) { - return t.stmt(line, "error", {}); + return t.stmt(loc, "error", {}); } const ident = this.eaten!.value!; const params: Param[] = []; if (!this.expect("(")) { - return t.stmt(line, "error", {}); + return t.stmt(loc, "error", {}); } if (!this.done && !this.test(")")) { params.push(this.parseParam()); @@ -90,7 +106,7 @@ export class Parser { } } if (!this.expect(")")) { - return t.stmt(line, "error", {}); + return t.stmt(loc, "error", {}); } let retTy: Ty | undefined = undefined; if (this.eat("->")) { @@ -98,21 +114,21 @@ export class Parser { } if (!this.test("{")) { this.expect("{"); - return t.stmt(line, "error", {}); + return t.stmt(loc, "error", {}); } const body = this.parseBlock(); - return t.stmt(line, "fn", { ident, params, retTy, body }); + return t.stmt(loc, "fn", { ident, params, retTy, body }); } parseLet(): Stmt { - const line = this.line(); + const loc = this.loc(); this.step(); const param = this.parseParam(); if (!this.expect("=")) { - return t.stmt(line, "error", {}); + return t.stmt(loc, "error", {}); } const init = this.parseExpr(); - return t.stmt(line, "let", { param, init }); + return t.stmt(loc, "let", { param, init }); } parseExpr(): Expr { @@ -122,7 +138,7 @@ export class Parser { parsePostfix(): Expr { let expr = this.parseOp(); while (true) { - const line = this.line(); + const loc = this.loc(); if (this.eat("(")) { const args: Expr[] = []; if (!this.done && !this.test(")")) { @@ -135,9 +151,9 @@ export class Parser { } } if (!this.expect(")")) { - return t.expr(line, "error", {}); + return t.expr(loc, "error", {}); } - expr = t.expr(line, "call", { expr, args }); + expr = t.expr(loc, "call", { expr, args }); } else { break; } @@ -146,75 +162,75 @@ export class Parser { } parseOp(): Expr { - const line = this.line(); + const loc = this.loc(); if (this.eat("ident")) { const ident = this.eaten!.value!; - return t.expr(line, "ident", { ident }); + return t.expr(loc, "ident", { ident }); } else if (this.eat("int")) { const value = this.eaten!.value!; - return t.expr(line, "int", { value }); + return t.expr(loc, "int", { value }); } else if (this.eat("char")) { const value = this.eaten!.value!; - return t.expr(line, "int", { value }); + return t.expr(loc, "int", { value }); } else if (this.eat("str")) { const value = this.eaten!.value!; - return t.expr(line, "int", { value }); + return t.expr(loc, "int", { value }); } else { this.expect("expr"); this.step(); - return t.expr(line, "error", {}); + return t.expr(loc, "error", {}); } } parseParam(): Param { - const line = this.line(); + const loc = this.loc(); const pat = this.parsePat(); let ty: Ty | undefined = undefined; if (this.eat(":")) { ty = this.parseTy(); } - return t.param(line, pat, ty); + return t.param(loc, pat, ty); } parsePat(): Pat { - const line = this.line(); + const loc = this.loc(); if (this.eat("ident")) { const ident = this.eaten!.value!; - return t.pat(line, "ident", { ident }); + return t.pat(loc, "ident", { ident }); } else { this.expect("pat"); this.step(); - return t.pat(line, "error", {}); + return t.pat(loc, "error", {}); } } parseTy(): Ty { - const line = this.line(); + const loc = this.loc(); if (this.eat("int")) { - return t.ty(line, "int", {}); + return t.ty(loc, "int", {}); } else if (this.eat("bool")) { - return t.ty(line, "bool", {}); + return t.ty(loc, "bool", {}); } else if (this.eat("char")) { - return t.ty(line, "char", {}); + return t.ty(loc, "char", {}); } else if (this.eat("str")) { - return t.ty(line, "str", {}); + return t.ty(loc, "str", {}); } else if (this.eat("ident")) { const ident = this.eaten!.value!; - return t.ty(line, "ident", { ident }); + return t.ty(loc, "ident", { ident }); } else { this.expect("ty"); this.step(); - return t.ty(line, "error", {}); + return t.ty(loc, "error", {}); } } private expect(type: string): boolean { - const line = this.line(); + const loc = this.loc(); if (!this.eat(type)) { if (this.done) { - this.error(line, `expected '${type}', got 'eof'`); + this.error(loc, `expected '${type}', got 'eof'`); } else { - this.error(line, `expected '${type}', got '${this.tok.type}'`); + this.error(loc, `expected '${type}', got '${this.tok.type}'`); } return false; } @@ -238,8 +254,8 @@ export class Parser { this.i += 1; } - private line(): number { - return this.tok.line; + private loc(): Loc { + return this.tok.loc; } private get tok(): Tok { @@ -250,15 +266,8 @@ export class Parser { return this.i >= this.toks.length; } - public errorOccured = false; - private error(line: number, message: string) { + private error(loc: Loc, message: string) { this.errorOccured = true; - console.error( - `%cerror%c: ${message}\n %c--> line ${line}%c`, - "font-weight: bold; color: red", - "font-weight: bold; color: while", - "color: cyan", - "", - ); + this.reporter.error(loc, message); } } diff --git a/src/resolve.ts b/src/resolve.ts index a6d8345..4deeb25 100644 --- a/src/resolve.ts +++ b/src/resolve.ts @@ -1,13 +1,6 @@ -import { - Block, - Expr, - File, - Pat, - Stmt, - Ty, - Visitor, - VisitorBreak, -} from "./ast.ts"; +import * as ast from "./ast.ts"; +import { Cx } from "./cx.ts"; +import { Loc, Reporter } from "./diagnostics.ts"; class Res { constructor( @@ -40,23 +33,23 @@ export class Def { } } - line(): number { + loc(): Loc { const k = this.kind; switch (k.tag) { case "fn": - return k.stmt.line; + return k.stmt.loc; case "param": - return k.pat.line; + return k.pat.loc; case "let": - return k.pat.line; + return k.pat.loc; } } } export type DefKind = - | { tag: "fn"; stmt: Stmt } - | { tag: "param"; stmt: Stmt; pat: Pat } - | { tag: "let"; stmt: Stmt; pat: Pat }; + | { tag: "fn"; stmt: ast.Stmt } + | { tag: "param"; stmt: ast.Stmt; pat: ast.Pat } + | { tag: "let"; stmt: ast.Stmt; pat: ast.Pat }; type DefineResult = | { ok: true } @@ -110,42 +103,57 @@ export class Syms { private astNodeDefs: Map, ) {} - exprRes(expr: Expr): Res { + exprRes(expr: ast.Expr): Res { const res = this.astNodeDefs.get(expr.id); if (!res) throw new Error(); return res; } - patRes(pat: Pat): Res { + patRes(pat: ast.Pat): Res { const res = this.astNodeDefs.get(pat.id); if (!res) throw new Error(); return res; } - tyRes(ty: Ty): Res { + tyRes(ty: ast.Ty): Res { const res = this.astNodeDefs.get(ty.id); if (!res) throw new Error(); return res; } } -export class Resolver implements Visitor { +export type ResolveResult = { ok: true } | { ok: false }; + +export class Resolver implements ast.Visitor { private rib = new Rib({ tag: "root" }); private scopeStack: Rib[] = []; private astNodeDefs = new Map(); + private errorOccured = false; - resolveFile(file: File): Syms { + private constructor( + private reporter: Reporter, + ) {} + + static resolveFile(cx: Cx, fileId: number): ResolveResult { + const file = cx.file(fileId); + const resolver = new Resolver(new Reporter(cx)); + const syms = resolver.resolveFile(file.ast!); + cx.setFileSyms(fileId, syms); + return { ok: !resolver.errorOccured }; + } + + resolveFile(file: ast.File): Syms { file.visit({ - visitStmt: (stmt): void | VisitorBreak => { + visitStmt: (stmt): void | ast.VisitorBreak => { const k = stmt.kind; if (k.tag === "fn") { this.tryDefine( - stmt.line, + stmt.loc, k.ident, new Def({ tag: "fn", stmt }), ); } - return VisitorBreak; + return ast.VisitorBreak; }, }); @@ -153,36 +161,36 @@ export class Resolver implements Visitor { return new Syms(this.astNodeDefs); } - visitStmt(stmt: Stmt): void | VisitorBreak { + visitStmt(stmt: ast.Stmt): void | ast.VisitorBreak { const k = stmt.kind; if (k.tag === "fn") { this.saveScope(); this.pushRib("fn", {}); k.body.visit({ - visitStmt: (stmt): void | VisitorBreak => { + visitStmt: (stmt): void | ast.VisitorBreak => { const k = stmt.kind; if (k.tag === "fn") { this.tryDefine( - stmt.line, + stmt.loc, k.ident, new Def({ tag: "fn", stmt }), ); } - return VisitorBreak; + return ast.VisitorBreak; }, }); this.pushRib("param", {}); for (const param of k.params) { param.pat.visit({ - visitPat: (pat: Pat) => { + visitPat: (pat: ast.Pat) => { const k = pat.kind; if (k.tag !== "ident") { return; } this.tryDefine( - pat.line, + pat.loc, k.ident, new Def({ tag: "param", stmt, pat }), ); @@ -194,17 +202,17 @@ export class Resolver implements Visitor { k.body.visit(this); this.restoreScope(); - return VisitorBreak; + return ast.VisitorBreak; } else if (k.tag === "let") { this.pushRib("let", {}); k.param.pat.visit({ - visitPat: (pat: Pat) => { + visitPat: (pat: ast.Pat) => { const k = pat.kind; if (k.tag !== "ident") { return; } this.tryDefine( - pat.line, + pat.loc, k.ident, new Def({ tag: "let", stmt, pat }), ); @@ -213,30 +221,30 @@ export class Resolver implements Visitor { } } - visitBlock(block: Block): void | VisitorBreak { + visitBlock(block: ast.Block): void | ast.VisitorBreak { this.saveScope(); this.pushRib("block", {}); block.stmts.forEach((stmt) => stmt.visit(this)); block.expr?.visit(this); this.restoreScope(); - return VisitorBreak; + return ast.VisitorBreak; } - visitExpr(expr: Expr): void | VisitorBreak { + visitExpr(expr: ast.Expr): void | ast.VisitorBreak { const k = expr.kind; if (k.tag !== "ident") { return; } - const res = this.resolve(expr.line, k.ident); + const res = this.resolve(expr.loc, k.ident); this.astNodeDefs.set(expr.id, res); } - visitTy(ty: Ty): void | VisitorBreak { + visitTy(ty: ast.Ty): void | ast.VisitorBreak { const k = ty.kind; if (k.tag !== "ident") { return; } - const res = this.resolve(ty.line, k.ident); + const res = this.resolve(ty.loc, k.ident); this.astNodeDefs.set(ty.id, res); } @@ -255,11 +263,11 @@ export class Resolver implements Visitor { this.rib = this.scopeStack.pop()!; } - private tryDefine(line: number, ident: string, def: Def) { + private tryDefine(loc: Loc, ident: string, def: Def) { const result = this.define(ident, def); if (!result.ok) { - this.error(line, `redefinition of '${ident}'`); - this.info(result.originalDef.line(), `'${ident}' defined here`); + this.error(loc, `redefinition of '${ident}'`); + this.info(result.originalDef.loc(), `'${ident}' defined here`); } } @@ -267,33 +275,20 @@ export class Resolver implements Visitor { return this.rib.define(ident, def); } - private resolve(line: number, ident: string): Res { + private resolve(loc: Loc, ident: string): Res { const res = this.rib.resolve(ident); if (res.is("unresolved")) { - this.error(line, `unresolved ident '${ident}'`); + this.error(loc, `unresolved ident '${ident}'`); } return res; } - public errorOccured = false; - private error(line: number, message: string) { + private error(loc: Loc, message: string) { this.errorOccured = true; - console.error( - `%cerror%c: ${message}\n %c--> line ${line}%c`, - "font-weight: bold; color: red", - "font-weight: bold; color: while", - "color: cyan", - "", - ); + this.reporter.error(loc, message); } - private info(line: number, message: string) { - console.error( - `%cinfo%c: ${message}\n %c--> line ${line}%c`, - "font-weight: bold; color: blue", - "font-weight: bold; color: while", - "color: cyan", - "", - ); + private info(loc: Loc, message: string) { + this.reporter.info(loc, message); } } diff --git a/src/tok.ts b/src/tok.ts index a4ca70a..e74f52c 100644 --- a/src/tok.ts +++ b/src/tok.ts @@ -1,40 +1,49 @@ +import { Cx } from "./cx.ts"; +import { Loc, Reporter } from "./diagnostics.ts"; + export type Tok = { type: string; - idx: number; - line: number; - col: number; - length: number; - value?: string; + loc: Loc; + value: string; }; -const keywords = ["true", "false", "bool", "int", "char", "str", "fn", "let"]; -export function tokenize(text: string): Tok[] { - const rules: Record = { - "whitespace": { match: /^[ \t\r]+/, ignore: true }, - "newline": { match: /^\n/s, ignore: true }, - "linecomment": { match: /^\/\/[^\n]*/, ignore: true }, - "blockcomment": { match: /^\/\*.*?\*\//s, ignore: true }, - _keywords: { - match: new RegExp( - `^(${keywords.map((s) => `(?:${s})`).join("|")})`, - ), - }, - _identity: { - match: new RegExp( - `^(?:(?:\-\>)|[${RegExp.escape("()[]{}+-*/,.;:!=<>&|?")}])`, - ), - }, - "ident": { match: /^[a-zA-Z_][a-zA-Z0-9_]*/ }, - "int": { match: /^[0-9_]+/ }, - "char": { match: /^'(?:(?:\\.)|[^'\n])'/ }, - "str": { match: /^"(?:(?:\\.)|[^"])*"/s }, - }; +const keywords = new Set([ + "true", + "false", + "bool", + "int", + "char", + "str", + "fn", + "let", +]); + +const rules: Record = { + "whitespace": { match: /^[ \t\r]+/, ignore: true }, + "newline": { match: /^\n/s, ignore: true }, + "linecomment": { match: /^\/\/[^\n]*/, ignore: true }, + "blockcomment": { match: /^\/\*.*?\*\//s, ignore: true }, + _identity: { + match: new RegExp( + `^(?:(?:\-\>)|[${RegExp.escape("()[]{}+-*/,.;:!=<>&|?")}])`, + ), + }, + "ident": { match: /^[a-zA-Z_][a-zA-Z0-9_]*/ }, + "int": { match: /^[0-9_]+/ }, + "char": { match: /^'(?:(?:\\.)|[^'\n])'/ }, + "str": { match: /^"(?:(?:\\.)|[^"])*"/s }, +}; + +export function tokenize(cx: Cx, fileId: number) { + const { text } = cx.file(fileId); + const rep = new Reporter(cx); const toks: Tok[] = []; let idx = 0; let line = 1; let col = 1; while (idx < text.length) { + const loc: Loc = { fileId, idx, line, col }; let found = false; for (const [id, rule] of Object.entries(rules)) { const match = text.slice(idx).match(rule.match); @@ -53,10 +62,15 @@ export function tokenize(text: string): Tok[] { if (rule.ignore) continue; - const length = match[0].length; - const tok: Tok = { type: id, idx, line, col, length }; - if (id === "_keywords" || id === "_identity") { + const tok: Tok = { + type: id, + loc, + value: match[0], + }; + if (id === "_identity") { tok.type = match[0]; + } else if (id === "ident") { + tok.type = keywords.has(match[0]) ? match[0] : id; } else { tok.value = match[0]; } @@ -64,20 +78,10 @@ export function tokenize(text: string): Tok[] { break; } if (!found) { - printError(line, `invalid character '${text[idx]}'`); + rep.error(loc, `invalid character '${text[idx]}'`); idx += 1; } } - return toks; -} - -function printError(line: number, message: string) { - console.error( - `%cerror%c: ${message}\n %c--> line ${line}%c`, - "font-weight: bold; color: red", - "font-weight: bold; color: while", - "color: cyan", - "", - ); + cx.setFileToks(fileId, toks); }