ast.h

@@ -1,64 +0,0 @@
-#ifndef AST_H
-#define AST_H
-
-#include "collection.h"
-
-typedef struct {
-    int line;
-    int col;
-} Loc;
-
-typedef enum : unsigned int {
-    AstSpec_Typedef = 1 << 0,
-    // AstSpec_Constexpr = 1 << 1,
-    // AstSpec_Auto = 1 << 2,
-    // AstSpec_Register = 1 << 3,
-    AstSpec_Static = 1 << 4,
-    // AstSpec_Extern = 1 << 5,
-    // AstSpec_ThreadLocal = 1 << 6,
-    AstSpec_Const = 1 << 7,
-    // AstSpec_Volatile = 1 << 8,
-    // AstSpec_Restrict = 1 << 9,
-    // AstSpec_Atomic = 1 << 10,
-    AstSpec_Inline = 1 << 11,
-} AstSpec;
-
-typedef enum {
-    AstDeclar_Ident,
-    AstDeclar_Pointer,
-    AstDeclar_Array,
-} AstDeclarTy;
-
-typedef struct AstDeclar AstDeclar;
-
-struct AstDeclar {
-    AstDeclarTy ty;
-    Loc loc;
-    union {
-        char* ident;
-        struct {
-            AstDeclar* declar;
-            AstSpec spec;
-        } pointer;
-    };
-};
-
-typedef struct {
-    AstSpec spec;
-    unsigned long type_id;
-    ARRAY_FIELD(AstDeclar, declars);
-    Loc loc;
-} AstDecl;
-
-typedef enum {
-    AstStmt_Decl,
-} AstStmtTy;
-
-typedef struct {
-    AstStmtTy ty;
-    union {
-        AstDecl decl;
-    };
-} AstStmt;
-
-#endif

main.c

@@ -20,20 +20,15 @@ int main(int argc, char* argv[])
 
     printf("--- text ---\n%s\n", text);
 
-    Arena tok_arena;
+    Preproc* preproc;
-    arena_init(&tok_arena, 4096);
+    preproc_init(&preproc, strdup(filename), text);
-
-    Preproc* preproc = preproc_new(strdup(filename), text, &tok_arena);
-
-    TokStream tokstream;
-    tokstream_init(&tokstream, preproc, &tok_arena);
 
     printf("--- tokens ---\n");
 
     int line = 1;
     int col = 1;
     Tok tok;
-    while ((tok = tokstream_next(&tokstream)).ty != Tok_Eof) {
+    while ((tok = preproc_next(preproc)).ty != Tok_Eof) {
         // printf("%s\t%s\n", tokty_str(tok.ty), tok.text);
         if (line > tok.line) {
             line = 0;
@@ -53,6 +48,5 @@ int main(int argc, char* argv[])
     }
     fputs("\n", stdout);
 
-    preproc_free(preproc);
+    preproc_deinit(preproc);
-    arena_deinit(&tok_arena);
 }

parse.c

@@ -1,183 +0,0 @@
-#include "parse.h"
-#include "collection.h"
-#include <assert.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-typedef struct {
-    char ch;
-    bool is_term;
-    unsigned short val;
-} KwNodeOpt;
-
-typedef struct {
-    ARRAY_FIELD(KwNodeOpt, opts);
-    size_t idx;
-} KwNode;
-
-typedef struct {
-    TokStream* tokstream;
-    Arena arena;
-    ARRAY_FIELD(KwNode, nodes);
-    KwNode* root;
-} KwMatcher;
-
-static KwNodeOpt* kwmatcher_create_path(
-    KwMatcher* m, KwNode* node, const char* kw)
-{
-    assert(kw[0]);
-    KwNodeOpt* opt = nullptr;
-    for (size_t i = 0; i < node->opts_size; ++i) {
-        if (node->opts[i].ch == kw[0]) {
-            opt = &node->opts[i];
-        }
-    }
-    if (!opt) {
-        size_t new_idx = m->nodes_size;
-
-        KwNode new_node = {
-            ARRAY_FIELD_INIT(opts),
-            .idx = new_idx,
-        };
-        arena_array_push(&m->arena,
-            (void**)&(m)->nodes,
-            &(m)->nodes_cap,
-            &(m)->nodes_size,
-            (&new_node),
-            sizeof(*(&new_node)));
-        KwNode* node = &m->nodes[new_idx];
-
-        KwNodeOpt new_opt = {
-            .ch = kw[0],
-            .is_term = false,
-            .val = new_idx,
-        };
-        arena_array_push(&m->arena,
-            (void**)&node->opts,
-            &node->opts_cap,
-            &node->opts_size,
-            (&new_opt),
-            sizeof(*(&new_opt)));
-
-        opt = &node->opts[node->opts_size - 1];
-    }
-    if (!kw[1])
-        return opt;
-
-    KwNode* next_node = &m->nodes[opt->val];
-    return kwmatcher_create_path(m, next_node, &kw[1]);
-}
-
-static void kwmatcher_add(KwMatcher* m, const char* kw, TokTy tokty)
-{
-    KwNodeOpt* opt = kwmatcher_create_path(m, m->root, kw);
-    opt->is_term = true;
-    opt->val = tokty;
-}
-
-static KwNodeOpt* kwmatcher_find_node(
-    KwMatcher* m, KwNode* node, const char* kw)
-{
-    assert(kw[0]);
-    KwNodeOpt* opt = nullptr;
-    for (size_t i = 0; i < node->opts_size; ++i) {
-        if (node->opts[i].ch == kw[0]) {
-            opt = &node->opts[i];
-        }
-    }
-    if (!opt)
-        return nullptr;
-    if (!kw[1])
-        return opt;
-    return kwmatcher_find_node(m, &m->nodes[opt->val], &kw[1]);
-}
-
-static TokTy kwmatcher_match(KwMatcher* m, const char* kw)
-{
-    KwNodeOpt* opt = kwmatcher_find_node(m, m->root, kw);
-    if (!opt->is_term)
-        return Tok_Ident;
-    return opt->val;
-}
-
-static void kwmatcher_init(KwMatcher* m, TokStream* tokstream)
-{
-    *m = (KwMatcher) {
-        .tokstream = tokstream,
-        .arena = { },
-        ARRAY_FIELD_INIT(nodes),
-        .root = nullptr,
-    };
-
-    arena_init(&m->arena, 512);
-
-    arena_array_push(&m->arena,
-        (void**)&(m)->nodes,
-        &(m)->nodes_cap,
-        &(m)->nodes_size,
-        (&(KwNode) { .idx = 0 }),
-        sizeof(*(&(KwNode) { .idx = 0 })));
-
-    m->root = &m->nodes[0];
-
-    struct {
-        TokTy ty;
-        const char* kw;
-    } kws[] = {
-        { Tok_KwInt, "int" },
-        { Tok_KwChar, "char" },
-        { Tok_KwConst, "const" },
-    };
-
-    for (size_t i = 0; i < sizeof(kws) / sizeof(kws[0]); ++i) {
-        kwmatcher_add(m, kws[i].kw, kws[i].ty);
-    }
-}
-
-static void kwmatcher_deinit(KwMatcher* m)
-{
-    arena_deinit(&m->arena);
-}
-
-static Tok kwmatcher_next(KwMatcher* m)
-{
-    Tok tok = tokstream_next(m->tokstream);
-    if (tok.ty != Tok_Ident)
-        return tok;
-    TokTy ty = kwmatcher_match(m, tok.text);
-    if (ty != Tok_Ident)
-        printf("info: matched '%s' on line %d\n", tok.text, tok.line);
-    tok.ty = ty;
-    return tok;
-}
-
-struct Parser {
-    KwMatcher kwmatcher;
-    Arena* arena;
-
-    Tok curr;
-};
-
-Parser* parser_new(TokStream* tokstream, Arena* arena)
-{
-    Parser* p = malloc(sizeof(Parser));
-
-    *p = (Parser) {
-        .kwmatcher = { },
-        .arena = arena,
-
-        .curr = { },
-    };
-
-    kwmatcher_init(&p->kwmatcher, tokstream);
-    p->curr = kwmatcher_next(&p->kwmatcher);
-
-    return p;
-}
-
-void parser_free(Parser* p)
-{
-    kwmatcher_deinit(&p->kwmatcher);
-    free(p);
-}

parse.h

@@ -57,10 +57,6 @@ typedef enum {
     Tok_Asterisk = '*',
     Tok_Hashtag = '#',
 
-    Tok_KwInt,
-    Tok_KwChar,
-    Tok_KwConst,
-
     // used internally in preprocessor
     Tok_EndOfMacroInvocation,
 } TokTy;
@@ -89,24 +85,9 @@ Tok lexer_next(Lexer* l);
 
 typedef struct Preproc Preproc;
 
-Preproc* preproc_new(char* filename, char* text, Arena* arena);
+void preproc_init(Preproc** p, char* filename, char* text);
-void preproc_free(Preproc* p);
+void preproc_deinit(Preproc* p);
+Tok preproc_next_with_newline(Preproc* p);
 Tok preproc_next(Preproc* p);
 
-// phase 5, 6 - preprocessor cleanup, concat strings
-
-typedef struct {
-    Preproc* preproc;
-    Arena* arena;
-    Tok curr;
-} TokStream;
-
-void tokstream_init(TokStream* s, Preproc* preproc, Arena* arena);
-Tok tokstream_next(TokStream* s);
-
-typedef struct Parser Parser;
-
-Parser* parser_new(TokStream* tokstream, Arena* arena);
-void parser_free(Parser* p);
-
 #endif

preproc.c

@@ -107,7 +107,7 @@ typedef struct {
 } Def;
 
 struct Preproc {
-    Arena* arena;
+    Arena tok_arena;
     ARRAY_FIELD(File, files);
 
     ARRAY_FIELD(Tok, tok_stack);
@@ -121,12 +121,12 @@ struct Preproc {
     bool next_can_be_directive;
 };
 
-Preproc* preproc_new(char* filename, char* text, Arena* arena)
+void preproc_init(Preproc** p, char* filename, char* text)
 {
-    Preproc* p = malloc(sizeof(Preproc));
+    *p = malloc(sizeof(Preproc));
 
-    *p = (Preproc) {
+    **p = (Preproc) {
-        .arena = arena,
+        .tok_arena = { },
         ARRAY_FIELD_INIT(files),
 
         ARRAY_FIELD_INIT(tok_stack),
@@ -140,7 +140,9 @@ Preproc* preproc_new(char* filename, char* text, Arena* arena)
         .next_can_be_directive = true,
     };
 
-    ARRAY_FIELD_PUSH(p,
+    arena_init(&(*p)->tok_arena, 512);
+
+    ARRAY_FIELD_PUSH(*p,
         files,
         (&(File) {
             .filename = filename,
@@ -150,21 +152,19 @@ Preproc* preproc_new(char* filename, char* text, Arena* arena)
             .lexer = { },
         }));
 
-    File* file = &p->files[0];
+    File* file = &(*p)->files[0];
     source_iter_init(&file->source_iter, text);
     lsl_iter_init(&file->lsl_iter, &file->source_iter);
-    lexer_init(&file->lexer, &file->lsl_iter, p->arena);
+    lexer_init(&file->lexer, &file->lsl_iter, &(*p)->tok_arena);
-    p->curr = lexer_next(&file->lexer);
+    (*p)->curr = lexer_next(&file->lexer);
 
     // null-definition at idx 0
-    ARRAY_FIELD_PUSH(p, defs, &(Def) { });
+    ARRAY_FIELD_PUSH(*p, defs, &(Def) { });
 
-    syms_init(&p->syms);
+    syms_init(&(*p)->syms);
-
-    return p;
 }
 
-void preproc_free(Preproc* p)
+void preproc_deinit(Preproc* p)
 {
     for (size_t i = 0; i < p->files_size; ++i) {
         free(p->files[i].filename);
@@ -181,6 +181,7 @@ void preproc_free(Preproc* p)
         free(p->defs);
     }
     syms_deinit(&p->syms);
+    arena_deinit(&p->tok_arena);
     free(p);
 }
 
@@ -236,7 +237,7 @@ static void collect_arg(Preproc* p, size_t def_idx, size_t arg_idx)
             depth -= 1;
         }
 
-        arena_array_push(p->arena,
+        arena_array_push(&p->tok_arena,
             (void**)&toks,
             &toks_cap,
             &toks_size,
@@ -389,7 +390,7 @@ static Tok do_define(Preproc* p)
     ARRAY_VAR(Tok, toks);
 
     while (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
-        arena_array_push(p->arena,
+        arena_array_push(&p->tok_arena,
             (void**)&toks,
             &toks_cap,
             &toks_size,
@@ -434,7 +435,7 @@ static Tok do_undef(Preproc* p)
     return p->curr;
 }
 
-Tok preproc_next(Preproc* p);
+Tok preproc_next_with_newline(Preproc* p);
 
 static Tok do_include(Preproc* p)
 {
@@ -494,12 +495,12 @@ static Tok do_include(Preproc* p)
     File* file = &p->files[p->files_size - 1];
     source_iter_init(&file->source_iter, text);
     lsl_iter_init(&file->lsl_iter, &file->source_iter);
-    lexer_init(&file->lexer, &file->lsl_iter, p->arena);
+    lexer_init(&file->lexer, &file->lsl_iter, &p->tok_arena);
     p->curr = lexer_next(&file->lexer);
 
     p->next_can_be_directive = true;
 
-    return preproc_next(p);
+    return preproc_next_with_newline(p);
 }
 
 static Tok interpret_directive(Preproc* p);
@@ -621,7 +622,7 @@ static Tok interpret_directive(Preproc* p)
     exit(1);
 }
 
-Tok preproc_next(Preproc* p)
+Tok preproc_next_with_newline(Preproc* p)
 {
     if (p->next_can_be_directive && p->curr.ty == '#') {
         step(p);
@@ -630,39 +631,10 @@ Tok preproc_next(Preproc* p)
     return expand_tok(p);
 }
 
-void tokstream_init(TokStream* s, Preproc* preproc, Arena* arena)
+Tok preproc_next(Preproc* p)
-{
-    *s = (TokStream) {
-        .preproc = preproc,
-        .arena = arena,
-        .curr = { },
-    };
-    s->curr = preproc_next(s->preproc);
-}
-
-static Tok tokstream_next_filtered(TokStream* s)
 {
     Tok tok;
-    while ((tok = preproc_next(s->preproc)).ty == Tok_Eol
+    while ((tok = preproc_next_with_newline(p)).ty == Tok_Eol
         || tok.ty == Tok_Whitespace) { }
     return tok;
 }
-
-Tok tokstream_next(TokStream* s)
-{
-    Tok curr = s->curr;
-    Tok next = tokstream_next_filtered(s);
-
-    if (curr.ty == Tok_Str && next.ty == Tok_Str) {
-        char* text
-            = arena_alloc(s->arena, strlen(curr.text) + strlen(next.text) + 1);
-        text[0] = '\0';
-        strncat(text, curr.text, strlen(curr.text) - 1);
-        strcat(text, next.text + 1);
-        s->curr = (Tok) { .ty = Tok_Str, text, curr.line, curr.col };
-        return tokstream_next(s);
-    }
-
-    s->curr = next;
-    return curr;
-}