add parser + ast

ast.h

@@ -0,0 +1,64 @@
+#ifndef AST_H
+#define AST_H
+
+#include "collection.h"
+
+typedef struct {
+    int line;
+    int col;
+} Loc;
+
+typedef enum : unsigned int {
+    AstSpec_Typedef = 1 << 0,
+    // AstSpec_Constexpr = 1 << 1,
+    // AstSpec_Auto = 1 << 2,
+    // AstSpec_Register = 1 << 3,
+    AstSpec_Static = 1 << 4,
+    // AstSpec_Extern = 1 << 5,
+    // AstSpec_ThreadLocal = 1 << 6,
+    AstSpec_Const = 1 << 7,
+    // AstSpec_Volatile = 1 << 8,
+    // AstSpec_Restrict = 1 << 9,
+    // AstSpec_Atomic = 1 << 10,
+    AstSpec_Inline = 1 << 11,
+} AstSpec;
+
+typedef enum {
+    AstDeclar_Ident,
+    AstDeclar_Pointer,
+    AstDeclar_Array,
+} AstDeclarTy;
+
+typedef struct AstDeclar AstDeclar;
+
+struct AstDeclar {
+    AstDeclarTy ty;
+    Loc loc;
+    union {
+        char* ident;
+        struct {
+            AstDeclar* declar;
+            AstSpec spec;
+        } pointer;
+    };
+};
+
+typedef struct {
+    AstSpec spec;
+    unsigned long type_id;
+    ARRAY_FIELD(AstDeclar, declars);
+    Loc loc;
+} AstDecl;
+
+typedef enum {
+    AstStmt_Decl,
+} AstStmtTy;
+
+typedef struct {
+    AstStmtTy ty;
+    union {
+        AstDecl decl;
+    };
+} AstStmt;
+
+#endif

main.c

@@ -20,15 +20,20 @@ int main(int argc, char* argv[])
 
     printf("--- text ---\n%s\n", text);
 
-    Preproc* preproc;
-    preproc_init(&preproc, strdup(filename), text);
+    Arena tok_arena;
+    arena_init(&tok_arena, 4096);
+
+    Preproc* preproc = preproc_new(strdup(filename), text, &tok_arena);
+
+    TokStream tokstream;
+    tokstream_init(&tokstream, preproc, &tok_arena);
 
     printf("--- tokens ---\n");
 
     int line = 1;
     int col = 1;
     Tok tok;
-    while ((tok = preproc_next(preproc)).ty != Tok_Eof) {
+    while ((tok = tokstream_next(&tokstream)).ty != Tok_Eof) {
         // printf("%s\t%s\n", tokty_str(tok.ty), tok.text);
         if (line > tok.line) {
             line = 0;
@@ -48,5 +53,6 @@ int main(int argc, char* argv[])
     }
     fputs("\n", stdout);
 
-    preproc_deinit(preproc);
+    preproc_free(preproc);
+    arena_deinit(&tok_arena);
 }

parse.c

@@ -0,0 +1,183 @@
+#include "parse.h"
+#include "collection.h"
+#include <assert.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+typedef struct {
+    char ch;
+    bool is_term;
+    unsigned short val;
+} KwNodeOpt;
+
+typedef struct {
+    ARRAY_FIELD(KwNodeOpt, opts);
+    size_t idx;
+} KwNode;
+
+typedef struct {
+    TokStream* tokstream;
+    Arena arena;
+    ARRAY_FIELD(KwNode, nodes);
+    KwNode* root;
+} KwMatcher;
+
+static KwNodeOpt* kwmatcher_create_path(
+    KwMatcher* m, KwNode* node, const char* kw)
+{
+    assert(kw[0]);
+    KwNodeOpt* opt = nullptr;
+    for (size_t i = 0; i < node->opts_size; ++i) {
+        if (node->opts[i].ch == kw[0]) {
+            opt = &node->opts[i];
+        }
+    }
+    if (!opt) {
+        size_t new_idx = m->nodes_size;
+
+        KwNode new_node = {
+            ARRAY_FIELD_INIT(opts),
+            .idx = new_idx,
+        };
+        arena_array_push(&m->arena,
+            (void**)&(m)->nodes,
+            &(m)->nodes_cap,
+            &(m)->nodes_size,
+            (&new_node),
+            sizeof(*(&new_node)));
+        KwNode* node = &m->nodes[new_idx];
+
+        KwNodeOpt new_opt = {
+            .ch = kw[0],
+            .is_term = false,
+            .val = new_idx,
+        };
+        arena_array_push(&m->arena,
+            (void**)&node->opts,
+            &node->opts_cap,
+            &node->opts_size,
+            (&new_opt),
+            sizeof(*(&new_opt)));
+
+        opt = &node->opts[node->opts_size - 1];
+    }
+    if (!kw[1])
+        return opt;
+
+    KwNode* next_node = &m->nodes[opt->val];
+    return kwmatcher_create_path(m, next_node, &kw[1]);
+}
+
+static void kwmatcher_add(KwMatcher* m, const char* kw, TokTy tokty)
+{
+    KwNodeOpt* opt = kwmatcher_create_path(m, m->root, kw);
+    opt->is_term = true;
+    opt->val = tokty;
+}
+
+static KwNodeOpt* kwmatcher_find_node(
+    KwMatcher* m, KwNode* node, const char* kw)
+{
+    assert(kw[0]);
+    KwNodeOpt* opt = nullptr;
+    for (size_t i = 0; i < node->opts_size; ++i) {
+        if (node->opts[i].ch == kw[0]) {
+            opt = &node->opts[i];
+        }
+    }
+    if (!opt)
+        return nullptr;
+    if (!kw[1])
+        return opt;
+    return kwmatcher_find_node(m, &m->nodes[opt->val], &kw[1]);
+}
+
+static TokTy kwmatcher_match(KwMatcher* m, const char* kw)
+{
+    KwNodeOpt* opt = kwmatcher_find_node(m, m->root, kw);
+    if (!opt->is_term)
+        return Tok_Ident;
+    return opt->val;
+}
+
+static void kwmatcher_init(KwMatcher* m, TokStream* tokstream)
+{
+    *m = (KwMatcher) {
+        .tokstream = tokstream,
+        .arena = { },
+        ARRAY_FIELD_INIT(nodes),
+        .root = nullptr,
+    };
+
+    arena_init(&m->arena, 512);
+
+    arena_array_push(&m->arena,
+        (void**)&(m)->nodes,
+        &(m)->nodes_cap,
+        &(m)->nodes_size,
+        (&(KwNode) { .idx = 0 }),
+        sizeof(*(&(KwNode) { .idx = 0 })));
+
+    m->root = &m->nodes[0];
+
+    struct {
+        TokTy ty;
+        const char* kw;
+    } kws[] = {
+        { Tok_KwInt, "int" },
+        { Tok_KwChar, "char" },
+        { Tok_KwConst, "const" },
+    };
+
+    for (size_t i = 0; i < sizeof(kws) / sizeof(kws[0]); ++i) {
+        kwmatcher_add(m, kws[i].kw, kws[i].ty);
+    }
+}
+
+static void kwmatcher_deinit(KwMatcher* m)
+{
+    arena_deinit(&m->arena);
+}
+
+static Tok kwmatcher_next(KwMatcher* m)
+{
+    Tok tok = tokstream_next(m->tokstream);
+    if (tok.ty != Tok_Ident)
+        return tok;
+    TokTy ty = kwmatcher_match(m, tok.text);
+    if (ty != Tok_Ident)
+        printf("info: matched '%s' on line %d\n", tok.text, tok.line);
+    tok.ty = ty;
+    return tok;
+}
+
+struct Parser {
+    KwMatcher kwmatcher;
+    Arena* arena;
+
+    Tok curr;
+};
+
+Parser* parser_new(TokStream* tokstream, Arena* arena)
+{
+    Parser* p = malloc(sizeof(Parser));
+
+    *p = (Parser) {
+        .kwmatcher = { },
+        .arena = arena,
+
+        .curr = { },
+    };
+
+    kwmatcher_init(&p->kwmatcher, tokstream);
+    p->curr = kwmatcher_next(&p->kwmatcher);
+
+    return p;
+}
+
+void parser_free(Parser* p)
+{
+    kwmatcher_deinit(&p->kwmatcher);
+    free(p);
+}

parse.h

@@ -57,6 +57,10 @@ typedef enum {
     Tok_Asterisk = '*',
     Tok_Hashtag = '#',
 
+    Tok_KwInt,
+    Tok_KwChar,
+    Tok_KwConst,
+
     // used internally in preprocessor
     Tok_EndOfMacroInvocation,
 } TokTy;
@@ -85,9 +89,24 @@ Tok lexer_next(Lexer* l);
 
 typedef struct Preproc Preproc;
 
-void preproc_init(Preproc** p, char* filename, char* text);
-void preproc_deinit(Preproc* p);
-Tok preproc_next_with_newline(Preproc* p);
+Preproc* preproc_new(char* filename, char* text, Arena* arena);
+void preproc_free(Preproc* p);
 Tok preproc_next(Preproc* p);
 
+// phase 5, 6 - preprocessor cleanup, concat strings
+
+typedef struct {
+    Preproc* preproc;
+    Arena* arena;
+    Tok curr;
+} TokStream;
+
+void tokstream_init(TokStream* s, Preproc* preproc, Arena* arena);
+Tok tokstream_next(TokStream* s);
+
+typedef struct Parser Parser;
+
+Parser* parser_new(TokStream* tokstream, Arena* arena);
+void parser_free(Parser* p);
+
 #endif

preproc.c

@@ -107,7 +107,7 @@ typedef struct {
 } Def;
 
 struct Preproc {
-    Arena tok_arena;
+    Arena* arena;
     ARRAY_FIELD(File, files);
 
     ARRAY_FIELD(Tok, tok_stack);
@@ -121,12 +121,12 @@ struct Preproc {
     bool next_can_be_directive;
 };
 
-void preproc_init(Preproc** p, char* filename, char* text)
+Preproc* preproc_new(char* filename, char* text, Arena* arena)
 {
-    *p = malloc(sizeof(Preproc));
+    Preproc* p = malloc(sizeof(Preproc));
 
-    **p = (Preproc) {
-        .tok_arena = { },
+    *p = (Preproc) {
+        .arena = arena,
         ARRAY_FIELD_INIT(files),
 
         ARRAY_FIELD_INIT(tok_stack),
@@ -140,9 +140,7 @@ void preproc_init(Preproc** p, char* filename, char* text)
         .next_can_be_directive = true,
     };
 
-    arena_init(&(*p)->tok_arena, 512);
-
-    ARRAY_FIELD_PUSH(*p,
+    ARRAY_FIELD_PUSH(p,
         files,
         (&(File) {
             .filename = filename,
@@ -152,19 +150,21 @@ void preproc_init(Preproc** p, char* filename, char* text)
             .lexer = { },
         }));
 
-    File* file = &(*p)->files[0];
+    File* file = &p->files[0];
     source_iter_init(&file->source_iter, text);
     lsl_iter_init(&file->lsl_iter, &file->source_iter);
-    lexer_init(&file->lexer, &file->lsl_iter, &(*p)->tok_arena);
-    (*p)->curr = lexer_next(&file->lexer);
+    lexer_init(&file->lexer, &file->lsl_iter, p->arena);
+    p->curr = lexer_next(&file->lexer);
 
     // null-definition at idx 0
-    ARRAY_FIELD_PUSH(*p, defs, &(Def) { });
+    ARRAY_FIELD_PUSH(p, defs, &(Def) { });
 
-    syms_init(&(*p)->syms);
+    syms_init(&p->syms);
+
+    return p;
 }
 
-void preproc_deinit(Preproc* p)
+void preproc_free(Preproc* p)
 {
     for (size_t i = 0; i < p->files_size; ++i) {
         free(p->files[i].filename);
@@ -181,7 +181,6 @@ void preproc_deinit(Preproc* p)
         free(p->defs);
     }
     syms_deinit(&p->syms);
-    arena_deinit(&p->tok_arena);
     free(p);
 }
 
@@ -237,7 +236,7 @@ static void collect_arg(Preproc* p, size_t def_idx, size_t arg_idx)
             depth -= 1;
         }
 
-        arena_array_push(&p->tok_arena,
+        arena_array_push(p->arena,
             (void**)&toks,
             &toks_cap,
             &toks_size,
@@ -390,7 +389,7 @@ static Tok do_define(Preproc* p)
     ARRAY_VAR(Tok, toks);
 
     while (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
-        arena_array_push(&p->tok_arena,
+        arena_array_push(p->arena,
             (void**)&toks,
             &toks_cap,
             &toks_size,
@@ -435,7 +434,7 @@ static Tok do_undef(Preproc* p)
     return p->curr;
 }
 
-Tok preproc_next_with_newline(Preproc* p);
+Tok preproc_next(Preproc* p);
 
 static Tok do_include(Preproc* p)
 {
@@ -495,12 +494,12 @@ static Tok do_include(Preproc* p)
     File* file = &p->files[p->files_size - 1];
     source_iter_init(&file->source_iter, text);
     lsl_iter_init(&file->lsl_iter, &file->source_iter);
-    lexer_init(&file->lexer, &file->lsl_iter, &p->tok_arena);
+    lexer_init(&file->lexer, &file->lsl_iter, p->arena);
     p->curr = lexer_next(&file->lexer);
 
     p->next_can_be_directive = true;
 
-    return preproc_next_with_newline(p);
+    return preproc_next(p);
 }
 
 static Tok interpret_directive(Preproc* p);
@@ -622,7 +621,7 @@ static Tok interpret_directive(Preproc* p)
     exit(1);
 }
 
-Tok preproc_next_with_newline(Preproc* p)
+Tok preproc_next(Preproc* p)
 {
     if (p->next_can_be_directive && p->curr.ty == '#') {
         step(p);
@@ -631,10 +630,39 @@ Tok preproc_next_with_newline(Preproc* p)
     return expand_tok(p);
 }
 
-Tok preproc_next(Preproc* p)
+void tokstream_init(TokStream* s, Preproc* preproc, Arena* arena)
+{
+    *s = (TokStream) {
+        .preproc = preproc,
+        .arena = arena,
+        .curr = { },
+    };
+    s->curr = preproc_next(s->preproc);
+}
+
+static Tok tokstream_next_filtered(TokStream* s)
 {
     Tok tok;
-    while ((tok = preproc_next_with_newline(p)).ty == Tok_Eol
+    while ((tok = preproc_next(s->preproc)).ty == Tok_Eol
         || tok.ty == Tok_Whitespace) { }
     return tok;
 }
+
+Tok tokstream_next(TokStream* s)
+{
+    Tok curr = s->curr;
+    Tok next = tokstream_next_filtered(s);
+
+    if (curr.ty == Tok_Str && next.ty == Tok_Str) {
+        char* text
+            = arena_alloc(s->arena, strlen(curr.text) + strlen(next.text) + 1);
+        text[0] = '\0';
+        strncat(text, curr.text, strlen(curr.text) - 1);
+        strcat(text, next.text + 1);
+        s->curr = (Tok) { .ty = Tok_Str, text, curr.line, curr.col };
+        return tokstream_next(s);
+    }
+
+    s->curr = next;
+    return curr;
+}