macros and includes

collection.c

@@ -53,21 +53,31 @@ void array_push(
         nullptr);
 }
 
+struct KwPair {
+    char* key;
+    unsigned char val[];
+};
+
+static KwPair* kwpair_at(Hashmap* m, size_t idx)
+{
+    return (KwPair*)((unsigned char*)m->entries + m->pair_size * idx);
+}
+
 static void push_kwpair(Hashmap* m, KwPair* pair)
 {
     array_push((void**)&m->entries,
-        &m->entries_capacity,
+        &m->entries_cap,
         &m->entries_size,
         pair,
-        sizeof(*pair));
+        m->pair_size);
 }
 
-void hashmap_init(Hashmap* m)
+void hashmap_init(Hashmap* m, size_t elem_size)
 {
     *m = (Hashmap) {
-        .entries = nullptr,
+        ARRAY_FIELD_INIT(entries),
-        .entries_capacity = 8,
+        .elem_size = elem_size,
-        .entries_size = 0,
+        .pair_size = sizeof(KwPair) + ((elem_size + 7) & ~7ul),
     };
 }
 
@@ -75,7 +85,7 @@ void hashmap_deinit(Hashmap* m)
 {
     if (m->entries) {
         for (size_t i = 0; i < m->entries_size; ++i) {
-            free(m->entries[i].key);
+            free(kwpair_at(m, i)->key);
         }
         free(m->entries);
     }
@@ -84,8 +94,9 @@ void hashmap_deinit(Hashmap* m)
 static KwPair* find_entry(Hashmap* m, const char* key)
 {
     for (size_t i = 0; i < m->entries_size; ++i) {
-        if (strcmp(m->entries[i].key, key) == 0) {
+        KwPair* entry = kwpair_at(m, i);
-            return &m->entries[i];
+        if (entry->key == key || strcmp(entry->key, key) == 0) {
+            return entry;
         }
     }
     return nullptr;
@@ -95,11 +106,15 @@ void hashmap_set(Hashmap* m, const char* key, void* val)
 {
     KwPair* existing = find_entry(m, key);
     if (existing) {
-        existing->val = val;
+        memcpy(&existing->val, val, m->elem_size);
         return;
     }
 
-    push_kwpair(m, &(KwPair) { strdup(key), val });
+    KwPair* entry = malloc(m->pair_size);
+    entry->key = strdup(key);
+    memcpy(&entry->val, val, m->elem_size);
+    push_kwpair(m, entry);
+    free(entry);
 }
 
 void* hashmap_get(Hashmap* m, const char* key)
@@ -108,6 +123,15 @@ void* hashmap_get(Hashmap* m, const char* key)
     return existing ? existing->val : nullptr;
 }
 
+void hashmap_for_each(
+    Hashmap* m, void fn(void* self, const char* key, void* val), void* self)
+{
+    for (size_t i = 0; i < m->entries_size; ++i) {
+        KwPair* entry = kwpair_at(m, i);
+        fn(self, entry->key, &entry->val);
+    }
+}
+
 void arena_init(Arena* a, size_t bucket_cap)
 {
     *a = (Arena) {
@@ -142,9 +166,12 @@ void* arena_alloc(Arena* a, size_t size)
 {
     ArenaBucket* b = &a->buckets[a->buckets_size - 1];
 
-    if (b->offset + size > b->cap) {
+    size_t total_size = sizeof(size_t) + size;
-        size_t buffer_cap
+
-            = a->default_bucket_cap > size ? a->default_bucket_cap : size;
+    if (b->offset + total_size > b->cap) {
+        size_t buffer_cap = a->default_bucket_cap > total_size
+            ? a->default_bucket_cap
+            : total_size;
 
         array_push((void**)&a->buckets,
             &a->buckets_cap,
@@ -160,10 +187,11 @@ void* arena_alloc(Arena* a, size_t size)
     }
 
     unsigned char* ptr = b->data + b->offset;
+    *(size_t*)ptr = size;
 
-    b->offset += (size + 7ul) & ~7ul;
+    b->offset += (total_size + 7ul) & ~7ul;
 
-    return ptr;
+    return ptr + sizeof(size_t);
 }
 
 static void* arena_alloc_wrap(void* self, size_t size)
@@ -175,8 +203,9 @@ static void* arena_alloc_wrap(void* self, size_t size)
 static void* arena_realloc_wrap(void* self, void* p, size_t size)
 {
     Arena* a = self;
+    size_t prev_size = *(size_t*)((unsigned char*)p - sizeof(size_t));
     void* new_buf = arena_alloc(a, size);
-    memcpy(new_buf, p, size);
+    memcpy(new_buf, p, prev_size);
     return new_buf;
 }
 

collection.h

@@ -6,23 +6,50 @@
 void array_push(
     void** data, size_t* capacity, size_t* size, void* elem, size_t elem_size);
 
-typedef struct {
+#define ARRAY_FIELD(TYPE, FIELD_NAME)                                          \
-    char* key;
+    TYPE* FIELD_NAME;                                                          \
-    void* val;
+    size_t FIELD_NAME##_cap;                                                   \
-} KwPair;
+    size_t FIELD_NAME##_size
+
+#define ARRAY_FIELD_INIT(FIELD_NAME)                                           \
+    .FIELD_NAME = nullptr, .FIELD_NAME##_cap = 0, .FIELD_NAME##_size = 0
+
+#define ARRAY_FIELD_PUSH(STRUCT_PTR, FIELD_NAME, ELEM_PTR)                     \
+    array_push((void**)&(STRUCT_PTR)->FIELD_NAME,                              \
+        &(STRUCT_PTR)->FIELD_NAME##_cap,                                       \
+        &(STRUCT_PTR)->FIELD_NAME##_size,                                      \
+        (ELEM_PTR),                                                            \
+        sizeof(*(ELEM_PTR)))
+
+#define ARRAY_VAR(TYPE, FIELD_NAME)                                            \
+    TYPE* FIELD_NAME = nullptr;                                                \
+    size_t FIELD_NAME##_cap = 0;                                               \
+    size_t FIELD_NAME##_size = 0;
+
+#define ARRAY_VAR_PUSH(VAR_NAME, ELEM_PTR)                                     \
+    array_push((void**)&VAR_NAME,                                              \
+        &VAR_NAME##_cap,                                                       \
+        &VAR_NAME##_size,                                                      \
+        (ELEM_PTR),                                                            \
+        sizeof(*(ELEM_PTR)))
+
+typedef struct KwPair KwPair;
 
 typedef struct {
-    KwPair* entries;
+    ARRAY_FIELD(KwPair, entries);
-    size_t entries_capacity;
+    size_t elem_size;
-    size_t entries_size;
+    size_t pair_size;
 } Hashmap;
 
-void hashmap_init(Hashmap* m);
+void hashmap_init(Hashmap* m, size_t elem_size);
 void hashmap_deinit(Hashmap* m);
 
 void hashmap_set(Hashmap* m, const char* key, void* val);
 void* hashmap_get(Hashmap* m, const char* key);
 
+void hashmap_for_each(
+    Hashmap* m, void fn(void* self, const char* key, void* val), void* self);
+
 typedef struct {
     unsigned char* data;
     size_t cap;

examples/hello.c

@@ -1,8 +1,9 @@
-int puts(const char* __s);
+#include "not_stdio.h"
 
-#define A puts
 #define B(V, W) A(V)
-#define C B("hello world", 123)
+#define D B
+#define C D("hello world", 123)
+#define A puts
 
 int main()
 {

examples/not_stdio.h

@@ -0,0 +1,2 @@
+
+int puts(const char* __s);

lex.c

@@ -1,10 +1,50 @@
 #include "collection.h"
 #include "parse.h"
 #include <assert.h>
+#include <errno.h>
 #include <stddef.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
+char* read_textfile(const char* filename)
+{
+    FILE* fp = fopen(filename, "r");
+    if (!fp) {
+        goto couldnt_read_file;
+    }
+
+    if (fseek(fp, 0, SEEK_END) != 0) {
+        goto couldnt_read_file;
+    }
+
+    long file_size = ftell(fp);
+    if (file_size < 0) {
+        goto couldnt_read_file;
+    }
+
+    if (fseek(fp, 0, SEEK_SET) != 0) {
+        goto couldnt_read_file;
+    }
+
+    char* buffer = malloc(file_size + 1);
+    assert(buffer);
+
+    if (fread(buffer, sizeof(char), file_size, fp) != (size_t)file_size) {
+        goto couldnt_read_file;
+    }
+
+    buffer[file_size] = '\0';
+    return buffer;
+
+couldnt_read_file:
+    fprintf(stderr,
+        "error: couldn't read file '%s': %s\n",
+        filename,
+        strerror(errno));
+    return nullptr;
+}
+
 const char* tokty_str(TokTy ty)
 {
     switch (ty) {
@@ -180,6 +220,7 @@ Tok lexer_next(Lexer* l)
 
     if (l->lex_headername && (l->curr.ch == '"' || l->curr.ch == '<')) {
         char ter = l->curr.ch == '<' ? '>' : '"';
+        step(l, text, &text_len);
         while (l->curr.ch != '\0' && l->curr.ch != '\n' && l->curr.ch != ter) {
             step(l, text, &text_len);
         }

main.c

@@ -1,50 +1,11 @@
 #include "collection.h"
 #include "parse.h"
 #include <assert.h>
-#include <errno.h>
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-char* read_textfile(const char* filename)
-{
-    FILE* fp = fopen(filename, "r");
-    if (!fp) {
-        goto couldnt_read_file;
-    }
-
-    if (fseek(fp, 0, SEEK_END) != 0) {
-        goto couldnt_read_file;
-    }
-
-    long file_size = ftell(fp);
-    if (file_size < 0) {
-        goto couldnt_read_file;
-    }
-
-    if (fseek(fp, 0, SEEK_SET) != 0) {
-        goto couldnt_read_file;
-    }
-
-    char* buffer = malloc(file_size + 1);
-    assert(buffer);
-
-    if (fread(buffer, sizeof(char), file_size, fp) != (size_t)file_size) {
-        goto couldnt_read_file;
-    }
-
-    buffer[file_size] = '\0';
-    return buffer;
-
-couldnt_read_file:
-    fprintf(stderr,
-        "error: couldn't read file '%s': %s\n",
-        filename,
-        strerror(errno));
-    return nullptr;
-}
-
 int main(int argc, char* argv[])
 {
     if (argc < 2) {
@@ -59,27 +20,15 @@ int main(int argc, char* argv[])
 
     printf("--- text ---\n%s\n", text);
 
-    SourceIter source_chars;
+    Preproc* preproc;
-    source_iter_init(&source_chars, text);
+    preproc_init(&preproc, strdup(filename), text);
-
-    LslIter lsl_chars;
-    lsl_iter_init(&lsl_chars, &source_chars);
-
-    Arena tok_arena;
-    arena_init(&tok_arena, strlen(text) * 4);
-
-    Lexer lexer;
-    lexer_init(&lexer, &lsl_chars, &tok_arena);
-
-    Preproc preproc;
-    preproc_init(&preproc, &lexer);
 
     printf("--- tokens ---\n");
 
     int line = 1;
     int col = 1;
     Tok tok;
-    while ((tok = preproc_next(&preproc)).ty != Tok_Eof) {
+    while ((tok = preproc_next(preproc)).ty != Tok_Eof) {
         // printf("%s\t%s\n", tokty_str(tok.ty), tok.text);
         while (line < tok.line) {
             printf("\n");
@@ -96,7 +45,5 @@ int main(int argc, char* argv[])
     }
     fputs("\n", stdout);
 
-    preproc_deinit(&preproc);
+    preproc_deinit(preproc);
-    free(text);
-    arena_deinit(&tok_arena);
 }

parse.h

@@ -4,6 +4,8 @@
 #include "collection.h"
 #include <stddef.h>
 
+char* read_textfile(const char* filename);
+
 // phases:
 // https://en.cppreference.com/c/language/translation_phases
 
@@ -54,6 +56,9 @@ typedef enum {
     Tok_Comma = ',',
     Tok_Asterisk = '*',
     Tok_Hashtag = '#',
+
+    // used internally in preprocessor
+    Tok_EndOfMacroInvocation,
 } TokTy;
 
 const char* tokty_str(TokTy ty);
@@ -78,33 +83,9 @@ Tok lexer_next(Lexer* l);
 
 // phase 4 - preprocessor
 
-typedef struct MacroDef MacroDef;
+typedef struct Preproc Preproc;
-typedef struct MacroInv MacroInv;
 
-typedef struct {
+void preproc_init(Preproc** p, char* filename, char* text);
-    MacroDef* data;
-    size_t cap;
-    size_t size;
-} MacroDefs;
-
-typedef struct {
-    MacroInv* data;
-    size_t cap;
-    size_t size;
-} MacroInvs;
-
-typedef struct {
-    Lexer* lexer;
-    Tok curr;
-
-    MacroDefs defs;
-    Hashmap def_map;
-
-    Arena arg_arena;
-    MacroInvs macro_invs;
-} Preproc;
-
-void preproc_init(Preproc* p, Lexer* lexer);
 void preproc_deinit(Preproc* p);
 Tok preproc_next_with_newline(Preproc* p);
 Tok preproc_next(Preproc* p);

preproc.c

@@ -6,66 +6,310 @@
 #include <stdlib.h>
 #include <string.h>
 
-struct MacroDef {
+typedef struct {
-    Tok* toks;
+    ARRAY_FIELD(Hashmap, maps);
-    size_t toks_cap;
+} Syms;
-    size_t toks_size;
 
-    char** params;
+static void syms_push_layer(Syms* s);
-    size_t params_size;
-};
 
-struct MacroInv {
+static void syms_init(Syms* s)
-    MacroDef* def;
+{
-    size_t tok_idx;
+    *s = (Syms) {
-    Hashmap arg_map;
+        ARRAY_FIELD_INIT(maps),
-};
+    };
+    syms_push_layer(s);
+}
+
+static void syms_deinit(Syms* s)
+{
+    if (s->maps) {
+        for (size_t i = 0; i < s->maps_size; ++i) {
+            hashmap_deinit(&s->maps[i]);
+        }
+        free(s->maps);
+    }
+}
+
+static void syms_push_layer(Syms* s)
+{
+    Hashmap m = { };
+    hashmap_init(&m, sizeof(size_t));
+    array_push((void**)&s->maps, &s->maps_cap, &s->maps_size, &m, sizeof(m));
+}
+
+static void syms_pop_layer(Syms* s)
+{
+    assert(s->maps_size > 0);
+
+    hashmap_deinit(&s->maps[s->maps_size - 1]);
+    s->maps_size -= 1;
+}
+
+static Hashmap* syms_current(Syms* s)
+{
+    assert(s->maps_size > 0);
+    return &s->maps[s->maps_size - 1];
+}
+
+static void syms_define(Syms* s, const char* ident, size_t def_idx)
+{
+    Hashmap* m = syms_current(s);
+    hashmap_set(m, ident, &def_idx);
+}
+
+static size_t* syms_get(Syms* s, const char* ident)
+{
+    for (size_t i = 0; i < s->maps_size; ++i) {
+        Hashmap* m = &s->maps[s->maps_size - i - 1];
+        size_t* def_idx = hashmap_get(m, ident);
+        if (def_idx)
+            return def_idx;
+    }
+    return nullptr;
+}
+
+typedef struct {
+    char* filename;
+    char* text;
+    SourceIter source_iter;
+    LslIter lsl_iter;
+    Lexer lexer;
+} File;
 
 typedef struct {
     Tok* toks;
     size_t toks_size;
-} MacroArg;
 
-void preproc_init(Preproc* p, Lexer* lexer)
+    char** params;
+    size_t params_size;
+} Def;
+
+struct Preproc {
+    Arena tok_arena;
+    ARRAY_FIELD(File, files);
+
+    ARRAY_FIELD(Tok, tok_stack);
+    Tok curr;
+
+    ARRAY_FIELD(Def, defs);
+    Syms syms;
+
+    bool next_can_be_directive;
+};
+
+void preproc_init(Preproc** p, char* filename, char* text)
 {
-    Tok curr = lexer_next(lexer);
+    *p = malloc(sizeof(Preproc));
 
-    *p = (Preproc) {
+    **p = (Preproc) {
-        .lexer = lexer,
+        ARRAY_FIELD_INIT(files),
-        .curr = curr,
+        ARRAY_FIELD_INIT(tok_stack),
+        .curr = { },
 
-        .defs = { },
+        ARRAY_FIELD_INIT(defs),
-        .def_map = { },
+        .syms = { },
 
-        .arg_arena = { },
+        .tok_arena = { },
-        .macro_invs = { },
+
+        .next_can_be_directive = true,
     };
 
-    hashmap_init(&p->def_map);
+    arena_init(&(*p)->tok_arena, 512);
-    arena_init(&p->arg_arena, 512);
+
+    ARRAY_FIELD_PUSH(*p,
+        files,
+        (&(File) {
+            .filename = filename,
+            .text = text,
+            .source_iter = { },
+            .lsl_iter = { },
+            .lexer = { },
+        }));
+
+    File* file = &(*p)->files[0];
+    source_iter_init(&file->source_iter, text);
+    lsl_iter_init(&file->lsl_iter, &file->source_iter);
+    lexer_init(&file->lexer, &file->lsl_iter, &(*p)->tok_arena);
+    (*p)->curr = lexer_next(&file->lexer);
+
+    // null-definition at idx 0
+    ARRAY_FIELD_PUSH(*p, defs, &(Def) { });
+
+    syms_init(&(*p)->syms);
 }
 
 void preproc_deinit(Preproc* p)
 {
-    if (p->defs.data) {
+    for (size_t i = 0; i < p->files_size; ++i) {
-        for (size_t i = 0; i < p->defs.size; ++i) {
+        free(p->files[i].filename);
-            free(p->defs.data[i].toks);
+        free(p->files[i].text);
-            if (p->defs.data[i].params) {
-                free(p->defs.data[i].params);
     }
+    free(p->files);
+    if (p->tok_stack)
+        free(p->tok_stack);
+    if (p->defs) {
+        for (size_t i = 0; i < p->defs_size; ++i) {
+            if (p->defs[i].params)
+                free(p->defs[i].params);
         }
-        free(p->defs.data);
+        free(p->defs);
     }
-    hashmap_deinit(&p->def_map);
+    syms_deinit(&p->syms);
-    arena_deinit(&p->arg_arena);
+    arena_deinit(&p->tok_arena);
+    free(p);
 }
 
 static void step(Preproc* p)
 {
-    p->curr = lexer_next(p->lexer);
+    p->next_can_be_directive = p->curr.ty == Tok_Eol;
+
+    if (p->tok_stack_size == 0) {
+        p->curr = lexer_next(&p->files[p->files_size - 1].lexer);
+
+        if (p->curr.ty == Tok_Eof && p->files_size > 1) {
+            File* curr_file = &p->files[p->files_size - 1];
+            free(curr_file->filename);
+            free(curr_file->text);
+
+            p->files_size -= 1;
+            step(p);
+        }
+
+        return;
+    }
+    p->curr = p->tok_stack[p->tok_stack_size - 1];
+    p->tok_stack_size -= 1;
+
+    if (p->curr.ty == Tok_EndOfMacroInvocation) {
+        syms_pop_layer(&p->syms);
+        step(p);
+        return;
+    }
 }
 
-static Tok p_define(Preproc* p)
+static void collect_arg(Preproc* p, size_t def_idx, size_t arg_idx)
+{
+    if (arg_idx >= p->defs[def_idx].params_size) {
+        fprintf(stderr,
+            "error: too many arguments for macro on line %d\n",
+            p->curr.line);
+        exit(1);
+    }
+
+    ARRAY_VAR(Tok, toks);
+
+    int depth = 0;
+
+    while (p->curr.ty != Tok_Eof
+        && (depth != 0 || (p->curr.ty != ',' && p->curr.ty != ')'))) {
+
+        if (p->curr.ty == '(')
+            depth += 1;
+        if (p->curr.ty == ')') {
+            if (depth == 0)
+                continue;
+            depth -= 1;
+        }
+
+        arena_array_push(&p->tok_arena,
+            (void**)&toks,
+            &toks_cap,
+            &toks_size,
+            (&p->curr),
+            sizeof(*(&p->curr)));
+
+        step(p);
+    }
+
+    if (p->curr.ty != ',' && p->curr.ty != ')') {
+        assert(p->curr.ty == Tok_Eof);
+        fprintf(
+            stderr, "error: expected ',' or ')' on line %d\n", p->curr.line);
+        exit(1);
+    }
+
+    ARRAY_FIELD_PUSH(p,
+        defs,
+        (&(Def) {
+            .toks = toks,
+            .toks_size = toks_size,
+        }));
+
+    syms_define(&p->syms, p->defs[def_idx].params[arg_idx], p->defs_size - 1);
+}
+
+#define A(B, C) B C
+
+static void collect_args(Preproc* p, size_t def_idx)
+{
+    assert(p->curr.ty == '(');
+    step(p);
+
+    size_t idx = 0;
+
+    if (p->curr.ty != Tok_Eof && p->curr.ty != ')') {
+        collect_arg(p, def_idx, idx);
+        idx += 1;
+        while (p->curr.ty == ',') {
+            step(p);
+            collect_arg(p, def_idx, idx);
+            idx += 1;
+        }
+    }
+    if (p->curr.ty != ')') {
+        fprintf(stderr, "error: expected ')' on line %d\n", p->curr.line);
+        exit(1);
+    }
+
+    assert(idx >= p->defs[def_idx].params_size);
+    if (idx > p->defs[def_idx].params_size) {
+        fprintf(stderr,
+            "error: too few arguments for macro on line %d\n",
+            p->curr.line);
+        exit(1);
+    }
+}
+
+static Tok expand_tok(Preproc* p)
+{
+    Tok tok = p->curr;
+    size_t* def_idx;
+    if (tok.ty != Tok_Ident || !(def_idx = syms_get(&p->syms, tok.text))) {
+        step(p);
+        return tok;
+    }
+
+    // macro expension doesn't support recursion.
+    // when seeing the same macro a second time
+    // in the invocation tree, treat it as a normal token.
+    // undefine it at this layer.
+    syms_push_layer(&p->syms);
+    syms_define(&p->syms, tok.text, 0);
+
+    if (p->defs[*def_idx].params) {
+        step(p);
+        // function-like macros without arg-lists are ignored.
+        if (p->curr.ty != '(')
+            return tok;
+        collect_args(p, *def_idx);
+    }
+
+    ARRAY_FIELD_PUSH(p, tok_stack, (&(Tok) { .ty = Tok_EndOfMacroInvocation }));
+    for (size_t i = 0; i < p->defs[*def_idx].toks_size; ++i) {
+        Def* def = &p->defs[*def_idx];
+        Tok expanded_tok = def->toks[def->toks_size - i - 1];
+        expanded_tok.line = tok.line;
+        expanded_tok.col = tok.col;
+        ARRAY_FIELD_PUSH(p, tok_stack, &expanded_tok);
+    }
+
+    // step into the new invocation
+    step(p);
+
+    return expand_tok(p);
+}
+
+static Tok do_define(Preproc* p)
 {
     step(p);
     if (p->curr.ty != Tok_Ident) {
@@ -75,50 +319,36 @@ static Tok p_define(Preproc* p)
     }
     Tok ident_tok = p->curr;
 
-    p->lexer->lex_whitespace = true;
+    p->files[p->files_size - 1].lexer.lex_whitespace = true;
     step(p);
-    p->lexer->lex_whitespace = false;
+    p->files[p->files_size - 1].lexer.lex_whitespace = false;
 
-    MacroDef def = { };
+    Def def = { };
 
     if (p->curr.ty == '(') {
         step(p);
 
-        char** params = nullptr;
+        ARRAY_VAR(char*, params);
-        size_t params_cap = 0;
-        size_t params_size = 0;
 
         if (p->curr.ty != Tok_Eof && p->curr.ty != ')') {
-            if (p->curr.ty != Tok_Ident) {
+            if (p->curr.ty != Tok_Ident)
-                fprintf(stderr,
+                goto error_expected_identifier;
-                    "error: expected identifier on line %d\n",
+
-                    p->curr.line);
+            ARRAY_VAR_PUSH(params, &p->curr.text);
-                exit(1);
+            step(p);
-            }
-            array_push((void**)&params,
-                &params_cap,
-                &params_size,
-                p->curr.text,
-                sizeof(p->curr.text));
 
             while (p->curr.ty == ',') {
                 step(p);
 
-                if (p->curr.ty != Tok_Ident) {
+                if (p->curr.ty != Tok_Ident)
-                    fprintf(stderr,
+                    goto error_expected_identifier;
-                        "error: expected identifier on line %d\n",
+
-                        p->curr.line);
+                ARRAY_VAR_PUSH(params, &p->curr.text);
-                    exit(1);
+                step(p);
-                }
-                array_push((void**)&params,
-                    &params_cap,
-                    &params_size,
-                    p->curr.text,
-                    sizeof(p->curr.text));
             }
         }
 
-        if (p->curr.ty == Tok_Eof || p->curr.ty == ')') {
+        if (p->curr.ty != ')') {
             fprintf(stderr, "error: expected ')' on line %d\n", p->curr.line);
             exit(1);
         }
@@ -128,25 +358,35 @@ static Tok p_define(Preproc* p)
         def.params_size = params_size;
     }
 
+    if (p->curr.ty == Tok_Whitespace)
+        step(p);
+
+    ARRAY_VAR(Tok, toks);
+
     while (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
-        array_push((void**)&def.toks,
+        arena_array_push(&p->tok_arena,
-            &def.toks_cap,
+            (void**)&toks,
-            &def.toks_size,
+            &toks_cap,
-            &p->curr,
+            &toks_size,
-            sizeof(p->curr));
+            (&p->curr),
+            sizeof(*(&p->curr)));
         step(p);
     }
 
-    array_push(
+    def.toks = toks;
-        (void**)&p->defs, &p->defs.cap, &p->defs.size, &def, sizeof(def));
+    def.toks_size = toks_size;
 
-    MacroDef* deftoks = &p->defs.data[p->defs.size - 1];
+    ARRAY_FIELD_PUSH(p, defs, &def);
-    hashmap_set(&p->def_map, ident_tok.text, deftoks);
+    syms_define(&p->syms, ident_tok.text, p->defs_size - 1);
 
     return p->curr;
+
+error_expected_identifier:
+    fprintf(stderr, "error: expected identifier on line %d\n", p->curr.line);
+    exit(1);
 }
 
-static Tok p_undef(Preproc* p)
+static Tok do_undef(Preproc* p)
 {
     step(p);
 
@@ -164,121 +404,87 @@ static Tok p_undef(Preproc* p)
         exit(1);
     }
 
-    hashmap_set(&p->def_map, ident_tok.text, nullptr);
+    syms_define(&p->syms, ident_tok.text, 0);
 
     return p->curr;
 }
 
-static Tok p_include(Preproc* p)
+static Tok do_include(Preproc* p)
 {
-    p->lexer->lex_headername = true;
+    File* curr_file = &p->files[p->files_size - 1];
+
+    curr_file->lexer.lex_headername = true;
     step(p);
-    p->lexer->lex_headername = false;
+    curr_file->lexer.lex_headername = false;
 
     if (p->curr.ty != Tok_HeaderName) {
-        fprintf(stderr, "error: expected header name line %d\n", p->curr.line);
+        fprintf(
+            stderr, "error: expected header name on line %d\n", p->curr.line);
         exit(1);
     }
 
-    assert(false && "not implemented");
+    Tok headername_tok = p->curr;
-}
-
-static Tok p_paste_macro_body(Preproc* p)
-{
-    MacroInv* inv = &p->macro_invs.data[p->macro_invs.size - 1];
-
-    if (inv->tok_idx >= inv->def->toks_size) {
-        p->macro_invs.size -= 1;
-        return preproc_next_with_newline(p);
-    }
-
-    Tok tok = inv->def->toks[inv->tok_idx];
-    inv->tok_idx += 1;
-
-    return tok;
-}
-
-static void collect_arg_toks(Tok** toks, size_t* toks_size)
-{
-}
-
-struct arg_def {
-    Tok* toks;
-    size_t toks_size;
-};
-
-static Tok p_detect_macro(Preproc* p, Tok head)
-{
-    MacroDef* def = hashmap_get(&p->def_map, head.text);
-    if (!def)
-        return head;
-
-    MacroInv inv = {
-        .def = def,
-        .tok_idx = 0,
-        .arg_map = { },
-    };
-    hashmap_init(&inv.arg_map);
-
-    array_push(
-        (void**)&p->macro_invs, &p->defs.cap, &p->defs.size, &inv, sizeof(inv));
-
-    if (!def->params)
-        return preproc_next_with_newline(p);
-
-    if (p->curr.ty != '(')
-        return head;
-
     step(p);
 
-    struct arg_def* arg_defs;
+    if (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
-    size_t arg_defs_cap = 0;
-    size_t arg_defs_size = 0;
-
-    size_t arg_idx = 0;
-
-    bool first = true;
-
-    while ((first && (p->curr.ty != Tok_Eof && p->curr.ty != ')'))
-        || p->curr.ty != ',') {
-        first = false;
-
-        if (arg_idx + 1 >= def->params_size) {
         fprintf(stderr,
-                "error: too many arguments for macro line %d\n",
+            "error: unexpected extra tokens on line %d\n",
             p->curr.line);
         exit(1);
     }
 
-        Tok* toks;
+    char* rel_path
-        size_t toks_size;
+        = strndup(headername_tok.text + 1, strlen(headername_tok.text) - 2);
-        collect_arg_toks(&toks, &toks_size);
 
-        arena_array_push(&p->arg_arena,
+    char* last_delim = strrchr(curr_file->filename, '/');
-            (void**)&arg_defs,
-            &arg_defs_cap,
-            &arg_defs_size,
-            &(struct arg_def) { toks, toks_size },
-            sizeof(struct arg_def));
-        hashmap_set(
-            &inv.arg_map, def->params[arg_idx], &arg_defs[arg_defs_size - 1]);
 
-        arg_idx += 1;
+    char* filename;
+    if (last_delim) {
+        size_t path_len = last_delim - curr_file->filename;
+
+        filename = calloc(path_len + 1 + strlen(rel_path) + 1, 1);
+
+        strncat(filename, curr_file->filename, path_len);
+        strcat(filename, "/");
+        strcat(filename, rel_path);
+    } else {
+        filename = strdup(rel_path);
     }
+    free(rel_path);
 
-    return preproc_next_with_newline(p);
+    char* text = read_textfile(filename);
+
+    ARRAY_FIELD_PUSH(p,
+        files,
+        (&(File) {
+            .filename = filename,
+            .text = text,
+            .source_iter = { },
+            .lsl_iter = { },
+            .lexer = { },
+        }));
+
+    File* file = &p->files[p->files_size - 1];
+    source_iter_init(&file->source_iter, text);
+    lsl_iter_init(&file->lsl_iter, &file->source_iter);
+    lexer_init(&file->lexer, &file->lsl_iter, &p->tok_arena);
+    p->curr = lexer_next(&file->lexer);
+
+    return p->curr;
 }
 
-static Tok p_interpret_directive(Preproc* p)
+static Tok interpret_directive(Preproc* p)
 {
+    assert(p->curr.ty == '#');
+    step(p);
     Tok kw = p->curr;
 
     if (strcmp(kw.text, "define") == 0) {
-        return p_define(p);
+        return do_define(p);
     } else if (strcmp(kw.text, "undef") == 0) {
-        return p_undef(p);
+        return do_undef(p);
     } else if (strcmp(kw.text, "include") == 0) {
-        return p_include(p);
+        return do_include(p);
     } else if (strcmp(kw.text, "if") == 0) {
         assert(false && "not implemented");
     } else if (strcmp(kw.text, "ifdef") == 0) {
@@ -314,38 +520,11 @@ static Tok p_interpret_directive(Preproc* p)
     assert(false && "unreachable");
 }
 
-static Tok replace_candidate_tok(Preproc* p, Tok tok, size_t inv_idx)
-{
-    if (inv_idx == 0)
-        return tok;
-    MacroInv* inv = &p->macro_invs.data[inv_idx];
-    if (tok.ty != Tok_Ident)
-        return tok;
-    MacroArg* arg = hashmap_get(&inv->arg_map, tok.text);
-    if (!arg)
-        return replace_candidate_tok(p, tok, inv_idx - 1);
-}
-
-static Tok expand_tok(Preproc* p, Tok tok)
-{
-    if (p->macro_invs.size == 0)
-        return tok;
-
-    MacroInv* inv = &p->macro_invs.data[p->macro_invs.size - 1];
-
-    if (inv->tok_idx >= inv->def->toks_size) {
-        p->macro_invs.size -= 1;
-        return expand_tok(p, tok);
-    }
-}
-
 Tok preproc_next_with_newline(Preproc* p)
 {
-    Tok tok = p->curr;
+    if (p->next_can_be_directive && p->curr.ty == '#')
-    step(p);
+        return interpret_directive(p);
-    if (tok.ty == '#')
+    return expand_tok(p);
-        return p_interpret_directive(p);
-    return expand_tok(p, tok);
 }
 
 Tok preproc_next(Preproc* p)