preproc toks

.gitignore

@@ -0,0 +1,2 @@
+larpcc
+.vscode/

Makefile

@@ -0,0 +1,5 @@
+
+larpcc: main.c lex.c preproc.c collection.c
+	gcc $^ -o $@ -std=c23 -pedantic -Wall -Wextra -g -fsanitize=address
+
+

collection.c

@@ -1,5 +1,6 @@
 #include "collection.h"
 #include <stddef.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -10,14 +11,14 @@ void array_push(
         *capacity = 4;
     }
     if (!*data) {
-        *data = malloc(elem_size * *capacity);
+        *data = malloc(*capacity * elem_size);
         *size = 0;
     }
     if (*size >= *capacity) {
         *capacity *= 2;
         *data = realloc(*data, *capacity * elem_size);
     }
-    memcpy((unsigned char*)*data + *size, elem, elem_size);
+    memcpy((unsigned char*)*data + *size * elem_size, elem, elem_size);
     *size += 1;
 }
 
@@ -75,3 +76,61 @@ void* hashmap_get(Hashmap* m, const char* key)
     KwPair* existing = find_entry(m, key);
     return existing ? existing->val : nullptr;
 }
+
+void arena_init(Arena* a, size_t bucket_cap)
+{
+    *a = (Arena) {
+        .buckets = nullptr,
+        .buckets_cap = 0,
+        .buckets_size = 0,
+        .default_bucket_cap = bucket_cap,
+    };
+
+    array_push((void**)&a->buckets,
+        &a->buckets_cap,
+        &a->buckets_size,
+        &(ArenaBucket) {
+            .data = malloc(bucket_cap),
+            .cap = bucket_cap,
+            .offset = 0,
+        },
+        sizeof(ArenaBucket));
+}
+
+void arena_deinit(Arena* a)
+{
+    if (a->buckets) {
+        for (size_t i = 0; i < a->buckets_size; ++i) {
+            free(a->buckets[i].data);
+        }
+        free(a->buckets);
+    }
+}
+
+void* arena_alloc(Arena* a, size_t size)
+{
+    ArenaBucket* b = &a->buckets[a->buckets_size - 1];
+
+    if (b->offset + size > b->cap) {
+        size_t buffer_cap
+            = a->default_bucket_cap > size ? a->default_bucket_cap : size;
+
+        array_push((void**)&a->buckets,
+            &a->buckets_cap,
+            &a->buckets_size,
+            &(ArenaBucket) {
+                .data = malloc(buffer_cap),
+                .cap = buffer_cap,
+                .offset = 0,
+            },
+            sizeof(ArenaBucket));
+
+        b = &a->buckets[a->buckets_size - 1];
+    }
+
+    unsigned char* ptr = b->data + b->offset;
+
+    b->offset += (size + 7ul) & ~7ul;
+
+    return ptr;
+}

collection.h

@@ -23,4 +23,21 @@ void hashmap_deinit(Hashmap* m);
 void hashmap_set(Hashmap* m, const char* key, void* val);
 void* hashmap_get(Hashmap* m, const char* key);
 
+typedef struct {
+    unsigned char* data;
+    size_t cap;
+    size_t offset;
+} ArenaBucket;
+
+typedef struct {
+    ArenaBucket* buckets;
+    size_t buckets_cap;
+    size_t buckets_size;
+    size_t default_bucket_cap;
+} Arena;
+
+void arena_init(Arena* a, size_t bucket_cap);
+void arena_deinit(Arena* a);
+void* arena_alloc(Arena* a, size_t size);
+
 #endif

examples/hello.c

@@ -1,6 +1,8 @@
 int puts(const char* __s);
 
+#define PUTS puts
+
 int main()
 {
-    puts("hello");
+    PUTS("hello");
 }

lex.c

@@ -1,12 +1,51 @@
+#include "collection.h"
 #include "parse.h"
 #include <assert.h>
 #include <stddef.h>
 #include <stdio.h>
 #include <string.h>
 
-void source_char_iter_init(SourceCharIter* r, const char* text)
+const char* tokty_str(TokTy ty)
 {
-    *r = (SourceCharIter) {
+    switch (ty) {
+        case Tok_Eof:
+            return "Eof";
+        case Tok_Ident:
+            return "Ident";
+        case Tok_Int:
+            return "Int";
+        case Tok_Str:
+            return "Str";
+        case Tok_Char:
+            return "Char";
+        case Tok_Whitespace:
+            return "Whitespace";
+        case Tok_HeaderName:
+            return "HeaderName";
+        case Tok_Eol:
+            return "Eol";
+        case Tok_Semicolon:
+            return "Semicolon";
+        case Tok_LParen:
+            return "LParen";
+        case Tok_RParen:
+            return "RParen";
+        case Tok_LBrace:
+            return "LBrace";
+        case Tok_RBrace:
+            return "RBrace";
+        case Tok_Asterisk:
+            return "Asterisk";
+        case Tok_Hashtag:
+            return "Hashtag";
+        default:
+            return "<unknown>";
+    }
+}
+
+void source_iter_init(SourceIter* r, const char* text)
+{
+    *r = (SourceIter) {
         .text = text,
         .i = 0,
         .line = 1,
@@ -14,7 +53,7 @@ void source_char_iter_init(SourceCharIter* r, const char* text)
     };
 }
 
-Char source_iter_next(SourceCharIter* r)
+Char source_iter_next(SourceIter* r)
 {
     char ch = r->text[r->i];
     int line = r->line;
@@ -33,17 +72,17 @@ Char source_iter_next(SourceCharIter* r)
     return (Char) { ch, line, col };
 }
 
-void lsl_iter_init(LslCharIter* r, SourceCharIter* source_char_iter)
+void lsl_iter_init(LslIter* r, SourceIter* source_char_iter)
 {
     Char curr = source_iter_next(source_char_iter);
 
-    *r = (LslCharIter) {
+    *r = (LslIter) {
         .source_char_iter = source_char_iter,
         .curr = curr,
     };
 }
 
-Char lsl_iter_next(LslCharIter* r)
+Char lsl_iter_next(LslIter* r)
 {
     Char next = source_iter_next(r->source_char_iter);
 
@@ -52,27 +91,31 @@ Char lsl_iter_next(LslCharIter* r)
         return lsl_iter_next(r);
     }
 
-    Char lsl_char = { r->curr.ch, r->curr.line, r->curr.col };
-
+    Char curr = r->curr;
     r->curr = next;
-    return lsl_char;
+    return curr;
 }
 
-void lexer_init(Lexer* r, LslCharIter* lsl_char_iter)
+void lexer_init(Lexer* r, LslIter* lsl_char_iter, Arena* arena)
 {
     Char curr = lsl_iter_next(lsl_char_iter);
 
     *r = (Lexer) {
-        .lsl_char_iter = lsl_char_iter,
+        .lsl_iter = lsl_char_iter,
+        .arena = arena,
         .curr = curr,
         .lex_whitespace = false,
         .lex_headername = false,
     };
 }
 
-static Tok tok(TokTy ty, const char* text, int line, int col)
+static Tok tok(Lexer* l, TokTy ty, const char* text, int line, int col)
 {
-    return (Tok) { ty, strdup(text), line, col };
+    char* text_dup = arena_alloc(l->arena, strlen(text) + 1);
+    text_dup[0] = '\0';
+    strcpy(text_dup, text);
+
+    return (Tok) { ty, text_dup, line, col };
 }
 
 static bool done(Lexer* l)
@@ -88,7 +131,7 @@ static void step(Lexer* l, char text[], size_t* text_len)
     text[*text_len] = l->curr.ch;
     *text_len += 1;
 
-    l->curr = lsl_iter_next(l->lsl_char_iter);
+    l->curr = lsl_iter_next(l->lsl_iter);
 }
 
 Tok lexer_next(Lexer* l)
@@ -100,7 +143,7 @@ Tok lexer_next(Lexer* l)
     size_t text_len = 0;
 
     if (done(l)) {
-        return tok(Tok_Eof, text, line, col);
+        return tok(l, Tok_Eof, text, line, col);
     }
 
     bool matched = false;
@@ -110,7 +153,7 @@ Tok lexer_next(Lexer* l)
     }
     if (matched) {
         if (l->lex_whitespace) {
-            return tok(Tok_Whitespace, text, line, col);
+            return tok(l, Tok_Whitespace, text, line, col);
         }
         return lexer_next(l);
     }
@@ -123,7 +166,7 @@ Tok lexer_next(Lexer* l)
         step(l, text, &text_len);
     }
     if (matched) {
-        return tok(Tok_Ident, text, line, col);
+        return tok(l, Tok_Ident, text, line, col);
     }
 
     while ((l->curr.ch >= '1' && l->curr.ch <= '9')
@@ -132,7 +175,7 @@ Tok lexer_next(Lexer* l)
         step(l, text, &text_len);
     }
     if (matched) {
-        return tok(Tok_Int, text, line, col);
+        return tok(l, Tok_Int, text, line, col);
     }
 
     if (l->lex_headername && (l->curr.ch == '"' || l->curr.ch == '<')) {
@@ -148,7 +191,7 @@ Tok lexer_next(Lexer* l)
             return lexer_next(l);
         }
         step(l, text, &text_len);
-        return tok(Tok_HeaderName, text, line, col);
+        return tok(l, Tok_HeaderName, text, line, col);
     }
 
     if (l->curr.ch == '"') {
@@ -168,7 +211,7 @@ Tok lexer_next(Lexer* l)
             return lexer_next(l);
         }
         step(l, text, &text_len);
-        return tok(Tok_Str, text, line, col);
+        return tok(l, Tok_Str, text, line, col);
     }
 
     if (l->curr.ch == '\'') {
@@ -186,13 +229,13 @@ Tok lexer_next(Lexer* l)
             return lexer_next(l);
         }
         step(l, text, &text_len);
-        return tok(Tok_Char, text, line, col);
+        return tok(l, Tok_Char, text, line, col);
     }
 
-    if (strchr("\n;(){}#", l->curr.ch)) {
+    if (strchr("\n;(){}*#", l->curr.ch)) {
         TokTy ty = (TokTy)l->curr.ch;
         step(l, text, &text_len);
-        return tok(ty, text, line, col);
+        return tok(l, ty, text, line, col);
     }
     fprintf(stderr,
         "error: illegal character '%c' on line %d\n",

main.c

@@ -1,3 +1,5 @@
+#include "collection.h"
+#include "parse.h"
 #include <assert.h>
 #include <errno.h>
 #include <stddef.h>
@@ -33,6 +35,7 @@ char* read_textfile(const char* filename)
     }
 
     buffer[file_size] = '\0';
+    return buffer;
 
 couldnt_read_file:
     fprintf(stderr,
@@ -53,4 +56,30 @@ int main(int argc, char* argv[])
     if (!text) {
         return EXIT_FAILURE;
     }
+
+    printf("%s\n", text);
+
+    SourceIter source_chars;
+    source_iter_init(&source_chars, text);
+
+    LslIter lsl_chars;
+    lsl_iter_init(&lsl_chars, &source_chars);
+
+    Arena tok_arena;
+    arena_init(&tok_arena, strlen(text) * 4);
+
+    Lexer lexer;
+    lexer_init(&lexer, &lsl_chars, &tok_arena);
+
+    Preproc preproc;
+    preproc_init(&preproc, &lexer);
+
+    Tok tok;
+    while ((tok = preproc_next(&preproc)).ty != Tok_Eof) {
+        printf("%s\t%s\n", tokty_str(tok.ty), tok.text);
+    }
+
+    preproc_deinit(&preproc);
+    free(text);
+    arena_deinit(&tok_arena);
 }

parse.h

@@ -20,20 +20,20 @@ typedef struct {
     size_t i;
     int line;
     int col;
-} SourceCharIter;
+} SourceIter;
 
-void source_char_iter_init(SourceCharIter* r, const char* text);
-Char source_iter_next(SourceCharIter* r);
+void source_iter_init(SourceIter* r, const char* text);
+Char source_iter_next(SourceIter* r);
 
 // phase 2 - logical source lines
 
 typedef struct {
-    SourceCharIter* source_char_iter;
+    SourceIter* source_char_iter;
     Char curr;
-} LslCharIter;
+} LslIter;
 
-void lsl_iter_init(LslCharIter* r, SourceCharIter* source_char_iter);
-Char lsl_iter_next(LslCharIter* r);
+void lsl_iter_init(LslIter* r, SourceIter* source_iter);
+Char lsl_iter_next(LslIter* r);
 
 // phase 3 - preprocessor tokens
 
@@ -51,9 +51,12 @@ typedef enum {
     Tok_RParen = ')',
     Tok_LBrace = '{',
     Tok_RBrace = '}',
+    Tok_Asterisk = '*',
     Tok_Hashtag = '#',
 } TokTy;
 
+const char* tokty_str(TokTy ty);
+
 typedef struct {
     TokTy ty;
     char* text;
@@ -62,13 +65,14 @@ typedef struct {
 } Tok;
 
 typedef struct {
-    LslCharIter* lsl_char_iter;
+    LslIter* lsl_iter;
+    Arena* arena;
     Char curr;
     bool lex_whitespace;
     bool lex_headername;
 } Lexer;
 
-void lexer_init(Lexer* r, LslCharIter* lsl_char_iter);
+void lexer_init(Lexer* r, LslIter* lsl_iter, Arena* arena);
 Tok lexer_next(Lexer* l);
 
 // phase 4 - preprocessor
@@ -83,6 +87,9 @@ typedef struct {
     size_t deftoks_cap;
     size_t deftoks_size;
     Hashmap defs;
+
+    DefToks* pasting_toks;
+    size_t pasting_toks_count;
 } Preproc;
 
 void preproc_init(Preproc* p, Lexer* lexer);

preproc.c

@@ -14,7 +14,7 @@ struct DefToks {
 
 void preproc_init(Preproc* p, Lexer* lexer)
 {
-    Tok curr = lexer_next(p->lexer);
+    Tok curr = lexer_next(lexer);
 
     *p = (Preproc) {
         .lexer = lexer,
@@ -23,6 +23,8 @@ void preproc_init(Preproc* p, Lexer* lexer)
         .deftoks_cap = 0,
         .deftoks_size = 0,
         .defs = { },
+        .pasting_toks = nullptr,
+        .pasting_toks_count = 0,
     };
 
     hashmap_init(&p->defs);
@@ -31,6 +33,9 @@ void preproc_init(Preproc* p, Lexer* lexer)
 void preproc_deinit(Preproc* p)
 {
     if (p->deftoks) {
+        for (size_t i = 0; i < p->deftoks_size; ++i) {
+            free(p->deftoks[i].toks);
+        }
         free(p->deftoks);
     }
     hashmap_deinit(&p->defs);
@@ -41,86 +46,122 @@ static void step(Preproc* p)
     p->curr = lexer_next(p->lexer);
 }
 
-Tok preproc_next(Preproc* p)
+static Tok p_define(Preproc* p)
 {
-    if (p->curr.ty == '#') {
+    step(p);
+    if (p->curr.ty != Tok_Ident) {
+        fprintf(
+            stderr, "error: expected identifier on line %d\n", p->curr.line);
+        exit(1);
+    }
+    Tok ident_tok = p->curr;
+
+    p->lexer->lex_whitespace = true;
+    step(p);
+    p->lexer->lex_whitespace = false;
+
+    if (p->curr.ty == '(') {
+        // function-like macro
+        assert(false && "not implemented");
+    }
+
+    DefToks toks = { };
+
+    while (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
+        array_push((void**)&toks.toks,
+            &toks.toks_cap,
+            &toks.toks_size,
+            &p->curr,
+            sizeof(p->curr));
         step(p);
+    }
+
+    array_push((void**)&p->deftoks,
+        &p->deftoks_cap,
+        &p->deftoks_size,
+        &toks,
+        sizeof(toks));
+
+    DefToks* deftoks = &p->deftoks[p->deftoks_size - 1];
+    hashmap_set(&p->defs, ident_tok.text, deftoks);
+
+    return p->curr;
+}
+
+static Tok p_undef(Preproc* p)
+{
+    step(p);
+
+    if (p->curr.ty != Tok_Ident) {
+        fprintf(
+            stderr, "error: expected identifier on line %d\n", p->curr.line);
+        exit(1);
+    }
+    Tok ident_tok = p->curr;
+
+    if (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
+        fprintf(stderr,
+            "error: unexpected extra tokens on line %d\n",
+            p->curr.line);
+        exit(1);
+    }
+
+    hashmap_set(&p->defs, ident_tok.text, nullptr);
+
+    return p->curr;
+}
+
+static Tok p_include(Preproc* p)
+{
+    p->lexer->lex_headername = true;
+    step(p);
+    p->lexer->lex_headername = false;
+
+    if (p->curr.ty != Tok_HeaderName) {
+        fprintf(stderr, "error: expected header name line %d\n", p->curr.line);
+        exit(1);
+    }
+
+    assert(false && "not implemented");
+}
+
+static Tok preproc_next_nl(Preproc* p)
+{
+    if (p->pasting_toks) {
+        if (p->pasting_toks_count >= p->pasting_toks->toks_size) {
+            p->pasting_toks = nullptr;
+            p->pasting_toks_count = 0;
+            return preproc_next_nl(p);
+        }
+
+        Tok tok = p->pasting_toks->toks[p->pasting_toks_count];
+        p->pasting_toks_count += 1;
+        return tok;
+    }
+
+    Tok curr = p->curr;
+    step(p);
+
+    if (curr.ty == Tok_Ident) {
+        DefToks* def = hashmap_get(&p->defs, curr.text);
+        if (!def)
+            return curr;
+
+        step(p);
+        p->pasting_toks = def;
+        p->pasting_toks_count = 0;
+        return preproc_next_nl(p);
+    }
+
+    if (curr.ty == '#') {
         Tok kw = p->curr;
 
         if (strcmp(kw.text, "define") == 0) {
-            step(p);
-            if (p->curr.ty != Tok_Ident) {
-                fprintf(stderr,
-                    "error: expected identifier on line %d\n",
-                    p->curr.line);
-                exit(1);
-            }
-            Tok ident_tok = p->curr;
-
-            p->lexer->lex_whitespace = true;
-            step(p);
-            p->lexer->lex_whitespace = false;
-
-            if (p->curr.ty == '(') {
-                // function-like macro
-                assert(false && "not implemented");
-            }
-
-            DefToks toks = { };
-
-            while (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
-                array_push((void**)&toks.toks,
-                    &toks.toks_cap,
-                    &toks.toks_size,
-                    &p->curr,
-                    sizeof(p->curr));
-                step(p);
-            }
-
-            array_push((void**)&p->deftoks,
-                &p->deftoks_cap,
-                &p->deftoks_size,
-                &toks,
-                sizeof(toks));
-
-            DefToks* deftoks = &p->deftoks[p->deftoks_size - 1];
-            hashmap_set(&p->defs, ident_tok.text, deftoks);
-
-            return p->curr;
+            return p_define(p);
         } else if (strcmp(kw.text, "undef") == 0) {
-            step(p);
-
-            if (p->curr.ty != Tok_Ident) {
-                fprintf(stderr,
-                    "error: expected identifier on line %d\n",
-                    p->curr.line);
-                exit(1);
-            }
-            Tok ident_tok = p->curr;
-
-            if (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
-                fprintf(stderr,
-                    "error: unexpected extra tokens on line %d\n",
-                    p->curr.line);
-                exit(1);
-            }
-
-            hashmap_set(&p->defs, ident_tok.text, nullptr);
-
-            return p->curr;
+            return p_undef(p);
         } else if (strcmp(kw.text, "include") == 0) {
-            p->lexer->lex_headername = true;
-            step(p);
-            p->lexer->lex_headername = false;
-
-            if (p->curr.ty != Tok_HeaderName) {
-                fprintf(stderr,
-                    "error: expected header name line %d\n",
-                    p->curr.line);
-                exit(1);
-            }
-
-            assert(false && "not implemented");
+            return p_include(p);
         } else if (strcmp(kw.text, "if") == 0) {
             assert(false && "not implemented");
         } else if (strcmp(kw.text, "ifdef") == 0) {
@@ -154,5 +195,13 @@ Tok preproc_next(Preproc* p)
         }
     }
 
-    return p->curr;
+    return curr;
+}
+
+Tok preproc_next(Preproc* p)
+{
+    Tok tok;
+    while ((tok = preproc_next_nl(p)).ty == Tok_Eol
+        || tok.ty == Tok_Whitespace) { }
+    return tok;
 }