phase 6,5 tokstream

main.c

@@ -20,15 +20,21 @@ int main(int argc, char* argv[])
 
     printf("--- text ---\n%s\n", text);
 
+    Arena arena;
+    arena_init(&arena, 4096);
+
     Preproc* preproc;
-    preproc_init(&preproc, strdup(filename), text);
+    preproc_init(&preproc, strdup(filename), text, &arena);
+
+    TokStream tokstream;
+    tokstream_init(&tokstream, preproc, &arena);
 
     printf("--- tokens ---\n");
 
     int line = 1;
     int col = 1;
     Tok tok;
-    while ((tok = preproc_next(preproc)).ty != Tok_Eof) {
+    while ((tok = tokstream_next(&tokstream)).ty != Tok_Eof) {
         // printf("%s\t%s\n", tokty_str(tok.ty), tok.text);
         if (line > tok.line) {
             line = 0;
@@ -49,4 +55,5 @@ int main(int argc, char* argv[])
     fputs("\n", stdout);
 
     preproc_deinit(preproc);
+    arena_deinit(&arena);
 }

parse.h

@@ -85,9 +85,19 @@ Tok lexer_next(Lexer* l);
 
 typedef struct Preproc Preproc;
 
-void preproc_init(Preproc** p, char* filename, char* text);
+void preproc_init(Preproc** p, char* filename, char* text, Arena* arena);
 void preproc_deinit(Preproc* p);
-Tok preproc_next_with_newline(Preproc* p);
 Tok preproc_next(Preproc* p);
 
+// phase 5, 6 - preprocessor cleanup, concat strings
+
+typedef struct {
+    Preproc* preproc;
+    Arena* arena;
+    Tok curr;
+} TokStream;
+
+void tokstream_init(TokStream* s, Preproc* preproc, Arena* arena);
+Tok tokstream_next(TokStream* s);
+
 #endif

preproc.c

@@ -107,7 +107,7 @@ typedef struct {
 } Def;
 
 struct Preproc {
-    Arena tok_arena;
+    Arena* arena;
     ARRAY_FIELD(File, files);
 
     ARRAY_FIELD(Tok, tok_stack);
@@ -121,12 +121,12 @@ struct Preproc {
     bool next_can_be_directive;
 };
 
-void preproc_init(Preproc** p, char* filename, char* text)
+void preproc_init(Preproc** p, char* filename, char* text, Arena* arena)
 {
     *p = malloc(sizeof(Preproc));
 
     **p = (Preproc) {
-        .tok_arena = { },
+        .arena = arena,
         ARRAY_FIELD_INIT(files),
 
         ARRAY_FIELD_INIT(tok_stack),
@@ -140,8 +140,6 @@ void preproc_init(Preproc** p, char* filename, char* text)
         .next_can_be_directive = true,
     };
 
-    arena_init(&(*p)->tok_arena, 512);
-
     ARRAY_FIELD_PUSH(*p,
         files,
         (&(File) {
@@ -155,7 +153,7 @@ void preproc_init(Preproc** p, char* filename, char* text)
     File* file = &(*p)->files[0];
     source_iter_init(&file->source_iter, text);
     lsl_iter_init(&file->lsl_iter, &file->source_iter);
-    lexer_init(&file->lexer, &file->lsl_iter, &(*p)->tok_arena);
+    lexer_init(&file->lexer, &file->lsl_iter, (*p)->arena);
     (*p)->curr = lexer_next(&file->lexer);
 
     // null-definition at idx 0
@@ -181,7 +179,6 @@ void preproc_deinit(Preproc* p)
         free(p->defs);
     }
     syms_deinit(&p->syms);
-    arena_deinit(&p->tok_arena);
     free(p);
 }
 
@@ -237,7 +234,7 @@ static void collect_arg(Preproc* p, size_t def_idx, size_t arg_idx)
             depth -= 1;
         }
 
-        arena_array_push(&p->tok_arena,
+        arena_array_push(p->arena,
             (void**)&toks,
             &toks_cap,
             &toks_size,
@@ -390,7 +387,7 @@ static Tok do_define(Preproc* p)
     ARRAY_VAR(Tok, toks);
 
     while (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) {
-        arena_array_push(&p->tok_arena,
+        arena_array_push(p->arena,
             (void**)&toks,
             &toks_cap,
             &toks_size,
@@ -435,7 +432,7 @@ static Tok do_undef(Preproc* p)
     return p->curr;
 }
 
-Tok preproc_next_with_newline(Preproc* p);
+Tok preproc_next(Preproc* p);
 
 static Tok do_include(Preproc* p)
 {
@@ -495,12 +492,12 @@ static Tok do_include(Preproc* p)
     File* file = &p->files[p->files_size - 1];
     source_iter_init(&file->source_iter, text);
     lsl_iter_init(&file->lsl_iter, &file->source_iter);
-    lexer_init(&file->lexer, &file->lsl_iter, &p->tok_arena);
+    lexer_init(&file->lexer, &file->lsl_iter, p->arena);
     p->curr = lexer_next(&file->lexer);
 
     p->next_can_be_directive = true;
 
-    return preproc_next_with_newline(p);
+    return preproc_next(p);
 }
 
 static Tok interpret_directive(Preproc* p);
@@ -622,7 +619,7 @@ static Tok interpret_directive(Preproc* p)
     exit(1);
 }
 
-Tok preproc_next_with_newline(Preproc* p)
+Tok preproc_next(Preproc* p)
 {
     if (p->next_can_be_directive && p->curr.ty == '#') {
         step(p);
@@ -631,10 +628,39 @@ Tok preproc_next_with_newline(Preproc* p)
     return expand_tok(p);
 }
 
-Tok preproc_next(Preproc* p)
+void tokstream_init(TokStream* s, Preproc* preproc, Arena* arena)
+{
+    *s = (TokStream) {
+        .preproc = preproc,
+        .arena = arena,
+        .curr = { },
+    };
+    s->curr = preproc_next(s->preproc);
+}
+
+static Tok tokstream_next_filtered(TokStream* s)
 {
     Tok tok;
-    while ((tok = preproc_next_with_newline(p)).ty == Tok_Eol
+    while ((tok = preproc_next(s->preproc)).ty == Tok_Eol
         || tok.ty == Tok_Whitespace) { }
     return tok;
 }
+
+Tok tokstream_next(TokStream* s)
+{
+    Tok curr = s->curr;
+    Tok next = tokstream_next_filtered(s);
+
+    if (curr.ty == Tok_Str && next.ty == Tok_Str) {
+        char* text
+            = arena_alloc(s->arena, strlen(curr.text) + strlen(next.text) + 1);
+        text[0] = '\0';
+        strcat(text, curr.text);
+        strcat(text, next.text);
+        s->curr = (Tok) { .ty = Tok_Str, text, curr.line, curr.col };
+        return tokstream_next(s);
+    }
+
+    s->curr = next;
+    return curr;
+}