From 99b5279032d6992b743f870a5cdb3ac6b6121c8e Mon Sep 17 00:00:00 2001 From: sfja Date: Sat, 26 Sep 2026 01:41:16 +0200 Subject: [PATCH] phase 6,5 tokstream --- main.c | 11 +++++++++-- parse.h | 14 ++++++++++++-- preproc.c | 56 ++++++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 62 insertions(+), 19 deletions(-) diff --git a/main.c b/main.c index c57a951..abffb05 100644 --- a/main.c +++ b/main.c @@ -20,15 +20,21 @@ int main(int argc, char* argv[]) printf("--- text ---\n%s\n", text); + Arena arena; + arena_init(&arena, 4096); + Preproc* preproc; - preproc_init(&preproc, strdup(filename), text); + preproc_init(&preproc, strdup(filename), text, &arena); + + TokStream tokstream; + tokstream_init(&tokstream, preproc, &arena); printf("--- tokens ---\n"); int line = 1; int col = 1; Tok tok; - while ((tok = preproc_next(preproc)).ty != Tok_Eof) { + while ((tok = tokstream_next(&tokstream)).ty != Tok_Eof) { // printf("%s\t%s\n", tokty_str(tok.ty), tok.text); if (line > tok.line) { line = 0; @@ -49,4 +55,5 @@ int main(int argc, char* argv[]) fputs("\n", stdout); preproc_deinit(preproc); + arena_deinit(&arena); } diff --git a/parse.h b/parse.h index e7d45cb..7b06542 100644 --- a/parse.h +++ b/parse.h @@ -85,9 +85,19 @@ Tok lexer_next(Lexer* l); typedef struct Preproc Preproc; -void preproc_init(Preproc** p, char* filename, char* text); +void preproc_init(Preproc** p, char* filename, char* text, Arena* arena); void preproc_deinit(Preproc* p); -Tok preproc_next_with_newline(Preproc* p); Tok preproc_next(Preproc* p); +// phase 5, 6 - preprocessor cleanup, concat strings + +typedef struct { + Preproc* preproc; + Arena* arena; + Tok curr; +} TokStream; + +void tokstream_init(TokStream* s, Preproc* preproc, Arena* arena); +Tok tokstream_next(TokStream* s); + #endif diff --git a/preproc.c b/preproc.c index e86ac86..001711d 100644 --- a/preproc.c +++ b/preproc.c @@ -107,7 +107,7 @@ typedef struct { } Def; struct Preproc { - Arena tok_arena; + Arena* arena; ARRAY_FIELD(File, files); ARRAY_FIELD(Tok, tok_stack); @@ -121,12 +121,12 @@ struct Preproc { bool next_can_be_directive; }; -void preproc_init(Preproc** p, char* filename, char* text) +void preproc_init(Preproc** p, char* filename, char* text, Arena* arena) { *p = malloc(sizeof(Preproc)); **p = (Preproc) { - .tok_arena = { }, + .arena = arena, ARRAY_FIELD_INIT(files), ARRAY_FIELD_INIT(tok_stack), @@ -140,8 +140,6 @@ void preproc_init(Preproc** p, char* filename, char* text) .next_can_be_directive = true, }; - arena_init(&(*p)->tok_arena, 512); - ARRAY_FIELD_PUSH(*p, files, (&(File) { @@ -155,7 +153,7 @@ void preproc_init(Preproc** p, char* filename, char* text) File* file = &(*p)->files[0]; source_iter_init(&file->source_iter, text); lsl_iter_init(&file->lsl_iter, &file->source_iter); - lexer_init(&file->lexer, &file->lsl_iter, &(*p)->tok_arena); + lexer_init(&file->lexer, &file->lsl_iter, (*p)->arena); (*p)->curr = lexer_next(&file->lexer); // null-definition at idx 0 @@ -181,7 +179,6 @@ void preproc_deinit(Preproc* p) free(p->defs); } syms_deinit(&p->syms); - arena_deinit(&p->tok_arena); free(p); } @@ -237,7 +234,7 @@ static void collect_arg(Preproc* p, size_t def_idx, size_t arg_idx) depth -= 1; } - arena_array_push(&p->tok_arena, + arena_array_push(p->arena, (void**)&toks, &toks_cap, &toks_size, @@ -390,7 +387,7 @@ static Tok do_define(Preproc* p) ARRAY_VAR(Tok, toks); while (p->curr.ty != Tok_Eof && p->curr.ty != Tok_Eol) { - arena_array_push(&p->tok_arena, + arena_array_push(p->arena, (void**)&toks, &toks_cap, &toks_size, @@ -435,7 +432,7 @@ static Tok do_undef(Preproc* p) return p->curr; } -Tok preproc_next_with_newline(Preproc* p); +Tok preproc_next(Preproc* p); static Tok do_include(Preproc* p) { @@ -495,12 +492,12 @@ static Tok do_include(Preproc* p) File* file = &p->files[p->files_size - 1]; source_iter_init(&file->source_iter, text); lsl_iter_init(&file->lsl_iter, &file->source_iter); - lexer_init(&file->lexer, &file->lsl_iter, &p->tok_arena); + lexer_init(&file->lexer, &file->lsl_iter, p->arena); p->curr = lexer_next(&file->lexer); p->next_can_be_directive = true; - return preproc_next_with_newline(p); + return preproc_next(p); } static Tok interpret_directive(Preproc* p); @@ -622,7 +619,7 @@ static Tok interpret_directive(Preproc* p) exit(1); } -Tok preproc_next_with_newline(Preproc* p) +Tok preproc_next(Preproc* p) { if (p->next_can_be_directive && p->curr.ty == '#') { step(p); @@ -631,10 +628,39 @@ Tok preproc_next_with_newline(Preproc* p) return expand_tok(p); } -Tok preproc_next(Preproc* p) +void tokstream_init(TokStream* s, Preproc* preproc, Arena* arena) +{ + *s = (TokStream) { + .preproc = preproc, + .arena = arena, + .curr = { }, + }; + s->curr = preproc_next(s->preproc); +} + +static Tok tokstream_next_filtered(TokStream* s) { Tok tok; - while ((tok = preproc_next_with_newline(p)).ty == Tok_Eol + while ((tok = preproc_next(s->preproc)).ty == Tok_Eol || tok.ty == Tok_Whitespace) { } return tok; } + +Tok tokstream_next(TokStream* s) +{ + Tok curr = s->curr; + Tok next = tokstream_next_filtered(s); + + if (curr.ty == Tok_Str && next.ty == Tok_Str) { + char* text + = arena_alloc(s->arena, strlen(curr.text) + strlen(next.text) + 1); + text[0] = '\0'; + strcat(text, curr.text); + strcat(text, next.text); + s->curr = (Tok) { .ty = Tok_Str, text, curr.line, curr.col }; + return tokstream_next(s); + } + + s->curr = next; + return curr; +}