refactor: define TokenValue to store semantic value

1 files changed, 44 insertions, 51 deletions

diff --git a/tokenize.c b/tokenize.c
index 283755e..a5a376b 100644
--- a/tokenize.c
+++ b/tokenize.c
@@ -24,9 +24,9 @@ void tokenize_all(Lexer* l) {
         tok->loc = pp_tok->loc;
         if (k == TokenKind_character_constant) {
             tok->kind = TokenKind_literal_int;
-            int ch = pp_tok->raw.data[1];
+            int ch = pp_tok->value.string.data[1];
             if (ch == '\\') {
-                ch = pp_tok->raw.data[2];
+                ch = pp_tok->value.string.data[2];
                 if (ch == 'a') {
                     ch = '\a';
                 } else if (ch == 'b') {
@@ -45,101 +45,94 @@ void tokenize_all(Lexer* l) {
                     ch = '\0';
                 }
             }
-            char* buf = calloc(4, sizeof(char));
-            sprintf(buf, "%d", ch);
-            tok->raw.data = buf;
-            tok->raw.len = strlen(buf);
-        } else if (k == TokenKind_literal_str) {
-            tok->kind = TokenKind_literal_str;
-            tok->raw.data = pp_tok->raw.data + 1;
-            tok->raw.len = pp_tok->raw.len - 2;
+            tok->value.integer = ch;
         } else if (k == TokenKind_ident) {
-            if (string_equals_cstr(&pp_tok->raw, "auto")) {
+            if (string_equals_cstr(&pp_tok->value.string, "auto")) {
                 tok->kind = TokenKind_keyword_auto;
-            } else if (string_equals_cstr(&pp_tok->raw, "break")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "break")) {
                 tok->kind = TokenKind_keyword_break;
-            } else if (string_equals_cstr(&pp_tok->raw, "case")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "case")) {
                 tok->kind = TokenKind_keyword_case;
-            } else if (string_equals_cstr(&pp_tok->raw, "char")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "char")) {
                 tok->kind = TokenKind_keyword_char;
-            } else if (string_equals_cstr(&pp_tok->raw, "const")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "const")) {
                 tok->kind = TokenKind_keyword_const;
-            } else if (string_equals_cstr(&pp_tok->raw, "continue")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "continue")) {
                 tok->kind = TokenKind_keyword_continue;
-            } else if (string_equals_cstr(&pp_tok->raw, "default")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "default")) {
                 tok->kind = TokenKind_keyword_default;
-            } else if (string_equals_cstr(&pp_tok->raw, "do")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "do")) {
                 tok->kind = TokenKind_keyword_do;
-            } else if (string_equals_cstr(&pp_tok->raw, "double")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "double")) {
                 tok->kind = TokenKind_keyword_double;
-            } else if (string_equals_cstr(&pp_tok->raw, "else")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "else")) {
                 tok->kind = TokenKind_keyword_else;
-            } else if (string_equals_cstr(&pp_tok->raw, "enum")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "enum")) {
                 tok->kind = TokenKind_keyword_enum;
-            } else if (string_equals_cstr(&pp_tok->raw, "extern")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "extern")) {
                 tok->kind = TokenKind_keyword_extern;
-            } else if (string_equals_cstr(&pp_tok->raw, "float")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "float")) {
                 tok->kind = TokenKind_keyword_float;
-            } else if (string_equals_cstr(&pp_tok->raw, "for")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "for")) {
                 tok->kind = TokenKind_keyword_for;
-            } else if (string_equals_cstr(&pp_tok->raw, "goto")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "goto")) {
                 tok->kind = TokenKind_keyword_goto;
-            } else if (string_equals_cstr(&pp_tok->raw, "if")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "if")) {
                 tok->kind = TokenKind_keyword_if;
-            } else if (string_equals_cstr(&pp_tok->raw, "inline")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "inline")) {
                 tok->kind = TokenKind_keyword_inline;
-            } else if (string_equals_cstr(&pp_tok->raw, "int")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "int")) {
                 tok->kind = TokenKind_keyword_int;
-            } else if (string_equals_cstr(&pp_tok->raw, "long")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "long")) {
                 tok->kind = TokenKind_keyword_long;
-            } else if (string_equals_cstr(&pp_tok->raw, "register")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "register")) {
                 tok->kind = TokenKind_keyword_register;
-            } else if (string_equals_cstr(&pp_tok->raw, "restrict")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "restrict")) {
                 tok->kind = TokenKind_keyword_restrict;
-            } else if (string_equals_cstr(&pp_tok->raw, "return")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "return")) {
                 tok->kind = TokenKind_keyword_return;
-            } else if (string_equals_cstr(&pp_tok->raw, "short")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "short")) {
                 tok->kind = TokenKind_keyword_short;
-            } else if (string_equals_cstr(&pp_tok->raw, "signed")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "signed")) {
                 tok->kind = TokenKind_keyword_signed;
-            } else if (string_equals_cstr(&pp_tok->raw, "sizeof")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "sizeof")) {
                 tok->kind = TokenKind_keyword_sizeof;
-            } else if (string_equals_cstr(&pp_tok->raw, "static")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "static")) {
                 tok->kind = TokenKind_keyword_static;
-            } else if (string_equals_cstr(&pp_tok->raw, "struct")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "struct")) {
                 tok->kind = TokenKind_keyword_struct;
-            } else if (string_equals_cstr(&pp_tok->raw, "switch")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "switch")) {
                 tok->kind = TokenKind_keyword_switch;
-            } else if (string_equals_cstr(&pp_tok->raw, "typedef")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "typedef")) {
                 tok->kind = TokenKind_keyword_typedef;
-            } else if (string_equals_cstr(&pp_tok->raw, "union")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "union")) {
                 tok->kind = TokenKind_keyword_union;
-            } else if (string_equals_cstr(&pp_tok->raw, "unsigned")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "unsigned")) {
                 tok->kind = TokenKind_keyword_unsigned;
-            } else if (string_equals_cstr(&pp_tok->raw, "void")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "void")) {
                 tok->kind = TokenKind_keyword_void;
-            } else if (string_equals_cstr(&pp_tok->raw, "volatile")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "volatile")) {
                 tok->kind = TokenKind_keyword_volatile;
-            } else if (string_equals_cstr(&pp_tok->raw, "while")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "while")) {
                 tok->kind = TokenKind_keyword_while;
-            } else if (string_equals_cstr(&pp_tok->raw, "_Bool")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "_Bool")) {
                 tok->kind = TokenKind_keyword__Bool;
-            } else if (string_equals_cstr(&pp_tok->raw, "_Complex")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "_Complex")) {
                 tok->kind = TokenKind_keyword__Complex;
-            } else if (string_equals_cstr(&pp_tok->raw, "_Imaginary")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "_Imaginary")) {
                 tok->kind = TokenKind_keyword__Imaginary;
-            } else if (string_equals_cstr(&pp_tok->raw, "va_start")) {
+            } else if (string_equals_cstr(&pp_tok->value.string, "va_start")) {
                 tok->kind = TokenKind_va_start;
-                tok->raw = pp_tok->raw;
+                tok->value = pp_tok->value;
             } else {
                 tok->kind = TokenKind_ident;
-                tok->raw = pp_tok->raw;
+                tok->value = pp_tok->value;
             }
         } else if (k == TokenKind_other) {
             unreachable();
         } else {
             tok->kind = pp_tok->kind;
-            tok->raw = pp_tok->raw;
+            tok->value = pp_tok->value;
         }
     }
 }