fix issue where identifier whose name contains underscore cannot be parsed

2 files changed, 22 insertions, 4 deletions

diff --git a/main.c b/main.c
index c8431b1..beaa35b 100644
--- a/main.c
+++ b/main.c
@@ -212,7 +212,7 @@ TOKEN* tokenize(char* src, int len) {
             tok += 1;
         } else if (isalpha(c)) {
             int start = pos;
-            while (isalnum(src[pos])) {
+            while (isalnum(src[pos]) || src[pos] == '_') {
                 pos += 1;
             }
             if (strstr(src + start, "break") == src + start) {
@@ -1379,10 +1379,11 @@ int main() {
     TOKEN* tokens = tokenize(source, source_len);
 
     // for (int i = 0; tokens[i].kind != TK_EOF; i++) {
-    //     for (int j = 0; j < tokens[i].len; j++) {
-    //         putchar(tokens[i].value[j]);
+    //     if (tokens[i].value) {
+    //         printf("%s\n", tokens[i].value);
+    //     } else {
+    //         printf("(%d)\n", tokens[i].kind);
     //     }
-    //     printf("\n");
     // }
 
     PARSER* parser = parser_new(tokens);
diff --git a/tests/025.sh b/tests/025.sh
new file mode 100644
index 0000000..1f673a1
--- /dev/null
+++ b/tests/025.sh
@@ -0,0 +1,17 @@
+set -e
+
+cat <<'EOF' > expected
+123
+EOF
+bash ../../test_diff.sh <<'EOF'
+int printf();
+
+void foo_bar(int hoge_piyo) {
+    printf("%d\n", hoge_piyo);
+}
+
+int main() {
+    foo_bar(123);
+    return 0;
+}
+EOF