refactor: merge contiguous whitespaces into one token

1 files changed, 9 insertions, 4 deletions

diff --git a/preprocess.c b/preprocess.c
index 468ff3e..3dc6926 100644
--- a/preprocess.c
+++ b/preprocess.c
@@ -778,12 +778,17 @@ void pplexer_tokenize_all(PpLexer* ppl) {
                 tok->kind = TokenKind_ident;
             }
         } else if (isspace(c)) {
-            if (c == '\n' || c == '\r') {
-                ++ppl->line;
+            --ppl->pos;
+            int start = ppl->pos;
+            while (isspace((c = ppl->src[ppl->pos]))) {
+                if (c == '\n' || c == '\r') {
+                    ++ppl->line;
+                }
+                ++ppl->pos;
             }
             tok->kind = TokenKind_whitespace;
-            tok->raw.len = 1;
-            tok->raw.data = ppl->src + ppl->pos - tok->raw.len;
+            tok->raw.data = ppl->src + start;
+            tok->raw.len = ppl->pos - start;
         } else {
             tok->kind = TokenKind_other;
             tok->raw.len = 1;