diff options
Diffstat (limited to 'src/tokenize.c')
| -rw-r--r-- | src/tokenize.c | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/src/tokenize.c b/src/tokenize.c index 7ba00cd..c897b02 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -19,7 +19,7 @@ static void tokenize_all(Lexer* l) { for (size_t pos = 0; pos < l->src->len; ++pos) { Token* pp_tok = &l->src->data[pos]; TokenKind k = pp_tok->kind; - if (k == TokenKind_whitespace || k == TokenKind_newline) { + if (k == TokenKind_removed || k == TokenKind_whitespace || k == TokenKind_newline) { continue; } Token* tok = tokens_push_new(l->tokens); |