refactor: tokenize() takes std.Io.Reader

2 files changed, 12 insertions, 12 deletions

diff --git a/src/jq/tokenize.zig b/src/jq/tokenize.zig
index 92f96ca..c4201b5 100644
--- a/src/jq/tokenize.zig
+++ b/src/jq/tokenize.zig
@@ -25,18 +25,14 @@ pub const Token = union(TokenKind) {
     }
 };
 
-pub fn tokenize(allocator: std.mem.Allocator, query: []const u8) ![]Token {
+pub fn tokenize(allocator: std.mem.Allocator, reader: *std.Io.Reader) ![]Token {
     var tokens = try std.array_list.Aligned(Token, null).initCapacity(allocator, 16);
 
-    const len = query.len;
-
-    if (len == 0) {
-        return error.UnexpectedEnd;
-    }
-
-    var i: usize = 0;
-    while (i < len) {
-        const c = query[i];
+    while (true) {
+        const c = reader.takeByte() catch |err| switch (err) {
+            error.EndOfStream => break,
+            error.ReadFailed => return error.ReadFailed,
+        };
         switch (c) {
             '.' => try tokens.append(allocator, .dot),
             '[' => try tokens.append(allocator, .bracket_left),
@@ -49,7 +45,10 @@ pub fn tokenize(allocator: std.mem.Allocator, query: []const u8) ![]Token {
                 }
             },
         }
-        i += 1;
+    }
+
+    if (tokens.items.len == 0) {
+        return error.UnexpectedEnd;
     }
 
     try tokens.append(allocator, .end);
diff --git a/src/root.zig b/src/root.zig
index d22a239..ab8fbbe 100644
--- a/src/root.zig
+++ b/src/root.zig
@@ -5,7 +5,8 @@ pub const jv = @import("./jv.zig");
 pub fn run(allocator: std.mem.Allocator, input: []const u8, query: []const u8) ![]const u8 {
     var compile_allocator = std.heap.ArenaAllocator.init(allocator);
     defer compile_allocator.deinit();
-    const tokens = try jq.tokenize(compile_allocator.allocator(), query);
+    var reader = std.Io.Reader.fixed(query);
+    const tokens = try jq.tokenize(compile_allocator.allocator(), &reader);
     const ast = try jq.parse(compile_allocator.allocator(), tokens);
     const instrs = try jq.compile(allocator, compile_allocator.allocator(), ast);
     defer allocator.free(instrs);