1 files changed, 33 insertions, 0 deletions
diff --git a/src/jq/tokenize.zig b/src/jq/tokenize.zig
new file mode 100644
index 0000000..0823ea1
--- /dev/null
+++ b/src/jq/tokenize.zig
@@ -0,0 +1,33 @@
+const std = @import("std");
+
+pub const TokenizeError = error{
+    UnexpectedEnd,
+};
+
+pub const TokenKind = enum {
+    end,
+    identity,
+};
+
+pub const Token = struct {
+    kind: TokenKind,
+};
+
+pub fn tokenize(allocator: std.mem.Allocator, query: []const u8) ![]Token {
+    var tokens = try std.array_list.Aligned(Token, null).initCapacity(allocator, 16);
+
+    const len = query.len;
+    var i: usize = 0;
+    while (i < len) {
+        const c = query[i];
+        if (c == '.') {
+            try tokens.append(allocator, .{ .kind = .identity });
+        } else {
+            return TokenizeError.UnexpectedEnd;
+        }
+        i += 1;
+    }
+
+    try tokens.append(allocator, .{ .kind = .end });
+    return tokens.toOwnedSlice(allocator);
+}