From 4efa8ba0c0b749fa37b39393d3c29ac75314cc21 Mon Sep 17 00:00:00 2001
From: nsfisis <nsfisis@gmail.com>
Date: Thu, 27 Mar 2025 20:22:53 +0900
Subject: add files

---
 tools/native-tokenizer.php | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100755 tools/native-tokenizer.php

(limited to 'tools')

diff --git a/tools/native-tokenizer.php b/tools/native-tokenizer.php
new file mode 100755
index 0000000..f153356
--- /dev/null
+++ b/tools/native-tokenizer.php
@@ -0,0 +1,11 @@
+#!/usr/bin/env php
+<?php
+
+declare(strict_types=1);
+
+$source = file_get_contents('php://stdin');
+assert($source !== false);
+$tokens = PhpToken::tokenize($source, TOKEN_PARSE);
+foreach ($tokens as $token) {
+    echo "{$token->getTokenName()}: {$token->text}", PHP_EOL;
+}
-- 
cgit v1.2.3-70-g09d2