add files

1 files changed, 11 insertions, 0 deletions

diff --git a/tools/native-tokenizer.php b/tools/native-tokenizer.php
new file mode 100755
index 0000000..f153356
--- /dev/null
+++ b/tools/native-tokenizer.php
@@ -0,0 +1,11 @@
+#!/usr/bin/env php
+<?php
+
+declare(strict_types=1);
+
+$source = file_get_contents('php://stdin');
+assert($source !== false);
+$tokens = PhpToken::tokenize($source, TOKEN_PARSE);
+foreach ($tokens as $token) {
+    echo "{$token->getTokenName()}: {$token->text}", PHP_EOL;
+}