aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/native-tokenizer.php
blob: f1533568c3d33f05e339dafa9728d1b0d3205203 (plain)
1
2
3
4
5
6
7
8
9
10
11
#!/usr/bin/env php
<?php

declare(strict_types=1);

$source = file_get_contents('php://stdin');
assert($source !== false);
$tokens = PhpToken::tokenize($source, TOKEN_PARSE);
foreach ($tokens as $token) {
    echo "{$token->getTokenName()}: {$token->text}", PHP_EOL;
}