From 2659ad460ef0c7bd03acd18e500caccef40088c4 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Sun, 30 Aug 2026 13:08:21 +0200 Subject: [PATCH 1/2] Scan directories natively, and drop the cache and lock behind it The optimized source locators build their symbol index by scanning every file of a directory: php_strip_whitespace(), a prefilter regex, PhpFileCleaner::clean() and a symbol regex, each materializing a PHP string. Two thirds of that was clean() alone - a byte-by-byte PHP loop over every file in the analysed tree. Both stages are now native, and with them the cache, the scan lock and the arena records that existed to stop parallel workers repeating the work: PhpFileCleaner::clean 1.321s -> 0.111s (11.9x) whole findSymbols pipeline 1.705s -> 0.445s (3.8x) hash_file('sha256') over the same corpus 0.419s raw file_get_contents 0.225s That third line is why the cache can go. It exists to be validated, and validating it means hashing every file in the tree - which now costs about what scanning them outright does. So when the extension is active and workers are forked, the directory is walked and scanned fresh: no file hashing, no persisted symbol table, no scan lock, no arena record, nothing to go stale. That works because the parent scans for everyone. PreForkDirectorySymbolScanner builds the directory locators in the main process just before it forks, so the children inherit them copy-on-write, and it collects every directory into one scan rather than one per directory (0.32s -> 0.16s; a file two directories both reach is read once). This is the follow-up https://github.com/phpstan/phpstan-src/pull/5846 left open. Its lock could only remove the duplicated symbol extraction - the per-worker FileFinder walk and sha256 of every file ran before the lock was reached, because those hashes were what validated the cache - and the pre-warm it started as was dropped for forcing buildSourceLocator() in the main thread on every parallel run (#5577). Fork-by-default plus a native scan turns that trade into a plain win: the classmap directories that PR measured as 43 of the 44 scans per worker are now scanned once. Measured against the current 2.2.x phar, cold caches, ABBA-interleaved, output byte-identical in every run: slevomat (12 runs) -1.66% user CPU -4.97% wall -101.7 MB peak RSS shipmonk (6 runs) -1.07% user CPU -4.37% wall RSS flat The memory comes free: each worker used to build its own classToFile / functionToFiles / constantToFile, and they now share the parent's copy-on-write. Slevomat's run-to-run stdev also drops from 7.09s to 1.41s - that was the flock queue. Three gates, each of which alone keeps today's behaviour: without the extension the PHP twin and the cache stay; with spawned workers rather than forked ones (Windows, OPcache on) there is no parent to inherit from, so the cache and lock stay in charge; and a run forking a single worker skips the parent scan, which keeps #5577's lazy main thread lazy. Correctness is checked as identical output, not plausible output: two differential tests compare the native implementations against the PHP twins over every PHP file in the repository in both supportsEnums modes - 43,656 symbol-triple and 42,392 cleaner-output checks - plus fixtures for heredocs, attributes vs hash comments, the define()/inDefine string leak, uppercase keywords, inline HTML and high-byte identifiers. Two things they caught that reasoning had got wrong: the prefilter is not just an optimization ($typeConfig always contains `enum`, so on a supportsEnums=false run the cleaner's early return fires on an enum the symbol regex cannot collect and truncates away everything after it), and php_strip_whitespace() deletes comments leaving no separator, so an identifier split by a comment really does reach the cleaner joined back together - comment removal therefore stays its own pass. SymbolFinderInFiles::findSymbolsInFile() also gains the `continue` its constant branch was missing: without it every global constant additionally registered its own namespace prefix - or the empty string - as a class name in the index. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LVTkJADSfAdKoypbfF6nWQ --- src/Parallel/ParallelAnalyser.php | 11 + .../OptimizedDirectorySourceLocator.php | 31 + ...OptimizedDirectorySourceLocatorFactory.php | 143 ++- .../SourceLocator/PhpFileCleaner.php | 2 + .../PreForkDirectorySymbolScanner.php | 110 ++ .../SourceLocator/SymbolFinderInFiles.php | 3 + src/Turbo/TurboExtensionEnabler.php | 2 +- .../SourceLocator/SymbolFinderInFilesTest.php | 45 + .../data/symbol-finder/global-constants.php | 10 + .../symbol-finder/namespaced-constants.php | 11 + turbo-ext/config.w32 | 2 +- turbo-ext/src/PhpFileCleaner.cpp | 49 + turbo-ext/src/SymbolFinderInFiles.cpp | 227 ++++ turbo-ext/src/SymbolScan.h | 1143 +++++++++++++++++ turbo-ext/src/main.cpp | 2 + turbo-ext/src/support.h | 2 + turbo-ext/tests/php-file-cleaner-corpus.php | 151 +++ turbo-ext/tests/smoke.php | 2 + turbo-ext/tests/symbol-finder-corpus.php | 156 +++ 19 files changed, 2097 insertions(+), 5 deletions(-) create mode 100644 src/Reflection/BetterReflection/SourceLocator/PreForkDirectorySymbolScanner.php create mode 100644 tests/PHPStan/Reflection/BetterReflection/SourceLocator/SymbolFinderInFilesTest.php create mode 100644 tests/PHPStan/Reflection/BetterReflection/SourceLocator/data/symbol-finder/global-constants.php create mode 100644 tests/PHPStan/Reflection/BetterReflection/SourceLocator/data/symbol-finder/namespaced-constants.php create mode 100644 turbo-ext/src/PhpFileCleaner.cpp create mode 100644 turbo-ext/src/SymbolFinderInFiles.cpp create mode 100644 turbo-ext/src/SymbolScan.h create mode 100644 turbo-ext/tests/php-file-cleaner-corpus.php create mode 100644 turbo-ext/tests/symbol-finder-corpus.php diff --git a/src/Parallel/ParallelAnalyser.php b/src/Parallel/ParallelAnalyser.php index e3aa2ca36c9..f64d9a69ed1 100644 --- a/src/Parallel/ParallelAnalyser.php +++ b/src/Parallel/ParallelAnalyser.php @@ -15,6 +15,7 @@ use PHPStan\DependencyInjection\AutowiredParameter; use PHPStan\DependencyInjection\AutowiredService; use PHPStan\Process\ProcessHelper; +use PHPStan\Reflection\BetterReflection\SourceLocator\PreForkDirectorySymbolScanner; use React\EventLoop\LoopInterface; use React\Promise\Deferred; use React\Promise\PromiseInterface; @@ -54,6 +55,7 @@ public function __construct( #[AutowiredParameter(ref: '%parallel.buffer%')] private int $decoderBufferSize, private ForkParallelChecker $forkParallelChecker, + private PreForkDirectorySymbolScanner $preForkDirectorySymbolScanner, private WorkerRunner $workerRunner, ) { @@ -216,6 +218,15 @@ public function analyse( $useFork = $this->forkParallelChecker->isSupported(); + if ($useFork && $numberOfProcesses > 1) { + // Build the directory symbol indexes here, in the parent, so the + // children inherit them copy-on-write instead of each scanning the + // same directories (see PreForkDirectorySymbolScanner). With a + // single worker there is nothing to share, and doing it here would + // only take work off the lazy path that worker might never reach. + $this->preForkDirectorySymbolScanner->scanBeforeFork(); + } + for ($i = 0; $i < $numberOfProcesses; $i++) { if (count($jobs) === 0) { break; diff --git a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php index b83b5a35a17..4548fe93a1d 100644 --- a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php +++ b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php @@ -63,10 +63,33 @@ public function __construct( private array $functionToFiles, private array $constantToFile, private ?string $arenaKeyPrefix = null, + private bool $awaitingBatchedScan = false, ) { } + /** + * Fills in the symbol maps of a locator created for a batched scan. + * + * The factory hands these out before the scan that produces their contents + * has run, so that one scan can cover every directory at once + * (see OptimizedDirectorySourceLocatorFactory::flushBatchedScan()). Nothing + * may look a symbol up in between - the maps are empty, so a lookup would + * quietly answer "not found" - which is what the flag guards. + * + * @param array $classToFile + * @param array> $functionToFiles + * @param array $constantToFile + * @internal + */ + public function fillBatchedScan(array $classToFile, array $functionToFiles, array $constantToFile): void + { + $this->classToFile = $classToFile; + $this->functionToFiles = $functionToFiles; + $this->constantToFile = $constantToFile; + $this->awaitingBatchedScan = false; + } + /** * @return array{non-empty-string, string} */ @@ -86,6 +109,10 @@ private function getCacheKeys(string $file, Identifier $identifier): array #[Override] public function locateIdentifier(Reflector $reflector, Identifier $identifier): ?Reflection { + if ($this->awaitingBatchedScan) { + throw new ShouldNotHappenException('Symbols were looked up in a directory whose batched scan has not been flushed yet.'); + } + if ($identifier->isClass()) { $identifierName = strtolower($identifier->getName()); $file = $this->findFileByClass($identifierName); @@ -334,6 +361,10 @@ private function hydrateSymbolsFromArena(): void #[Override] public function locateIdentifiersByType(Reflector $reflector, IdentifierType $identifierType): array { + if ($this->awaitingBatchedScan) { + throw new ShouldNotHappenException('Symbols were looked up in a directory whose batched scan has not been flushed yet.'); + } + $this->hydrateSymbolsFromArena(); $reflections = []; diff --git a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php index ff3429e1b4d..a4287665de8 100644 --- a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php +++ b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php @@ -10,9 +10,12 @@ use PHPStan\File\FileFinder; use PHPStan\Internal\DirectoryCreator; use PHPStan\Internal\DirectoryCreatorException; +use PHPStan\Parallel\ForkParallelChecker; use PHPStan\Php\PhpVersion; +use PHPStan\Turbo\TurboExtensionEnabler; use function array_key_exists; use function array_keys; +use function array_values; use function fclose; use function flock; use function fopen; @@ -50,6 +53,13 @@ final class OptimizedDirectorySourceLocatorFactory */ private const HASH_LOCK_POLL_INTERVAL_MICROSECONDS = 5_000; + /** + * Directories collected for a batched scan, null when not batching. + * + * @var list|null + */ + private ?array $batchedScan = null; + public function __construct( private FileNodesFetcher $fileNodesFetcher, #[AutowiredParameter(ref: '@fileFinderScan')] @@ -58,6 +68,7 @@ public function __construct( private SymbolFinderInFiles $symbolFinderInFiles, private Cache $cache, private FileContentHasher $fileContentHasher, + private ForkParallelChecker $forkParallelChecker, #[AutowiredParameter] private string $tmpDir, ) @@ -66,6 +77,10 @@ public function __construct( public function createByDirectory(string $directory): OptimizedDirectorySourceLocator { + if ($this->scansFresh()) { + return $this->createFreshDirectorySourceLocator($directory); + } + $cacheKey = sprintf('odsl-%s', $directory); $hashesRecordKey = 'odsl-filehashes-' . $directory; @@ -126,6 +141,119 @@ public function createByDirectory(string $directory): OptimizedDirectorySourceLo return $this->createCachedDirectorySourceLocator($fileHashes, $cacheKey); } + /** + * Whether the symbol index is built outright instead of being cached. + * + * Both halves are needed. The native scan is what makes the cache not + * worth its keep, and forking is what keeps the scan from happening once + * per worker: the parent scans before it forks and the children inherit + * the result (see PreForkDirectorySymbolScanner). Where the extension is + * active but workers are spawned rather than forked - Windows, or OPcache + * left on - there is nothing to inherit, so the cache and its scan lock + * stay in charge. + */ + private function scansFresh(): bool + { + return TurboExtensionEnabler::isActive() && $this->forkParallelChecker->isSupported(); + } + + /** + * With the turbo extension the symbol scan is native and costs about what + * hashing the directory to validate a cache costs, so a cache has nothing + * left to save: the directory is walked and scanned outright, with no file + * hashing, no persisted symbol table, no scan lock and no arena record — + * and therefore no cache that can go stale. PreForkDirectoryScanner runs + * this once in the main process before it forks its workers, so every + * worker inherits the finished locators instead of racing to build them. + */ + private function createFreshDirectorySourceLocator(string $directory): OptimizedDirectorySourceLocator + { + return $this->createFreshFileListSourceLocator($this->fileFinder->findFiles([$directory])->getFiles()); + } + + /** + * Starts collecting the directories asked for instead of scanning each one + * as it comes, so that flushBatchedScan() can cover all of them in a single + * scan: a file reachable from two directories is read once rather than + * twice, and the per-call costs are paid once instead of per directory. + */ + public function beginBatchedScan(): void + { + $this->batchedScan = []; + } + + /** + * Scans everything collected since beginBatchedScan() at once and fills in + * the locators handed out in the meantime. + */ + public function flushBatchedScan(): void + { + $batched = $this->batchedScan; + $this->batchedScan = null; + if ($batched === null || $batched === []) { + return; + } + + $allFiles = []; + foreach ($batched as [$files]) { + foreach ($files as $file) { + // a file reachable from two directories is scanned once + $allFiles[$file] = $file; + } + } + + $symbols = $this->symbolFinderInFiles->findSymbols(array_values($allFiles), $this->phpVersion->supportsEnums()); + + foreach ($batched as [$files, $locator]) { + $directorySymbols = []; + foreach ($files as $file) { + if (!array_key_exists($file, $symbols)) { + continue; + } + + $directorySymbols[$file] = $symbols[$file]; + } + + [$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($directorySymbols); + $locator->fillBatchedScan($classToFile, $functionToFiles, $constantToFile); + } + } + + /** + * @param string[] $files + */ + private function createFreshFileListSourceLocator(array $files): OptimizedDirectorySourceLocator + { + if ($this->batchedScan !== null) { + $locator = new OptimizedDirectorySourceLocator( + $this->fileNodesFetcher, + $this->cache, + $this->phpVersion, + $this->fileContentHasher, + [], + [], + [], + awaitingBatchedScan: true, + ); + $this->batchedScan[] = [$files, $locator]; + + return $locator; + } + + $symbols = $this->symbolFinderInFiles->findSymbols($files, $this->phpVersion->supportsEnums()); + [$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($symbols); + + return new OptimizedDirectorySourceLocator( + $this->fileNodesFetcher, + $this->cache, + $this->phpVersion, + $this->fileContentHasher, + $classToFile, + $functionToFiles, + $constantToFile, + ); + } + /** * @param array $fileHashes * @param non-empty-string $cacheKey @@ -233,7 +361,12 @@ private function createCachedDirectorySourceLocator(array $fileHashes, string $c } } - [$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($cached); + $symbols = []; + foreach ($cached as $file => [, $classes, $functions, $constants]) { + $symbols[$file] = [$classes, $functions, $constants]; + } + + [$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($symbols); // Publication order matters: the reader above requires all three // records, so a partially-published index is never consumed. @@ -322,6 +455,10 @@ private function releaseDirectoryScanLock($lockHandle): void */ public function createByFiles(array $files, string $uniqueCacheIdentifier): OptimizedDirectorySourceLocator { + if ($this->scansFresh()) { + return $this->createFreshFileListSourceLocator($files); + } + $fileHashes = []; foreach ($files as $file) { $hash = $this->fileContentHasher->hash($file); @@ -335,7 +472,7 @@ public function createByFiles(array $files, string $uniqueCacheIdentifier): Opti } /** - * @param array $symbols + * @param array $symbols * @return array{array, array>, array} */ private function changeStructure(array $symbols): array @@ -343,7 +480,7 @@ private function changeStructure(array $symbols): array $classToFile = []; $constantToFile = []; $functionToFiles = []; - foreach ($symbols as $file => [, $classes, $functions, $constants]) { + foreach ($symbols as $file => [$classes, $functions, $constants]) { foreach ($classes as $classInFile) { $classToFile[$classInFile] = $file; } diff --git a/src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php b/src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php index e1589495309..e6de2c8156c 100644 --- a/src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php +++ b/src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php @@ -3,6 +3,7 @@ namespace PHPStan\Reflection\BetterReflection\SourceLocator; use PHPStan\DependencyInjection\AutowiredService; +use PHPStan\Turbo\ShadowedByTurboExtension; use function array_keys; use function implode; use function in_array; @@ -18,6 +19,7 @@ * @see https://github.com/composer/composer/pull/10107 */ #[AutowiredService] +#[ShadowedByTurboExtension(turboClass: 'PHPStanTurbo\PhpFileCleaner', implementation: __DIR__ . '/../../../../turbo-ext/src/PhpFileCleaner.cpp')] final class PhpFileCleaner { diff --git a/src/Reflection/BetterReflection/SourceLocator/PreForkDirectorySymbolScanner.php b/src/Reflection/BetterReflection/SourceLocator/PreForkDirectorySymbolScanner.php new file mode 100644 index 00000000000..8d2a3629ec4 --- /dev/null +++ b/src/Reflection/BetterReflection/SourceLocator/PreForkDirectorySymbolScanner.php @@ -0,0 +1,110 @@ +analysedPaths, $this->analysedPathsFromConfig) as $analysedPath) { + if (!is_dir($analysedPath)) { + continue; + } + + $directories[] = $analysedPath; + } + + // Collect every directory first and scan them in one go. A file that + // two directories both reach is then read once instead of twice, and + // the scan pays its per-call costs once instead of per directory: + // measured over this repository's tree, 0.32s -> 0.16s. + $this->optimizedDirectorySourceLocatorFactory->beginBatchedScan(); + + try { + foreach (array_unique(array_merge($directories, $this->scanDirectories)) as $directory) { + $this->optimizedDirectorySourceLocatorRepository->getOrCreate($directory); + } + + foreach ($this->composerAutoloaderProjectPaths as $composerAutoloaderProjectPath) { + // the aggregate locator is thrown away - what matters is that the + // directory locators it builds land in the repository's memo, + // which the forked children inherit + $this->composerJsonAndInstalledJsonSourceLocatorMaker->create($composerAutoloaderProjectPath); + } + + $this->optimizedDirectorySourceLocatorFactory->flushBatchedScan(); + } finally { + // a throw must not leave the factory collecting into a batch that + // nobody will flush + $this->optimizedDirectorySourceLocatorFactory->flushBatchedScan(); + } + } + +} diff --git a/src/Reflection/BetterReflection/SourceLocator/SymbolFinderInFiles.php b/src/Reflection/BetterReflection/SourceLocator/SymbolFinderInFiles.php index e71b2b6aec3..b2bda25d4d0 100644 --- a/src/Reflection/BetterReflection/SourceLocator/SymbolFinderInFiles.php +++ b/src/Reflection/BetterReflection/SourceLocator/SymbolFinderInFiles.php @@ -3,6 +3,7 @@ namespace PHPStan\Reflection\BetterReflection\SourceLocator; use PHPStan\DependencyInjection\AutowiredService; +use PHPStan\Turbo\ShadowedByTurboExtension; use function array_filter; use function array_slice; use function count; @@ -19,6 +20,7 @@ use function strtolower; #[AutowiredService] +#[ShadowedByTurboExtension(turboClass: 'PHPStanTurbo\SymbolFinderInFiles', implementation: __DIR__ . '/../../../../turbo-ext/src/SymbolFinderInFiles.cpp')] final class SymbolFinderInFiles { @@ -91,6 +93,7 @@ private function findSymbolsInFile(string $file, bool $supportsEnums): array if ($matches['constant'][$i] !== '') { $constants[] = self::normalizeConstantName(ltrim($namespace . $matches['cname'][$i], '\\')); + continue; } if ($matches['define'][$i] !== '') { diff --git a/src/Turbo/TurboExtensionEnabler.php b/src/Turbo/TurboExtensionEnabler.php index 4db40afeb9b..4021947784d 100644 --- a/src/Turbo/TurboExtensionEnabler.php +++ b/src/Turbo/TurboExtensionEnabler.php @@ -22,7 +22,7 @@ final class TurboExtensionEnabler * version is the short SHA of the last commit touching turbo-ext/src/, * enforced by the phar.yml turbo-version job. */ - public const EXPECTED_EXTENSION_VERSION = '873ede9'; + public const EXPECTED_EXTENSION_VERSION = 'f531361'; private static bool $typeCombinatorCacheEnabled = false; diff --git a/tests/PHPStan/Reflection/BetterReflection/SourceLocator/SymbolFinderInFilesTest.php b/tests/PHPStan/Reflection/BetterReflection/SourceLocator/SymbolFinderInFilesTest.php new file mode 100644 index 00000000000..7667289b78b --- /dev/null +++ b/tests/PHPStan/Reflection/BetterReflection/SourceLocator/SymbolFinderInFilesTest.php @@ -0,0 +1,45 @@ + + */ + public static function dataFindSymbols(): iterable + { + yield 'namespaced constants do not leak a class entry' => [ + __DIR__ . '/data/symbol-finder/namespaced-constants.php', + [ + ['symbolfindertest\namespaced\thing'], + [], + ['symbolfindertest\namespaced\ALPHA', 'symbolfindertest\namespaced\BETA'], + ], + ]; + + yield 'global constants and defines' => [ + __DIR__ . '/data/symbol-finder/global-constants.php', + [ + [], + ['symbolfindertestfunction'], + ['GLOBAL_ALPHA', 'symbolfindertest\DEFINED'], + ], + ]; + } + + /** + * @param array{string[], string[], string[]} $expected + */ + #[DataProvider('dataFindSymbols')] + public function testFindSymbols(string $file, array $expected): void + { + $finder = new SymbolFinderInFiles(new PhpFileCleaner()); + $this->assertSame([$file => $expected], $finder->findSymbols([$file], true)); + } + +} diff --git a/tests/PHPStan/Reflection/BetterReflection/SourceLocator/data/symbol-finder/global-constants.php b/tests/PHPStan/Reflection/BetterReflection/SourceLocator/data/symbol-finder/global-constants.php new file mode 100644 index 00000000000..617a86932e5 --- /dev/null +++ b/tests/PHPStan/Reflection/BetterReflection/SourceLocator/data/symbol-finder/global-constants.php @@ -0,0 +1,10 @@ + +#include +#else +#include +#include +#include +#endif + +static zend_class_entry *pt_ce_symbol_finder = nullptr; + +namespace phpstanturbo { + +/* Mirrors PHPStan\...\SymbolFinderInFiles. The buffers live for the whole + * findSymbols() call so a directory of thousands of files reuses one pair of + * allocations. */ +class SymbolFinderInFiles +{ +public: + /* files far above this are not worth keeping the read buffer for */ + static constexpr size_t BUFFER_RETENTION_LIMIT = 4 * 1024 * 1024; + + zv::Val findSymbols(HashTable *files, bool supportsEnums); + +private: + std::string source; + std::string stripped; + std::string cleaned; + Symbols symbols; + + bool readFile(const char *path, size_t pathLen); + void scan(bool supportsEnums); + static void symbolsToArray(const Symbols &symbols, zval *out); +}; + +/* + * The twin reaches the file through php_strip_whitespace(), which goes past + * the stream wrappers; the locators only ever pass real paths from their own + * directory walk, so a plain open() is enough — and an unreadable file has to + * behave like the twin's suppressed warning, i.e. produce no symbols. + */ +bool SymbolFinderInFiles::readFile(const char *path, size_t pathLen) +{ + source.clear(); + + if (pathLen == 0 || memchr(path, '\0', pathLen) != NULL) { + return false; + } + +#ifdef PHP_WIN32 + int fd = _open(path, _O_RDONLY | _O_BINARY); +#else + int fd = open(path, O_RDONLY); +#endif + if (fd < 0) { + return false; + } + + char chunk[65536]; + for (;;) { +#ifdef PHP_WIN32 + int got = _read(fd, chunk, sizeof(chunk)); +#else + ssize_t got = read(fd, chunk, sizeof(chunk)); +#endif + if (got < 0) { +#ifdef PHP_WIN32 + _close(fd); +#else + close(fd); +#endif + source.clear(); + return false; + } + if (got == 0) { + break; + } + source.append(chunk, (size_t) got); + } + +#ifdef PHP_WIN32 + _close(fd); +#else + close(fd); +#endif + + return true; +} + +void SymbolFinderInFiles::scan(bool supportsEnums) +{ + symbols.clear(); + + if (source.empty()) { + return; + } + + CommentStripper stripper(source.data(), source.size(), shortOpenTagEnabled()); + stripper.strip(stripped); + + if (stripped.empty()) { + return; + } + + size_t matches = prefilterCount(stripped.data(), stripped.size(), supportsEnums); + if (matches == 0) { + return; + } + + PhpFileCleaner cleaner(stripped.data(), stripped.size()); + cleaner.clean((zend_long) matches, cleaned); + + SymbolMatcher matcher(cleaned.data(), cleaned.size(), supportsEnums); + matcher.match(symbols); +} + +void SymbolFinderInFiles::symbolsToArray(const Symbols &symbols, zval *out) +{ + zval &triple = *out; + array_init_size(&triple, 3); + + const std::vector *groups[3] = { &symbols.classes, &symbols.functions, &symbols.constants }; + for (const std::vector *group : groups) { + zval list; + array_init_size(&list, (uint32_t) group->size()); + for (const std::string &name : *group) { + zval item; + ZVAL_STRINGL(&item, name.data(), name.size()); + zend_hash_next_index_insert_new(Z_ARRVAL(list), &item); + } + zend_hash_next_index_insert_new(Z_ARRVAL(triple), &list); + } +} + +zv::Val SymbolFinderInFiles::findSymbols(HashTable *files, bool supportsEnums) +{ + zval result; + array_init_size(&result, zend_hash_num_elements(files)); + + for (zv::ArrayEntry file : zv::TableRef(files)) { + zv::Ref value = file.value().deref(); + if (!value.isString()) { + continue; + } + + zend_string *path = value.asString(); + if (readFile(ZSTR_VAL(path), ZSTR_LEN(path))) { + scan(supportsEnums); + } else { + symbols.clear(); + } + + zval triple; + symbolsToArray(symbols, &triple); + zend_hash_update(Z_ARRVAL(result), path, &triple); + + if (source.capacity() > BUFFER_RETENTION_LIMIT) { + std::string().swap(source); + } + } + + return zv::Val::adopt(result); +} + +} // namespace phpstanturbo + +/* {{{ registration */ + +#include "reg.h" + +#define CLEANER_CLASS "PHPStan\\Reflection\\BetterReflection\\SourceLocator\\PhpFileCleaner" + +void pt_register_symbol_finder_in_files() +{ + reg::Class cls("PHPStanTurbo\\SymbolFinderInFiles"); + + /* the arginfo has to keep the real parameter class name: Nette reflects + * this constructor while compiling the container (rule 6) */ + cls.method("__construct", reg::Public, 1, { reg::obj("cleaner", CLEANER_CLASS) }, [](INTERNAL_FUNCTION_PARAMETERS) { + zval *cleaner; + ZEND_PARSE_PARAMETERS_START(1, 1) + Z_PARAM_OBJECT(cleaner) + ZEND_PARSE_PARAMETERS_END(); + (void) cleaner; + }); + + cls.method("findSymbols", reg::Public, 2, { reg::arrayArg("files"), reg::boolArg("supportsEnums") }, [](INTERNAL_FUNCTION_PARAMETERS) { + HashTable *files; + bool supportsEnums; + ZEND_PARSE_PARAMETERS_START(2, 2) + Z_PARAM_ARRAY_HT(files) + Z_PARAM_BOOL(supportsEnums) + ZEND_PARSE_PARAMETERS_END(); + + phpstanturbo::SymbolFinderInFiles finder; + finder.findSymbols(files, supportsEnums).intoReturnValue(return_value); + }); + + /* not final: a PHP stub subclass may extend this class */ + pt_ce_symbol_finder = cls.register_(); +} + +/* }}} */ diff --git a/turbo-ext/src/SymbolScan.h b/turbo-ext/src/SymbolScan.h new file mode 100644 index 00000000000..15eff332ec2 --- /dev/null +++ b/turbo-ext/src/SymbolScan.h @@ -0,0 +1,1143 @@ +/* + * Shared scanning primitives behind the optimized source locators' directory + * symbol scan — the pipeline PHPStan runs per file: + * + * php_strip_whitespace() -> clean() -> symbol regex + * + * All three stages live here natively so SymbolFinderInFiles can run them + * back to back over one reusable pair of buffers, while PhpFileCleaner.cpp + * still exposes the middle stage on its own as the shadow of the PHP twin. + * + * The stages stay separate passes on purpose. php_strip_whitespace() deletes + * comments without leaving a separator, so an identifier split by a comment + * really does reach the cleaner joined back together — fusing comment removal + * into the cleaner would lose that join, and with it the parity the port is + * judged on. + */ + +#ifndef PHPSTANTURBO_SYMBOLSCAN_H +#define PHPSTANTURBO_SYMBOLSCAN_H + +#include "support.h" + +#include +#include + +/* The twin's $rejectChars: '{}?"\'= 0x80 are not word bytes */ +inline bool isWordByte(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* PCRE's \s */ +inline bool isSpaceByte(unsigned char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +/* [a-zA-Z_\x7f-\xff] */ +inline bool isNameStart(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c >= 0x7f; +} + +/* [a-zA-Z0-9_\x7f-\xff\-] — the dash is in the class, odd as it looks */ +inline bool isNameByte(unsigned char c) +{ + return isNameStart(c) || (c >= '0' && c <= '9') || c == '-'; +} + +/* [a-zA-Z_\x80-\xff] / [a-zA-Z0-9_\x80-\xff] — heredoc labels start at + * \x80, not \x7f, in the twin's patterns */ +inline bool isLabelStart(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c >= 0x80; +} + +inline bool isLabelByte(unsigned char c) +{ + return isLabelStart(c) || (c >= '0' && c <= '9'); +} + +inline bool equalsIgnoreCase(const char *a, const char *lowercaseB, size_t n) +{ + for (size_t i = 0; i < n; i++) { + char c = a[i]; + if (c >= 'A' && c <= 'Z') { + c = (char) (c - 'A' + 'a'); + } + if (c != lowercaseB[i]) { + return false; + } + } + return true; +} + +/* the define()/namespace name class: like isNameByte but without the dash */ +inline bool isDefineNameByte(unsigned char c) +{ + return isNameStart(c) || (c >= '0' && c <= '9'); +} + +/* whether a bare `])` anchored one byte before `at`: the byte before the + * keyword must exist, must not be a word byte (that is the \b, since the + * keyword starts with one) and must not be $, : or >. */ + bool prevByteOpensKeyword(size_t at) const + { + if (at == 0 || at > len) { + return false; + } + unsigned char prev = (unsigned char) contents[at - 1]; + return !isWordByte(prev) && prev != '$' && prev != ':' && prev != '>'; + } + + bool peek(char c) const { return index + 1 < len && contents[index + 1] == c; } + + /* `\s++[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff\-]*+` starting at `from`; + * on success `end` receives the offset just past the name. */ + bool matchSpacesAndName(size_t from, size_t *end) const + { + size_t p = from; + while (p < len && isSpaceByte((unsigned char) contents[p])) { + p++; + } + if (p == from || p >= len || !isNameStart((unsigned char) contents[p])) { + return false; + } + p++; + while (p < len && isNameByte((unsigned char) contents[p])) { + p++; + } + *end = p; + return true; + } + + void skipToPhp(); + void skipString(char delimiter); + void consumeString(char delimiter, std::string &clean); + void skipComment(); + void skipToNewline(); + bool matchHeredocStart(size_t *labelStart, size_t *labelLen, size_t *end) const; + void skipHeredoc(const char *label, size_t labelLen); +}; + +inline void PhpFileCleaner::skipToPhp() +{ + while (index < len) { + if (contents[index] == '<' && peek('?')) { + index += 2; + break; + } + + index += 1; + } +} + +/* The twin's consumeString(): copies the string body verbatim, keeping + * backslash escapes, up to and including the closing delimiter. */ +inline void PhpFileCleaner::consumeString(char delimiter, std::string &clean) +{ + index += 1; + while (index < len) { + if (contents[index] == '\\' && (peek('\\') || peek(delimiter))) { + clean.append(contents + index, 2); + index += 2; + continue; + } + + if (contents[index] == delimiter) { + clean.push_back(delimiter); + index += 1; + break; + } + + clean.push_back(contents[index]); + index += 1; + } +} + +inline void PhpFileCleaner::skipString(char delimiter) +{ + index += 1; + while (index < len) { + while (index < len && contents[index] != '\\' && contents[index] != delimiter) { + index++; + } + if (index >= len) { + break; + } + if (contents[index] == '\\' && (peek('\\') || peek(delimiter))) { + index += 2; + continue; + } + if (contents[index] == delimiter) { + index += 1; + break; + } + index += 1; + } +} + +inline void PhpFileCleaner::skipComment() +{ + index += 2; + while (index < len) { + while (index < len && contents[index] != '*') { + index++; + } + + if (peek('/')) { + index += 2; + break; + } + + index += 1; + } +} + +inline void PhpFileCleaner::skipToNewline() +{ + while (index < len && contents[index] != '\r' && contents[index] != '\n') { + index++; + } +} + +/* `{<<<[ \t]*+(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*+)\1(?:\r\n|\n|\r)}A` */ +inline bool PhpFileCleaner::matchHeredocStart(size_t *labelStart, size_t *labelLen, size_t *end) const +{ + size_t p = index; + if (p + 3 > len || contents[p] != '<' || contents[p + 1] != '<' || contents[p + 2] != '<') { + return false; + } + p += 3; + while (p < len && (contents[p] == ' ' || contents[p] == '\t')) { + p++; + } + char quote = '\0'; + if (p < len && (contents[p] == '\'' || contents[p] == '"')) { + quote = contents[p]; + p++; + } + if (p >= len || !isLabelStart((unsigned char) contents[p])) { + return false; + } + size_t start = p; + p++; + while (p < len && isLabelByte((unsigned char) contents[p])) { + p++; + } + *labelStart = start; + *labelLen = p - start; + if (quote != '\0') { + if (p >= len || contents[p] != quote) { + return false; + } + p++; + } + if (p < len && contents[p] == '\r') { + p += (p + 1 < len && contents[p + 1] == '\n') ? 2 : 1; + } else if (p < len && contents[p] == '\n') { + p += 1; + } else { + return false; + } + *end = p; + return true; +} + +inline void PhpFileCleaner::skipHeredoc(const char *label, size_t labelLen) +{ + char firstLabelByte = label[0]; + + while (index < len) { + /* the label may be preceded by indentation */ + char c = contents[index]; + if (c == '\t' || c == ' ') { + index += 1; + continue; + } + if (c == firstLabelByte + && index + labelLen <= len + && memcmp(contents + index, label, labelLen) == 0 + && (index + labelLen >= len || !isLabelByte((unsigned char) contents[index + labelLen])) + ) { + index += labelLen; + return; + } + + skipToNewline(); + while (index < len && (contents[index] == '\r' || contents[index] == '\n')) { + index++; + } + } +} + +inline void PhpFileCleaner::clean(zend_long maxMatches, std::string &out) +{ + /* keyed by first byte, exactly like the twin's $typeConfig */ + struct TypeConfig { + char firstByte; + const char *name; + size_t length; + }; + static const TypeConfig types[] = { + { 'c', "class", 5 }, + { 'i', "interface", 9 }, + { 't', "trait", 5 }, + { 'e', "enum", 4 }, + }; + + std::string &clean = out; + clean.clear(); + clean.reserve(len); + + bool inType = false; + zend_long typeLevel = 0; + bool inDefine = false; + + while (index < len) { + skipToPhp(); + clean.append("')) { + clean.append("?>", 2); + index += 2; + break; /* continue 2 */ + } + + if (c == '"' || c == '\'') { + if (inDefine) { + clean.push_back(c); + consumeString(c, clean); + inDefine = false; + } else { + skipString(c); + clean.append("null", 4); + } + + continue; + } + + if (c == '{') { + if (inType) { + typeLevel++; + } + + clean.push_back(c); + index++; + continue; + } + + if (c == '}') { + if (inType) { + typeLevel--; + + if (typeLevel == 0) { + inType = false; + } + } + + clean.push_back(c); + index++; + continue; + } + + if (c == '<' && peek('<')) { + size_t labelStart, labelLen, end; + if (matchHeredocStart(&labelStart, &labelLen, &end)) { + const char *label = contents + labelStart; + index = end; + skipHeredoc(label, labelLen); + clean.append("null", 4); + continue; + } + } + + if (c == '/') { + if (peek('/')) { + skipToNewline(); + continue; + } + if (peek('*')) { + skipComment(); + continue; + } + } + + /* `~.\b(?])const(\s++NAME)~Ais` at index - 1 */ + if (inType && c == 'c' && prevByteOpensKeyword(index) && index + 5 <= len + && equalsIgnoreCase(contents + index, "const", 5) + ) { + size_t end; + if (matchSpacesAndName(index + 5, &end)) { + /* invalid PHP, but it only has to stop the symbol regex + * from reading a class constant as a global one */ + clean.append("class_const", 11); + clean.append(contents + index + 5, end - (index + 5)); + index = end; + continue; + } + } + + /* `~.\b(?])define\s*+\(~Ais` at index - 1 */ + if (c == 'd' && prevByteOpensKeyword(index) && index + 6 <= len + && equalsIgnoreCase(contents + index, "define", 6) + ) { + size_t p = index + 6; + while (p < len && isSpaceByte((unsigned char) contents[p])) { + p++; + } + if (p < len && contents[p] == '(') { + /* the twin appends the whole match, which starts one byte + * before the keyword — that byte is already in the output, + * so it lands twice. Harmless for the symbol regex, and + * reproduced here to keep the output byte-identical. */ + clean.append(contents + index - 1, p + 1 - (index - 1)); + index = p + 1; + inDefine = true; + continue; + } + } + + for (const TypeConfig &type : types) { + if (type.firstByte != c) { + continue; + } + + if (index + type.length <= len && memcmp(contents + index, type.name, type.length) == 0) { + if (maxMatches == 1 && prevByteOpensKeyword(index)) { + size_t end; + if (matchSpacesAndName(index + type.length, &end)) { + clean.append(contents + index - 1, end - (index - 1)); + return; + } + } + + inType = true; + } + + break; + } + + index += 1; + size_t skipFrom = index; + while (index < len && !pt_reject_table.bytes[(unsigned char) contents[index]]) { + index++; + } + if (index > skipFrom) { + clean.push_back(c); + clean.append(contents + skipFrom, index - skipFrom); + } else { + clean.push_back(c); + } + } + } + +} + + + +/* {{{ stage 1 — php_strip_whitespace() equivalent */ + +/* Bytes that can start a construct the stripper must understand. */ +static const struct StripTable { + bool bytes[256]; + + StripTable() : bytes() + { + for (const char *p = "?/#'\"`<"; *p != '\0'; p++) { + bytes[(unsigned char) *p] = true; + } + } +} pt_strip_table; + +/* + * Removes comments the way php_strip_whitespace() does — emitting nothing in + * their place, so the bytes around them become adjacent — while copying + * everything else through verbatim. The twin's stripper also collapses each + * whitespace run to a single space; that is deliberately not reproduced, + * because every consumer downstream matches whitespace with \s+ or \s* and + * cannot tell the difference, and copying spans verbatim is faster. + * + * Unlike the cleaner, this stage has to know real lexer rules: # comments + * (but not #[ attributes), line comments ending at ?>, and backtick strings. + */ +class CommentStripper +{ +public: + CommentStripper(const char *contents, size_t len, bool shortOpenTag) + : contents(contents), len(len), index(0), shortOpenTag(shortOpenTag) {} + + void strip(std::string &out); + +private: + const char *contents; + size_t len; + size_t index; + bool shortOpenTag; + + /* length of the open tag at `at`, or 0 if there is none */ + size_t openTagLength(size_t at) const + { + if (at + 1 >= len || contents[at] != '<' || contents[at + 1] != '?') { + return 0; + } + if (at + 4 < len && equalsIgnoreCase(contents + at + 2, "php", 3) + && (at + 5 >= len || isSpaceByte((unsigned char) contents[at + 5])) + ) { + return 5; + } + if (at + 2 < len && contents[at + 2] == '=') { + return 3; + } + + return shortOpenTag ? 2 : 0; + } + + /* a // or # comment: ends at a newline (left in place, it is whitespace) + * or at ?>, which the caller then handles as the close tag it is */ + void skipLineComment() + { + while (index < len) { + char c = contents[index]; + if (c == '\n' || c == '\r') { + return; + } + if (c == '?' && index + 1 < len && contents[index + 1] == '>') { + return; + } + index++; + } + } + + void skipBlockComment() + { + index += 2; + while (index + 1 < len) { + if (contents[index] == '*' && contents[index + 1] == '/') { + index += 2; + return; + } + index++; + } + index = len; + } + + /* copies a quoted string verbatim; a backslash escapes the next byte, + * which finds the same closing quote as PHP's own rules do */ + void copyString(char delimiter, std::string &out) + { + size_t start = index; + index++; + while (index < len) { + char c = contents[index]; + if (c == '\\' && index + 1 < len) { + index += 2; + continue; + } + index++; + if (c == delimiter) { + break; + } + } + out.append(contents + start, index - start); + } + + void copyHeredoc(std::string &out); +}; + +inline void CommentStripper::copyHeredoc(std::string &out) +{ + size_t p = index + 3; + while (p < len && (contents[p] == ' ' || contents[p] == '\t')) { + p++; + } + char quote = '\0'; + if (p < len && (contents[p] == '\'' || contents[p] == '"')) { + quote = contents[p]; + p++; + } + if (p >= len || !isLabelStart((unsigned char) contents[p])) { + /* not a heredoc after all — let the caller copy the bytes */ + return; + } + size_t labelStart = p; + p++; + while (p < len && isLabelByte((unsigned char) contents[p])) { + p++; + } + size_t labelLen = p - labelStart; + if (quote != '\0') { + if (p >= len || contents[p] != quote) { + return; + } + p++; + } + if (p < len && contents[p] == '\r') { + p += (p + 1 < len && contents[p + 1] == '\n') ? 2 : 1; + } else if (p < len && contents[p] == '\n') { + p += 1; + } else { + return; + } + + size_t bodyStart = p; + const char *label = contents + labelStart; + while (p < len) { + char c = contents[p]; + if (c == '\t' || c == ' ') { + p++; + continue; + } + if (c == label[0] + && p + labelLen <= len + && memcmp(contents + p, label, labelLen) == 0 + && (p + labelLen >= len || !isLabelByte((unsigned char) contents[p + labelLen])) + ) { + p += labelLen; + break; + } + while (p < len && contents[p] != '\r' && contents[p] != '\n') { + p++; + } + while (p < len && (contents[p] == '\r' || contents[p] == '\n')) { + p++; + } + } + (void) bodyStart; + + out.append(contents + index, p - index); + index = p; +} + +inline void CommentStripper::strip(std::string &out) +{ + out.clear(); + out.reserve(len); + + while (index < len) { + /* inline HTML up to the next opening tag, copied verbatim */ + size_t htmlStart = index; + size_t tagLength = 0; + while (index < len) { + tagLength = openTagLength(index); + if (tagLength != 0) { + break; + } + index++; + } + out.append(contents + htmlStart, index - htmlStart); + if (index >= len) { + return; + } + out.append(contents + index, tagLength); + index += tagLength; + + while (index < len) { + char c = contents[index]; + + if (c == '?' && index + 1 < len && contents[index + 1] == '>') { + out.append("?>", 2); + index += 2; + break; + } + + if (c == '/' && index + 1 < len && contents[index + 1] == '/') { + skipLineComment(); + continue; + } + + if (c == '#') { + if (index + 1 < len && contents[index + 1] == '[') { + out.append("#[", 2); + index += 2; + continue; + } + skipLineComment(); + continue; + } + + if (c == '/' && index + 1 < len && contents[index + 1] == '*') { + skipBlockComment(); + continue; + } + + if (c == '\'' || c == '"' || c == '`') { + copyString(c, out); + continue; + } + + if (c == '<' && index + 2 < len && contents[index + 1] == '<' && contents[index + 2] == '<') { + size_t before = index; + copyHeredoc(out); + if (index != before) { + continue; + } + } + + size_t start = index; + index++; + while (index < len && !pt_strip_table.bytes[(unsigned char) contents[index]]) { + index++; + } + out.append(contents + start, index - start); + } + } +} + +/* }}} */ + + +/* {{{ stage 2a — the prefilter count */ + +/* + * The twin's prefilter, `{\b(?:(?:class|interface|trait|const|function|enum)\s) + * |(?:define\s*\()}i`, whose match count it hands to the cleaner as + * maxMatches. Only "is it exactly one" is ever asked (that is what arms the + * cleaner's early return) and zero means the twin returns no symbols at all, + * so counting stops at two. + * + * It cannot be skipped even though the full scan finds the same declarations: + * $typeConfig always contains `enum`, so on a supportsEnums=false run the + * early return can fire on an enum the symbol regex has no branch for and + * truncate away a function or constant that would otherwise be found. + * + * Note the pattern's shape: the \b applies to the keyword branch only, and + * the keyword must be followed by whitespace, both unlike the symbol regex. + */ +inline size_t prefilterCount(const char *contents, size_t len, bool supportsEnums) +{ + static const char *const keywords[] = { "class", "interface", "trait", "const", "function", "enum" }; + static const size_t keywordLengths[] = { 5, 9, 5, 5, 8, 4 }; + const size_t keywordCount = supportsEnums ? 6 : 5; + + size_t count = 0; + size_t i = 0; + while (i < len && count < 2) { + unsigned char c = (unsigned char) contents[i]; + if (!pt_keyword_start_table.bytes[c]) { + i++; + continue; + } + + if (i == 0 || !isWordByte((unsigned char) contents[i - 1])) { + bool matched = false; + for (size_t k = 0; k < keywordCount; k++) { + size_t length = keywordLengths[k]; + if (i + length < len + && equalsIgnoreCase(contents + i, keywords[k], length) + && isSpaceByte((unsigned char) contents[i + length]) + ) { + count++; + i += length + 1; + matched = true; + break; + } + } + if (matched) { + continue; + } + } + + /* the define branch carries no \b — `mydefine(` counts too */ + if ((c == 'd' || c == 'D') && i + 6 <= len && equalsIgnoreCase(contents + i, "define", 6)) { + size_t p = i + 6; + while (p < len && isSpaceByte((unsigned char) contents[p])) { + p++; + } + if (p < len && contents[p] == '(') { + count++; + i = p + 1; + continue; + } + } + + i++; + } + + return count; +} + +/* }}} */ + +/* {{{ stage 3 — the symbol regex */ + +struct Symbols { + std::vector classes; + std::vector functions; + std::vector constants; + + void clear() + { + classes.clear(); + functions.clear(); + constants.clear(); + } +}; + +/* + * The preg_match_all() over the cleaned text plus the loop that turns its + * captures into symbol names. The pattern is one alternation of five + * branches, all sharing a `\b(?])` prefix, so the walk only has to + * try a branch where that guard holds and the byte can start a keyword. + */ +class SymbolMatcher +{ +public: + SymbolMatcher(const char *contents, size_t len, bool supportsEnums) + : contents(contents), len(len), supportsEnums(supportsEnums) {} + + void match(Symbols &out); + +private: + const char *contents; + size_t len; + bool supportsEnums; + std::string currentNamespace; + + bool guard(size_t at) const + { + if (at == 0) { + return true; + } + unsigned char prev = (unsigned char) contents[at - 1]; + return !isWordByte(prev) && prev != '$' && prev != ':' && prev != '>'; + } + + bool keyword(size_t at, const char *lowercase, size_t length) const + { + return at + length <= len && equalsIgnoreCase(contents + at, lowercase, length) + && (at + length >= len || !isWordByte((unsigned char) contents[at + length])); + } + + size_t skipSpaces(size_t at) const + { + while (at < len && isSpaceByte((unsigned char) contents[at])) { + at++; + } + return at; + } + + /* [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff\-]*+ */ + size_t readName(size_t at) const + { + if (at >= len || !isNameStart((unsigned char) contents[at])) { + return 0; + } + size_t end = at + 1; + while (end < len && isNameByte((unsigned char) contents[end])) { + end++; + } + return end; + } + + /* the define() name: identifiers joined by one or two backslashes */ + size_t readDefineName(size_t at) const + { + if (at >= len || !isNameStart((unsigned char) contents[at])) { + return 0; + } + size_t end = at + 1; + while (end < len && isDefineNameByte((unsigned char) contents[end])) { + end++; + } + for (;;) { + size_t p = end; + size_t slashes = 0; + while (p < len && contents[p] == '\\' && slashes < 2) { + p++; + slashes++; + } + if (slashes == 0 || p >= len || !isNameStart((unsigned char) contents[p])) { + break; + } + p++; + while (p < len && isDefineNameByte((unsigned char) contents[p])) { + p++; + } + end = p; + } + return end; + } + + static void appendLowercase(std::string &out, const char *from, size_t length) + { + for (size_t i = 0; i < length; i++) { + char c = from[i]; + out.push_back(c >= 'A' && c <= 'Z' ? (char) (c - 'A' + 'a') : c); + } + } + + /* strtolower(ltrim($namespace . $name, '\\')) */ + std::string qualified(const char *name, size_t nameLen) const + { + std::string full = currentNamespace; + full.append(name, nameLen); + size_t start = 0; + while (start < full.size() && full[start] == '\\') { + start++; + } + std::string result; + result.reserve(full.size() - start); + appendLowercase(result, full.data() + start, full.size() - start); + return result; + } + + /* self::normalizeConstantName(): the namespace part lowercases, the + * constant's own name keeps its case */ + static std::string normalizeConstantName(const std::string &name) + { + if (name.find('\\') == std::string::npos) { + return name; + } + + std::vector parts; + size_t start = 0; + for (size_t i = 0; i <= name.size(); i++) { + if (i == name.size() || name[i] == '\\') { + if (i > start) { + parts.emplace_back(name, start, i - start); + } + start = i + 1; + } + } + if (parts.empty()) { + return std::string("\\"); + } + + std::string result; + for (size_t i = 0; i + 1 < parts.size(); i++) { + if (i > 0) { + result.push_back('\\'); + } + appendLowercase(result, parts[i].data(), parts[i].size()); + } + result.push_back('\\'); + result.append(parts.back()); + + return result; + } + + /* ltrim($namespace . $name, '\\') without the lowercasing */ + std::string qualifiedConstant(const char *name, size_t nameLen) const + { + std::string full = currentNamespace; + full.append(name, nameLen); + size_t start = 0; + while (start < full.size() && full[start] == '\\') { + start++; + } + return full.substr(start); + } +}; + +inline void SymbolMatcher::match(Symbols &out) +{ + currentNamespace.clear(); + + size_t i = 0; + while (i < len) { + unsigned char c = (unsigned char) contents[i]; + if (!pt_keyword_start_table.bytes[c] || !guard(i)) { + i++; + continue; + } + + /* class|interface|trait[|enum] \s++ NAME */ + static const char *const typeNames[] = { "class", "interface", "trait", "enum" }; + static const size_t typeLengths[] = { 5, 9, 5, 4 }; + bool matched = false; + for (size_t t = 0; t < 4; t++) { + if (t == 3 && !supportsEnums) { + break; + } + if (!keyword(i, typeNames[t], typeLengths[t])) { + continue; + } + size_t after = i + typeLengths[t]; + size_t nameStart = skipSpaces(after); + if (nameStart == after) { + break; + } + size_t nameEnd = readName(nameStart); + if (nameEnd == 0) { + break; + } + size_t nameLen = nameEnd - nameStart; + /* skip anonymous classes: `new class extends X` captures the + * keyword that follows as if it were the name */ + if (!(nameLen == 7 && memcmp(contents + nameStart, "extends", 7) == 0) + && !(nameLen == 10 && memcmp(contents + nameStart, "implements", 10) == 0) + ) { + out.classes.push_back(qualified(contents + nameStart, nameLen)); + } + i = nameEnd; + matched = true; + break; + } + if (matched) { + continue; + } + + /* function \s++ (&\s*)? NAME \s*+ [&(] */ + if (keyword(i, "function", 8)) { + size_t after = i + 8; + size_t p = skipSpaces(after); + if (p != after) { + if (p < len && contents[p] == '&') { + p = skipSpaces(p + 1); + } + size_t nameEnd = readName(p); + if (nameEnd != 0) { + size_t tail = skipSpaces(nameEnd); + if (tail < len && (contents[tail] == '&' || contents[tail] == '(')) { + out.functions.push_back(qualified(contents + p, nameEnd - p)); + i = tail + 1; + continue; + } + } + } + } + + /* const \s++ NAME \s*+ [^;] */ + if (keyword(i, "const", 5)) { + size_t after = i + 5; + size_t p = skipSpaces(after); + if (p != after) { + size_t nameEnd = readName(p); + if (nameEnd != 0) { + size_t tail = skipSpaces(nameEnd); + if (tail < len && contents[tail] != ';') { + out.constants.push_back(normalizeConstantName(qualifiedConstant(contents + p, nameEnd - p))); + i = tail + 1; + continue; + } + } + } + } + + /* define \s*+ \( \s*+ ['"] DNAME */ + if (keyword(i, "define", 6)) { + size_t p = skipSpaces(i + 6); + if (p < len && contents[p] == '(') { + p = skipSpaces(p + 1); + if (p < len && (contents[p] == '\'' || contents[p] == '"')) { + size_t nameStart = p + 1; + size_t nameEnd = readDefineName(nameStart); + if (nameEnd != 0) { + out.constants.push_back(normalizeConstantName(std::string(contents + nameStart, nameEnd - nameStart))); + i = nameEnd; + continue; + } + } + } + } + + /* namespace (\s++ NSNAME)? \s*+ [{;] */ + if (keyword(i, "namespace", 9)) { + size_t after = i + 9; + size_t nameStart = skipSpaces(after); + size_t nameEnd = nameStart; + if (nameStart != after && nameStart < len && isNameStart((unsigned char) contents[nameStart])) { + nameEnd = nameStart + 1; + while (nameEnd < len && isDefineNameByte((unsigned char) contents[nameEnd])) { + nameEnd++; + } + for (;;) { + size_t p = skipSpaces(nameEnd); + if (p >= len || contents[p] != '\\') { + break; + } + p = skipSpaces(p + 1); + if (p >= len || !isNameStart((unsigned char) contents[p])) { + break; + } + p++; + while (p < len && isDefineNameByte((unsigned char) contents[p])) { + p++; + } + nameEnd = p; + } + } else { + nameEnd = after; + nameStart = after; + } + + size_t tail = skipSpaces(nameEnd); + if (tail < len && (contents[tail] == '{' || contents[tail] == ';')) { + currentNamespace.clear(); + for (size_t p = nameStart; p < nameEnd; p++) { + char ch = contents[p]; + if (isSpaceByte((unsigned char) ch)) { + continue; + } + currentNamespace.push_back(ch >= 'A' && ch <= 'Z' ? (char) (ch - 'A' + 'a') : ch); + } + currentNamespace.push_back('\\'); + i = tail + 1; + continue; + } + } + + i++; + } +} + +/* }}} */ + +} // namespace phpstanturbo + +#endif diff --git a/turbo-ext/src/main.cpp b/turbo-ext/src/main.cpp index cdaef0609a3..9840738c1e2 100644 --- a/turbo-ext/src/main.cpp +++ b/turbo-ext/src/main.cpp @@ -97,6 +97,8 @@ static PHP_MINIT_FUNCTION(phpstan_turbo) pt_register_type_combinator_cache(); pt_register_arena_cache(); pt_register_expression_result_storage(); + pt_register_php_file_cleaner(); + pt_register_symbol_finder_in_files(); return SUCCESS; } diff --git a/turbo-ext/src/support.h b/turbo-ext/src/support.h index af8da094021..7359be4a14d 100644 --- a/turbo-ext/src/support.h +++ b/turbo-ext/src/support.h @@ -148,6 +148,8 @@ void pt_register_parser_runner(); void pt_register_type_combinator_cache(); void pt_register_arena_cache(); void pt_register_expression_result_storage(); +void pt_register_php_file_cleaner(); +void pt_register_symbol_finder_in_files(); /* per-request hooks of individual classes */ void pt_node_traverser_rinit(); diff --git a/turbo-ext/tests/php-file-cleaner-corpus.php b/turbo-ext/tests/php-file-cleaner-corpus.php new file mode 100644 index 00000000000..fa776a379f0 --- /dev/null +++ b/turbo-ext/tests/php-file-cleaner-corpus.php @@ -0,0 +1,151 @@ +clean($contents, $maxMatches); + $b = $reference->clean($contents, $maxMatches); + if ($a === $b) { + continue; + } + + $failures++; + if ($failures > 5) { + continue; + } + + $at = 0; + $min = min(strlen($a), strlen($b)); + while ($at < $min && $a[$at] === $b[$at]) { + $at++; + } + printf("FAIL: %s (maxMatches=%d) differs at byte %d (lengths %d/%d)\n", $label, $maxMatches, $at, strlen($a), strlen($b)); + printf(" native: %s\n", var_export(substr($a, max(0, $at - 40), 90), true)); + printf(" php : %s\n", var_export(substr($b, max(0, $at - 40), 90), true)); + } +}; + +// ---- synthetic fixtures ---- +// Constructs whose handling differs between the twin's regexes and the +// hand-rolled native matchers, or which the repo corpus may not contain. +$fixtures = [ + 'heredoc' => " " " " " " " " " " " " " " " " " " "\n

class NotAClass {}

\n "\n\n " "class;\n", + 'function by reference' => " " "just text, no php at all\n", + 'php tag at eof' => " " " " $source) { + $tmp = tempnam(sys_get_temp_dir(), 'pfc'); + file_put_contents($tmp, $source); + $stripped = @php_strip_whitespace($tmp); + unlink($tmp); + if ($stripped === '' || $stripped === false) { + continue; + } + $compare('fixture ' . $label, $stripped); +} +printf("fixtures: %d (file, maxMatches) pairs checked\n", $checked); + +// ---- repo corpus ---- +$corpusStart = $checked; +$files = []; +foreach (['src', 'tests', 'build', 'compiler', 'e2e', 'turbo-ext', 'vendor'] as $dir) { + if (!is_dir($root . '/' . $dir)) { + continue; + } + $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root . '/' . $dir, FilesystemIterator::SKIP_DOTS)); + foreach ($iterator as $file) { + if ($file->isFile() && in_array($file->getExtension(), ['php', 'inc', 'stub'], true)) { + $files[] = $file->getPathname(); + } + } +} +sort($files); + +foreach ($files as $file) { + $contents = @php_strip_whitespace($file); + if ($contents === '' || $contents === false) { + continue; + } + $compare($file, $contents); +} +printf("corpus: %d files, %d (file, maxMatches) pairs checked\n", count($files), $checked - $corpusStart); + +echo $failures === 0 ? "ALL OK\n" : "$failures FAILURES\n"; +exit($failures === 0 ? 0 : 1); diff --git a/turbo-ext/tests/smoke.php b/turbo-ext/tests/smoke.php index 9f3ac3291c8..4fdc3e2e9c7 100644 --- a/turbo-ext/tests/smoke.php +++ b/turbo-ext/tests/smoke.php @@ -630,6 +630,8 @@ public function enterNode(\PhpParser\Node $node) $coveredElsewhere = [ \PHPStan\Cache\ArenaCache::class => 'arena-smoke.php', \PHPStan\Parser\ParserRunner::class => 'parser-corpus.php', + \PHPStan\Reflection\BetterReflection\SourceLocator\PhpFileCleaner::class => 'php-file-cleaner-corpus.php', + \PHPStan\Reflection\BetterReflection\SourceLocator\SymbolFinderInFiles::class => 'symbol-finder-corpus.php', ]; foreach (array_keys($shadowedClasses) as $shadowedClass) { check( diff --git a/turbo-ext/tests/symbol-finder-corpus.php b/turbo-ext/tests/symbol-finder-corpus.php new file mode 100644 index 00000000000..f49b9002115 --- /dev/null +++ b/turbo-ext/tests/symbol-finder-corpus.php @@ -0,0 +1,156 @@ + prefilter -> PhpFileCleaner -> symbol regex) with +// three native passes, so the bar is identical symbol triples — over every +// PHP file in the repository plus fixtures for the constructs where the +// stages disagree with a naive reading. +// +// Run: php -d extension=.../phpstan_turbo.so turbo-ext/tests/symbol-finder-corpus.php + +$root = dirname(__DIR__, 2); +require $root . '/vendor/autoload.php'; + +if (!extension_loaded('phpstan_turbo')) { + fwrite(STDERR, "extension not loaded\n"); + exit(2); +} + +// Declares the stub subclasses before the autoloader can load the twins. +PHPStan\Turbo\TurboExtensionEnabler::enableIfLoaded(); + +use PHPStan\Reflection\BetterReflection\SourceLocator\PhpFileCleaner; +use PHPStan\Reflection\BetterReflection\SourceLocator\SymbolFinderInFiles; + +$native = new SymbolFinderInFiles(new PhpFileCleaner()); +if (get_parent_class($native) !== 'PHPStanTurbo\SymbolFinderInFiles') { + fwrite(STDERR, "the native class is not shadowing the twin — is the extension version current?\n"); + exit(2); +} + +// The references are the twins' own sources with the classes renamed, so they +// cannot drift from the files the port mirrors. +$load = static function (string $file, string $from, string $to) use ($root): void { + $source = file_get_contents($root . '/src/Reflection/BetterReflection/SourceLocator/' . $file); + $source = substr($source, strpos($source, 'final class ' . $from)); + eval(str_replace( + ['final class ' . $from, 'PhpFileCleaner $cleaner'], + ['final class ' . $to, 'ReferencePhpFileCleaner $cleaner'], + $source, + )); +}; +$load('PhpFileCleaner.php', 'PhpFileCleaner', 'ReferencePhpFileCleaner'); +$load('SymbolFinderInFiles.php', 'SymbolFinderInFiles', 'ReferenceSymbolFinderInFiles'); +$reference = new ReferenceSymbolFinderInFiles(new ReferencePhpFileCleaner()); + +$failures = 0; +$checked = 0; + +$compare = static function (array $files, string $label) use ($native, $reference, &$failures, &$checked): void { + foreach ([true, false] as $supportsEnums) { + $checked += count($files); + $a = $native->findSymbols($files, $supportsEnums); + $b = $reference->findSymbols($files, $supportsEnums); + if ($a === $b) { + continue; + } + + foreach ($files as $file) { + if (($a[$file] ?? null) === ($b[$file] ?? null)) { + continue; + } + + $failures++; + if ($failures > 10) { + continue; + } + printf("FAIL: %s%s (supportsEnums=%s)\n", $label, $file, $supportsEnums ? 'true' : 'false'); + printf(" native: %s\n", json_encode($a[$file] ?? null)); + printf(" php : %s\n", json_encode($b[$file] ?? null)); + } + } +}; + +// ---- synthetic fixtures ---- +$fixtures = [ + 'plain class' => " " " " " " " " " " " " " " " " " " " " " "\n\n", + 'heredoc' => " " " " " "\n

class NotAClass {}

\n "\n\n "plain text class NotAClass\n", + 'property named class' => " " " " "", + 'only open tag' => " $source) { + $path = sprintf('%s/fixture-%02d.php', $dir, $i++); + file_put_contents($path, $source); + $fixtureFiles[$path] = $label; +} +foreach ($fixtureFiles as $path => $label) { + $compare([$path], $label . ': '); +} +printf("fixtures: %d checks\n", $checked); +foreach (array_keys($fixtureFiles) as $path) { + @unlink($path); +} +@rmdir($dir); + +// ---- repo corpus ---- +$corpusStart = $checked; +$files = []; +foreach (['src', 'tests', 'build', 'compiler', 'e2e', 'turbo-ext', 'vendor'] as $sub) { + if (!is_dir($root . '/' . $sub)) { + continue; + } + $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root . '/' . $sub, FilesystemIterator::SKIP_DOTS)); + foreach ($iterator as $file) { + if ($file->isFile() && in_array($file->getExtension(), ['php', 'inc', 'stub'], true)) { + $files[] = $file->getPathname(); + } + } +} +sort($files); + +// batched, so the native side exercises its reusable buffers +foreach (array_chunk($files, 400) as $chunk) { + $compare($chunk, ''); +} +printf("corpus: %d files, %d checks\n", count($files), $checked - $corpusStart); + +echo $failures === 0 ? "ALL OK\n" : "$failures FAILURES\n"; +exit($failures === 0 ? 0 : 1); From 27843c978a02cdb2728982935f4d3ed47f29e0ff Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Sun, 30 Aug 2026 13:08:28 +0200 Subject: [PATCH 2/2] Bump expected turbo version --- src/Turbo/TurboExtensionEnabler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Turbo/TurboExtensionEnabler.php b/src/Turbo/TurboExtensionEnabler.php index 4021947784d..a7a4aa86fc5 100644 --- a/src/Turbo/TurboExtensionEnabler.php +++ b/src/Turbo/TurboExtensionEnabler.php @@ -22,7 +22,7 @@ final class TurboExtensionEnabler * version is the short SHA of the last commit touching turbo-ext/src/, * enforced by the phar.yml turbo-version job. */ - public const EXPECTED_EXTENSION_VERSION = 'f531361'; + public const EXPECTED_EXTENSION_VERSION = '2659ad4'; private static bool $typeCombinatorCacheEnabled = false;