diff --git a/src/Parallel/ParallelAnalyser.php b/src/Parallel/ParallelAnalyser.php index e3aa2ca36c9..f64d9a69ed1 100644 --- a/src/Parallel/ParallelAnalyser.php +++ b/src/Parallel/ParallelAnalyser.php @@ -15,6 +15,7 @@ use PHPStan\DependencyInjection\AutowiredParameter; use PHPStan\DependencyInjection\AutowiredService; use PHPStan\Process\ProcessHelper; +use PHPStan\Reflection\BetterReflection\SourceLocator\PreForkDirectorySymbolScanner; use React\EventLoop\LoopInterface; use React\Promise\Deferred; use React\Promise\PromiseInterface; @@ -54,6 +55,7 @@ public function __construct( #[AutowiredParameter(ref: '%parallel.buffer%')] private int $decoderBufferSize, private ForkParallelChecker $forkParallelChecker, + private PreForkDirectorySymbolScanner $preForkDirectorySymbolScanner, private WorkerRunner $workerRunner, ) { @@ -216,6 +218,15 @@ public function analyse( $useFork = $this->forkParallelChecker->isSupported(); + if ($useFork && $numberOfProcesses > 1) { + // Build the directory symbol indexes here, in the parent, so the + // children inherit them copy-on-write instead of each scanning the + // same directories (see PreForkDirectorySymbolScanner). With a + // single worker there is nothing to share, and doing it here would + // only take work off the lazy path that worker might never reach. + $this->preForkDirectorySymbolScanner->scanBeforeFork(); + } + for ($i = 0; $i < $numberOfProcesses; $i++) { if (count($jobs) === 0) { break; diff --git a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php index b83b5a35a17..4548fe93a1d 100644 --- a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php +++ b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php @@ -63,10 +63,33 @@ public function __construct( private array $functionToFiles, private array $constantToFile, private ?string $arenaKeyPrefix = null, + private bool $awaitingBatchedScan = false, ) { } + /** + * Fills in the symbol maps of a locator created for a batched scan. + * + * The factory hands these out before the scan that produces their contents + * has run, so that one scan can cover every directory at once + * (see OptimizedDirectorySourceLocatorFactory::flushBatchedScan()). Nothing + * may look a symbol up in between - the maps are empty, so a lookup would + * quietly answer "not found" - which is what the flag guards. + * + * @param array $classToFile + * @param array> $functionToFiles + * @param array $constantToFile + * @internal + */ + public function fillBatchedScan(array $classToFile, array $functionToFiles, array $constantToFile): void + { + $this->classToFile = $classToFile; + $this->functionToFiles = $functionToFiles; + $this->constantToFile = $constantToFile; + $this->awaitingBatchedScan = false; + } + /** * @return array{non-empty-string, string} */ @@ -86,6 +109,10 @@ private function getCacheKeys(string $file, Identifier $identifier): array #[Override] public function locateIdentifier(Reflector $reflector, Identifier $identifier): ?Reflection { + if ($this->awaitingBatchedScan) { + throw new ShouldNotHappenException('Symbols were looked up in a directory whose batched scan has not been flushed yet.'); + } + if ($identifier->isClass()) { $identifierName = strtolower($identifier->getName()); $file = $this->findFileByClass($identifierName); @@ -334,6 +361,10 @@ private function hydrateSymbolsFromArena(): void #[Override] public function locateIdentifiersByType(Reflector $reflector, IdentifierType $identifierType): array { + if ($this->awaitingBatchedScan) { + throw new ShouldNotHappenException('Symbols were looked up in a directory whose batched scan has not been flushed yet.'); + } + $this->hydrateSymbolsFromArena(); $reflections = []; diff --git a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php index ff3429e1b4d..a4287665de8 100644 --- a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php +++ b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php @@ -10,9 +10,12 @@ use PHPStan\File\FileFinder; use PHPStan\Internal\DirectoryCreator; use PHPStan\Internal\DirectoryCreatorException; +use PHPStan\Parallel\ForkParallelChecker; use PHPStan\Php\PhpVersion; +use PHPStan\Turbo\TurboExtensionEnabler; use function array_key_exists; use function array_keys; +use function array_values; use function fclose; use function flock; use function fopen; @@ -50,6 +53,13 @@ final class OptimizedDirectorySourceLocatorFactory */ private const HASH_LOCK_POLL_INTERVAL_MICROSECONDS = 5_000; + /** + * Directories collected for a batched scan, null when not batching. + * + * @var list|null + */ + private ?array $batchedScan = null; + public function __construct( private FileNodesFetcher $fileNodesFetcher, #[AutowiredParameter(ref: '@fileFinderScan')] @@ -58,6 +68,7 @@ public function __construct( private SymbolFinderInFiles $symbolFinderInFiles, private Cache $cache, private FileContentHasher $fileContentHasher, + private ForkParallelChecker $forkParallelChecker, #[AutowiredParameter] private string $tmpDir, ) @@ -66,6 +77,10 @@ public function __construct( public function createByDirectory(string $directory): OptimizedDirectorySourceLocator { + if ($this->scansFresh()) { + return $this->createFreshDirectorySourceLocator($directory); + } + $cacheKey = sprintf('odsl-%s', $directory); $hashesRecordKey = 'odsl-filehashes-' . $directory; @@ -126,6 +141,119 @@ public function createByDirectory(string $directory): OptimizedDirectorySourceLo return $this->createCachedDirectorySourceLocator($fileHashes, $cacheKey); } + /** + * Whether the symbol index is built outright instead of being cached. + * + * Both halves are needed. The native scan is what makes the cache not + * worth its keep, and forking is what keeps the scan from happening once + * per worker: the parent scans before it forks and the children inherit + * the result (see PreForkDirectorySymbolScanner). Where the extension is + * active but workers are spawned rather than forked - Windows, or OPcache + * left on - there is nothing to inherit, so the cache and its scan lock + * stay in charge. + */ + private function scansFresh(): bool + { + return TurboExtensionEnabler::isActive() && $this->forkParallelChecker->isSupported(); + } + + /** + * With the turbo extension the symbol scan is native and costs about what + * hashing the directory to validate a cache costs, so a cache has nothing + * left to save: the directory is walked and scanned outright, with no file + * hashing, no persisted symbol table, no scan lock and no arena record — + * and therefore no cache that can go stale. PreForkDirectoryScanner runs + * this once in the main process before it forks its workers, so every + * worker inherits the finished locators instead of racing to build them. + */ + private function createFreshDirectorySourceLocator(string $directory): OptimizedDirectorySourceLocator + { + return $this->createFreshFileListSourceLocator($this->fileFinder->findFiles([$directory])->getFiles()); + } + + /** + * Starts collecting the directories asked for instead of scanning each one + * as it comes, so that flushBatchedScan() can cover all of them in a single + * scan: a file reachable from two directories is read once rather than + * twice, and the per-call costs are paid once instead of per directory. + */ + public function beginBatchedScan(): void + { + $this->batchedScan = []; + } + + /** + * Scans everything collected since beginBatchedScan() at once and fills in + * the locators handed out in the meantime. + */ + public function flushBatchedScan(): void + { + $batched = $this->batchedScan; + $this->batchedScan = null; + if ($batched === null || $batched === []) { + return; + } + + $allFiles = []; + foreach ($batched as [$files]) { + foreach ($files as $file) { + // a file reachable from two directories is scanned once + $allFiles[$file] = $file; + } + } + + $symbols = $this->symbolFinderInFiles->findSymbols(array_values($allFiles), $this->phpVersion->supportsEnums()); + + foreach ($batched as [$files, $locator]) { + $directorySymbols = []; + foreach ($files as $file) { + if (!array_key_exists($file, $symbols)) { + continue; + } + + $directorySymbols[$file] = $symbols[$file]; + } + + [$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($directorySymbols); + $locator->fillBatchedScan($classToFile, $functionToFiles, $constantToFile); + } + } + + /** + * @param string[] $files + */ + private function createFreshFileListSourceLocator(array $files): OptimizedDirectorySourceLocator + { + if ($this->batchedScan !== null) { + $locator = new OptimizedDirectorySourceLocator( + $this->fileNodesFetcher, + $this->cache, + $this->phpVersion, + $this->fileContentHasher, + [], + [], + [], + awaitingBatchedScan: true, + ); + $this->batchedScan[] = [$files, $locator]; + + return $locator; + } + + $symbols = $this->symbolFinderInFiles->findSymbols($files, $this->phpVersion->supportsEnums()); + [$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($symbols); + + return new OptimizedDirectorySourceLocator( + $this->fileNodesFetcher, + $this->cache, + $this->phpVersion, + $this->fileContentHasher, + $classToFile, + $functionToFiles, + $constantToFile, + ); + } + /** * @param array $fileHashes * @param non-empty-string $cacheKey @@ -233,7 +361,12 @@ private function createCachedDirectorySourceLocator(array $fileHashes, string $c } } - [$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($cached); + $symbols = []; + foreach ($cached as $file => [, $classes, $functions, $constants]) { + $symbols[$file] = [$classes, $functions, $constants]; + } + + [$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($symbols); // Publication order matters: the reader above requires all three // records, so a partially-published index is never consumed. @@ -322,6 +455,10 @@ private function releaseDirectoryScanLock($lockHandle): void */ public function createByFiles(array $files, string $uniqueCacheIdentifier): OptimizedDirectorySourceLocator { + if ($this->scansFresh()) { + return $this->createFreshFileListSourceLocator($files); + } + $fileHashes = []; foreach ($files as $file) { $hash = $this->fileContentHasher->hash($file); @@ -335,7 +472,7 @@ public function createByFiles(array $files, string $uniqueCacheIdentifier): Opti } /** - * @param array $symbols + * @param array $symbols * @return array{array, array>, array} */ private function changeStructure(array $symbols): array @@ -343,7 +480,7 @@ private function changeStructure(array $symbols): array $classToFile = []; $constantToFile = []; $functionToFiles = []; - foreach ($symbols as $file => [, $classes, $functions, $constants]) { + foreach ($symbols as $file => [$classes, $functions, $constants]) { foreach ($classes as $classInFile) { $classToFile[$classInFile] = $file; } diff --git a/src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php b/src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php index e1589495309..e6de2c8156c 100644 --- a/src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php +++ b/src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php @@ -3,6 +3,7 @@ namespace PHPStan\Reflection\BetterReflection\SourceLocator; use PHPStan\DependencyInjection\AutowiredService; +use PHPStan\Turbo\ShadowedByTurboExtension; use function array_keys; use function implode; use function in_array; @@ -18,6 +19,7 @@ * @see https://github.com/composer/composer/pull/10107 */ #[AutowiredService] +#[ShadowedByTurboExtension(turboClass: 'PHPStanTurbo\PhpFileCleaner', implementation: __DIR__ . '/../../../../turbo-ext/src/PhpFileCleaner.cpp')] final class PhpFileCleaner { diff --git a/src/Reflection/BetterReflection/SourceLocator/PreForkDirectorySymbolScanner.php b/src/Reflection/BetterReflection/SourceLocator/PreForkDirectorySymbolScanner.php new file mode 100644 index 00000000000..8d2a3629ec4 --- /dev/null +++ b/src/Reflection/BetterReflection/SourceLocator/PreForkDirectorySymbolScanner.php @@ -0,0 +1,110 @@ +analysedPaths, $this->analysedPathsFromConfig) as $analysedPath) { + if (!is_dir($analysedPath)) { + continue; + } + + $directories[] = $analysedPath; + } + + // Collect every directory first and scan them in one go. A file that + // two directories both reach is then read once instead of twice, and + // the scan pays its per-call costs once instead of per directory: + // measured over this repository's tree, 0.32s -> 0.16s. + $this->optimizedDirectorySourceLocatorFactory->beginBatchedScan(); + + try { + foreach (array_unique(array_merge($directories, $this->scanDirectories)) as $directory) { + $this->optimizedDirectorySourceLocatorRepository->getOrCreate($directory); + } + + foreach ($this->composerAutoloaderProjectPaths as $composerAutoloaderProjectPath) { + // the aggregate locator is thrown away - what matters is that the + // directory locators it builds land in the repository's memo, + // which the forked children inherit + $this->composerJsonAndInstalledJsonSourceLocatorMaker->create($composerAutoloaderProjectPath); + } + + $this->optimizedDirectorySourceLocatorFactory->flushBatchedScan(); + } finally { + // a throw must not leave the factory collecting into a batch that + // nobody will flush + $this->optimizedDirectorySourceLocatorFactory->flushBatchedScan(); + } + } + +} diff --git a/src/Reflection/BetterReflection/SourceLocator/SymbolFinderInFiles.php b/src/Reflection/BetterReflection/SourceLocator/SymbolFinderInFiles.php index e71b2b6aec3..b2bda25d4d0 100644 --- a/src/Reflection/BetterReflection/SourceLocator/SymbolFinderInFiles.php +++ b/src/Reflection/BetterReflection/SourceLocator/SymbolFinderInFiles.php @@ -3,6 +3,7 @@ namespace PHPStan\Reflection\BetterReflection\SourceLocator; use PHPStan\DependencyInjection\AutowiredService; +use PHPStan\Turbo\ShadowedByTurboExtension; use function array_filter; use function array_slice; use function count; @@ -19,6 +20,7 @@ use function strtolower; #[AutowiredService] +#[ShadowedByTurboExtension(turboClass: 'PHPStanTurbo\SymbolFinderInFiles', implementation: __DIR__ . '/../../../../turbo-ext/src/SymbolFinderInFiles.cpp')] final class SymbolFinderInFiles { @@ -91,6 +93,7 @@ private function findSymbolsInFile(string $file, bool $supportsEnums): array if ($matches['constant'][$i] !== '') { $constants[] = self::normalizeConstantName(ltrim($namespace . $matches['cname'][$i], '\\')); + continue; } if ($matches['define'][$i] !== '') { diff --git a/src/Turbo/TurboExtensionEnabler.php b/src/Turbo/TurboExtensionEnabler.php index 4db40afeb9b..a7a4aa86fc5 100644 --- a/src/Turbo/TurboExtensionEnabler.php +++ b/src/Turbo/TurboExtensionEnabler.php @@ -22,7 +22,7 @@ final class TurboExtensionEnabler * version is the short SHA of the last commit touching turbo-ext/src/, * enforced by the phar.yml turbo-version job. */ - public const EXPECTED_EXTENSION_VERSION = '873ede9'; + public const EXPECTED_EXTENSION_VERSION = '2659ad4'; private static bool $typeCombinatorCacheEnabled = false; diff --git a/tests/PHPStan/Reflection/BetterReflection/SourceLocator/SymbolFinderInFilesTest.php b/tests/PHPStan/Reflection/BetterReflection/SourceLocator/SymbolFinderInFilesTest.php new file mode 100644 index 00000000000..7667289b78b --- /dev/null +++ b/tests/PHPStan/Reflection/BetterReflection/SourceLocator/SymbolFinderInFilesTest.php @@ -0,0 +1,45 @@ + + */ + public static function dataFindSymbols(): iterable + { + yield 'namespaced constants do not leak a class entry' => [ + __DIR__ . '/data/symbol-finder/namespaced-constants.php', + [ + ['symbolfindertest\namespaced\thing'], + [], + ['symbolfindertest\namespaced\ALPHA', 'symbolfindertest\namespaced\BETA'], + ], + ]; + + yield 'global constants and defines' => [ + __DIR__ . '/data/symbol-finder/global-constants.php', + [ + [], + ['symbolfindertestfunction'], + ['GLOBAL_ALPHA', 'symbolfindertest\DEFINED'], + ], + ]; + } + + /** + * @param array{string[], string[], string[]} $expected + */ + #[DataProvider('dataFindSymbols')] + public function testFindSymbols(string $file, array $expected): void + { + $finder = new SymbolFinderInFiles(new PhpFileCleaner()); + $this->assertSame([$file => $expected], $finder->findSymbols([$file], true)); + } + +} diff --git a/tests/PHPStan/Reflection/BetterReflection/SourceLocator/data/symbol-finder/global-constants.php b/tests/PHPStan/Reflection/BetterReflection/SourceLocator/data/symbol-finder/global-constants.php new file mode 100644 index 00000000000..617a86932e5 --- /dev/null +++ b/tests/PHPStan/Reflection/BetterReflection/SourceLocator/data/symbol-finder/global-constants.php @@ -0,0 +1,10 @@ + +#include +#else +#include +#include +#include +#endif + +static zend_class_entry *pt_ce_symbol_finder = nullptr; + +namespace phpstanturbo { + +/* Mirrors PHPStan\...\SymbolFinderInFiles. The buffers live for the whole + * findSymbols() call so a directory of thousands of files reuses one pair of + * allocations. */ +class SymbolFinderInFiles +{ +public: + /* files far above this are not worth keeping the read buffer for */ + static constexpr size_t BUFFER_RETENTION_LIMIT = 4 * 1024 * 1024; + + zv::Val findSymbols(HashTable *files, bool supportsEnums); + +private: + std::string source; + std::string stripped; + std::string cleaned; + Symbols symbols; + + bool readFile(const char *path, size_t pathLen); + void scan(bool supportsEnums); + static void symbolsToArray(const Symbols &symbols, zval *out); +}; + +/* + * The twin reaches the file through php_strip_whitespace(), which goes past + * the stream wrappers; the locators only ever pass real paths from their own + * directory walk, so a plain open() is enough — and an unreadable file has to + * behave like the twin's suppressed warning, i.e. produce no symbols. + */ +bool SymbolFinderInFiles::readFile(const char *path, size_t pathLen) +{ + source.clear(); + + if (pathLen == 0 || memchr(path, '\0', pathLen) != NULL) { + return false; + } + +#ifdef PHP_WIN32 + int fd = _open(path, _O_RDONLY | _O_BINARY); +#else + int fd = open(path, O_RDONLY); +#endif + if (fd < 0) { + return false; + } + + char chunk[65536]; + for (;;) { +#ifdef PHP_WIN32 + int got = _read(fd, chunk, sizeof(chunk)); +#else + ssize_t got = read(fd, chunk, sizeof(chunk)); +#endif + if (got < 0) { +#ifdef PHP_WIN32 + _close(fd); +#else + close(fd); +#endif + source.clear(); + return false; + } + if (got == 0) { + break; + } + source.append(chunk, (size_t) got); + } + +#ifdef PHP_WIN32 + _close(fd); +#else + close(fd); +#endif + + return true; +} + +void SymbolFinderInFiles::scan(bool supportsEnums) +{ + symbols.clear(); + + if (source.empty()) { + return; + } + + CommentStripper stripper(source.data(), source.size(), shortOpenTagEnabled()); + stripper.strip(stripped); + + if (stripped.empty()) { + return; + } + + size_t matches = prefilterCount(stripped.data(), stripped.size(), supportsEnums); + if (matches == 0) { + return; + } + + PhpFileCleaner cleaner(stripped.data(), stripped.size()); + cleaner.clean((zend_long) matches, cleaned); + + SymbolMatcher matcher(cleaned.data(), cleaned.size(), supportsEnums); + matcher.match(symbols); +} + +void SymbolFinderInFiles::symbolsToArray(const Symbols &symbols, zval *out) +{ + zval &triple = *out; + array_init_size(&triple, 3); + + const std::vector *groups[3] = { &symbols.classes, &symbols.functions, &symbols.constants }; + for (const std::vector *group : groups) { + zval list; + array_init_size(&list, (uint32_t) group->size()); + for (const std::string &name : *group) { + zval item; + ZVAL_STRINGL(&item, name.data(), name.size()); + zend_hash_next_index_insert_new(Z_ARRVAL(list), &item); + } + zend_hash_next_index_insert_new(Z_ARRVAL(triple), &list); + } +} + +zv::Val SymbolFinderInFiles::findSymbols(HashTable *files, bool supportsEnums) +{ + zval result; + array_init_size(&result, zend_hash_num_elements(files)); + + for (zv::ArrayEntry file : zv::TableRef(files)) { + zv::Ref value = file.value().deref(); + if (!value.isString()) { + continue; + } + + zend_string *path = value.asString(); + if (readFile(ZSTR_VAL(path), ZSTR_LEN(path))) { + scan(supportsEnums); + } else { + symbols.clear(); + } + + zval triple; + symbolsToArray(symbols, &triple); + zend_hash_update(Z_ARRVAL(result), path, &triple); + + if (source.capacity() > BUFFER_RETENTION_LIMIT) { + std::string().swap(source); + } + } + + return zv::Val::adopt(result); +} + +} // namespace phpstanturbo + +/* {{{ registration */ + +#include "reg.h" + +#define CLEANER_CLASS "PHPStan\\Reflection\\BetterReflection\\SourceLocator\\PhpFileCleaner" + +void pt_register_symbol_finder_in_files() +{ + reg::Class cls("PHPStanTurbo\\SymbolFinderInFiles"); + + /* the arginfo has to keep the real parameter class name: Nette reflects + * this constructor while compiling the container (rule 6) */ + cls.method("__construct", reg::Public, 1, { reg::obj("cleaner", CLEANER_CLASS) }, [](INTERNAL_FUNCTION_PARAMETERS) { + zval *cleaner; + ZEND_PARSE_PARAMETERS_START(1, 1) + Z_PARAM_OBJECT(cleaner) + ZEND_PARSE_PARAMETERS_END(); + (void) cleaner; + }); + + cls.method("findSymbols", reg::Public, 2, { reg::arrayArg("files"), reg::boolArg("supportsEnums") }, [](INTERNAL_FUNCTION_PARAMETERS) { + HashTable *files; + bool supportsEnums; + ZEND_PARSE_PARAMETERS_START(2, 2) + Z_PARAM_ARRAY_HT(files) + Z_PARAM_BOOL(supportsEnums) + ZEND_PARSE_PARAMETERS_END(); + + phpstanturbo::SymbolFinderInFiles finder; + finder.findSymbols(files, supportsEnums).intoReturnValue(return_value); + }); + + /* not final: a PHP stub subclass may extend this class */ + pt_ce_symbol_finder = cls.register_(); +} + +/* }}} */ diff --git a/turbo-ext/src/SymbolScan.h b/turbo-ext/src/SymbolScan.h new file mode 100644 index 00000000000..15eff332ec2 --- /dev/null +++ b/turbo-ext/src/SymbolScan.h @@ -0,0 +1,1143 @@ +/* + * Shared scanning primitives behind the optimized source locators' directory + * symbol scan — the pipeline PHPStan runs per file: + * + * php_strip_whitespace() -> clean() -> symbol regex + * + * All three stages live here natively so SymbolFinderInFiles can run them + * back to back over one reusable pair of buffers, while PhpFileCleaner.cpp + * still exposes the middle stage on its own as the shadow of the PHP twin. + * + * The stages stay separate passes on purpose. php_strip_whitespace() deletes + * comments without leaving a separator, so an identifier split by a comment + * really does reach the cleaner joined back together — fusing comment removal + * into the cleaner would lose that join, and with it the parity the port is + * judged on. + */ + +#ifndef PHPSTANTURBO_SYMBOLSCAN_H +#define PHPSTANTURBO_SYMBOLSCAN_H + +#include "support.h" + +#include +#include + +/* The twin's $rejectChars: '{}?"\'= 0x80 are not word bytes */ +inline bool isWordByte(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* PCRE's \s */ +inline bool isSpaceByte(unsigned char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +/* [a-zA-Z_\x7f-\xff] */ +inline bool isNameStart(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c >= 0x7f; +} + +/* [a-zA-Z0-9_\x7f-\xff\-] — the dash is in the class, odd as it looks */ +inline bool isNameByte(unsigned char c) +{ + return isNameStart(c) || (c >= '0' && c <= '9') || c == '-'; +} + +/* [a-zA-Z_\x80-\xff] / [a-zA-Z0-9_\x80-\xff] — heredoc labels start at + * \x80, not \x7f, in the twin's patterns */ +inline bool isLabelStart(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c >= 0x80; +} + +inline bool isLabelByte(unsigned char c) +{ + return isLabelStart(c) || (c >= '0' && c <= '9'); +} + +inline bool equalsIgnoreCase(const char *a, const char *lowercaseB, size_t n) +{ + for (size_t i = 0; i < n; i++) { + char c = a[i]; + if (c >= 'A' && c <= 'Z') { + c = (char) (c - 'A' + 'a'); + } + if (c != lowercaseB[i]) { + return false; + } + } + return true; +} + +/* the define()/namespace name class: like isNameByte but without the dash */ +inline bool isDefineNameByte(unsigned char c) +{ + return isNameStart(c) || (c >= '0' && c <= '9'); +} + +/* whether a bare `])` anchored one byte before `at`: the byte before the + * keyword must exist, must not be a word byte (that is the \b, since the + * keyword starts with one) and must not be $, : or >. */ + bool prevByteOpensKeyword(size_t at) const + { + if (at == 0 || at > len) { + return false; + } + unsigned char prev = (unsigned char) contents[at - 1]; + return !isWordByte(prev) && prev != '$' && prev != ':' && prev != '>'; + } + + bool peek(char c) const { return index + 1 < len && contents[index + 1] == c; } + + /* `\s++[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff\-]*+` starting at `from`; + * on success `end` receives the offset just past the name. */ + bool matchSpacesAndName(size_t from, size_t *end) const + { + size_t p = from; + while (p < len && isSpaceByte((unsigned char) contents[p])) { + p++; + } + if (p == from || p >= len || !isNameStart((unsigned char) contents[p])) { + return false; + } + p++; + while (p < len && isNameByte((unsigned char) contents[p])) { + p++; + } + *end = p; + return true; + } + + void skipToPhp(); + void skipString(char delimiter); + void consumeString(char delimiter, std::string &clean); + void skipComment(); + void skipToNewline(); + bool matchHeredocStart(size_t *labelStart, size_t *labelLen, size_t *end) const; + void skipHeredoc(const char *label, size_t labelLen); +}; + +inline void PhpFileCleaner::skipToPhp() +{ + while (index < len) { + if (contents[index] == '<' && peek('?')) { + index += 2; + break; + } + + index += 1; + } +} + +/* The twin's consumeString(): copies the string body verbatim, keeping + * backslash escapes, up to and including the closing delimiter. */ +inline void PhpFileCleaner::consumeString(char delimiter, std::string &clean) +{ + index += 1; + while (index < len) { + if (contents[index] == '\\' && (peek('\\') || peek(delimiter))) { + clean.append(contents + index, 2); + index += 2; + continue; + } + + if (contents[index] == delimiter) { + clean.push_back(delimiter); + index += 1; + break; + } + + clean.push_back(contents[index]); + index += 1; + } +} + +inline void PhpFileCleaner::skipString(char delimiter) +{ + index += 1; + while (index < len) { + while (index < len && contents[index] != '\\' && contents[index] != delimiter) { + index++; + } + if (index >= len) { + break; + } + if (contents[index] == '\\' && (peek('\\') || peek(delimiter))) { + index += 2; + continue; + } + if (contents[index] == delimiter) { + index += 1; + break; + } + index += 1; + } +} + +inline void PhpFileCleaner::skipComment() +{ + index += 2; + while (index < len) { + while (index < len && contents[index] != '*') { + index++; + } + + if (peek('/')) { + index += 2; + break; + } + + index += 1; + } +} + +inline void PhpFileCleaner::skipToNewline() +{ + while (index < len && contents[index] != '\r' && contents[index] != '\n') { + index++; + } +} + +/* `{<<<[ \t]*+(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*+)\1(?:\r\n|\n|\r)}A` */ +inline bool PhpFileCleaner::matchHeredocStart(size_t *labelStart, size_t *labelLen, size_t *end) const +{ + size_t p = index; + if (p + 3 > len || contents[p] != '<' || contents[p + 1] != '<' || contents[p + 2] != '<') { + return false; + } + p += 3; + while (p < len && (contents[p] == ' ' || contents[p] == '\t')) { + p++; + } + char quote = '\0'; + if (p < len && (contents[p] == '\'' || contents[p] == '"')) { + quote = contents[p]; + p++; + } + if (p >= len || !isLabelStart((unsigned char) contents[p])) { + return false; + } + size_t start = p; + p++; + while (p < len && isLabelByte((unsigned char) contents[p])) { + p++; + } + *labelStart = start; + *labelLen = p - start; + if (quote != '\0') { + if (p >= len || contents[p] != quote) { + return false; + } + p++; + } + if (p < len && contents[p] == '\r') { + p += (p + 1 < len && contents[p + 1] == '\n') ? 2 : 1; + } else if (p < len && contents[p] == '\n') { + p += 1; + } else { + return false; + } + *end = p; + return true; +} + +inline void PhpFileCleaner::skipHeredoc(const char *label, size_t labelLen) +{ + char firstLabelByte = label[0]; + + while (index < len) { + /* the label may be preceded by indentation */ + char c = contents[index]; + if (c == '\t' || c == ' ') { + index += 1; + continue; + } + if (c == firstLabelByte + && index + labelLen <= len + && memcmp(contents + index, label, labelLen) == 0 + && (index + labelLen >= len || !isLabelByte((unsigned char) contents[index + labelLen])) + ) { + index += labelLen; + return; + } + + skipToNewline(); + while (index < len && (contents[index] == '\r' || contents[index] == '\n')) { + index++; + } + } +} + +inline void PhpFileCleaner::clean(zend_long maxMatches, std::string &out) +{ + /* keyed by first byte, exactly like the twin's $typeConfig */ + struct TypeConfig { + char firstByte; + const char *name; + size_t length; + }; + static const TypeConfig types[] = { + { 'c', "class", 5 }, + { 'i', "interface", 9 }, + { 't', "trait", 5 }, + { 'e', "enum", 4 }, + }; + + std::string &clean = out; + clean.clear(); + clean.reserve(len); + + bool inType = false; + zend_long typeLevel = 0; + bool inDefine = false; + + while (index < len) { + skipToPhp(); + clean.append("')) { + clean.append("?>", 2); + index += 2; + break; /* continue 2 */ + } + + if (c == '"' || c == '\'') { + if (inDefine) { + clean.push_back(c); + consumeString(c, clean); + inDefine = false; + } else { + skipString(c); + clean.append("null", 4); + } + + continue; + } + + if (c == '{') { + if (inType) { + typeLevel++; + } + + clean.push_back(c); + index++; + continue; + } + + if (c == '}') { + if (inType) { + typeLevel--; + + if (typeLevel == 0) { + inType = false; + } + } + + clean.push_back(c); + index++; + continue; + } + + if (c == '<' && peek('<')) { + size_t labelStart, labelLen, end; + if (matchHeredocStart(&labelStart, &labelLen, &end)) { + const char *label = contents + labelStart; + index = end; + skipHeredoc(label, labelLen); + clean.append("null", 4); + continue; + } + } + + if (c == '/') { + if (peek('/')) { + skipToNewline(); + continue; + } + if (peek('*')) { + skipComment(); + continue; + } + } + + /* `~.\b(?])const(\s++NAME)~Ais` at index - 1 */ + if (inType && c == 'c' && prevByteOpensKeyword(index) && index + 5 <= len + && equalsIgnoreCase(contents + index, "const", 5) + ) { + size_t end; + if (matchSpacesAndName(index + 5, &end)) { + /* invalid PHP, but it only has to stop the symbol regex + * from reading a class constant as a global one */ + clean.append("class_const", 11); + clean.append(contents + index + 5, end - (index + 5)); + index = end; + continue; + } + } + + /* `~.\b(?])define\s*+\(~Ais` at index - 1 */ + if (c == 'd' && prevByteOpensKeyword(index) && index + 6 <= len + && equalsIgnoreCase(contents + index, "define", 6) + ) { + size_t p = index + 6; + while (p < len && isSpaceByte((unsigned char) contents[p])) { + p++; + } + if (p < len && contents[p] == '(') { + /* the twin appends the whole match, which starts one byte + * before the keyword — that byte is already in the output, + * so it lands twice. Harmless for the symbol regex, and + * reproduced here to keep the output byte-identical. */ + clean.append(contents + index - 1, p + 1 - (index - 1)); + index = p + 1; + inDefine = true; + continue; + } + } + + for (const TypeConfig &type : types) { + if (type.firstByte != c) { + continue; + } + + if (index + type.length <= len && memcmp(contents + index, type.name, type.length) == 0) { + if (maxMatches == 1 && prevByteOpensKeyword(index)) { + size_t end; + if (matchSpacesAndName(index + type.length, &end)) { + clean.append(contents + index - 1, end - (index - 1)); + return; + } + } + + inType = true; + } + + break; + } + + index += 1; + size_t skipFrom = index; + while (index < len && !pt_reject_table.bytes[(unsigned char) contents[index]]) { + index++; + } + if (index > skipFrom) { + clean.push_back(c); + clean.append(contents + skipFrom, index - skipFrom); + } else { + clean.push_back(c); + } + } + } + +} + + + +/* {{{ stage 1 — php_strip_whitespace() equivalent */ + +/* Bytes that can start a construct the stripper must understand. */ +static const struct StripTable { + bool bytes[256]; + + StripTable() : bytes() + { + for (const char *p = "?/#'\"`<"; *p != '\0'; p++) { + bytes[(unsigned char) *p] = true; + } + } +} pt_strip_table; + +/* + * Removes comments the way php_strip_whitespace() does — emitting nothing in + * their place, so the bytes around them become adjacent — while copying + * everything else through verbatim. The twin's stripper also collapses each + * whitespace run to a single space; that is deliberately not reproduced, + * because every consumer downstream matches whitespace with \s+ or \s* and + * cannot tell the difference, and copying spans verbatim is faster. + * + * Unlike the cleaner, this stage has to know real lexer rules: # comments + * (but not #[ attributes), line comments ending at ?>, and backtick strings. + */ +class CommentStripper +{ +public: + CommentStripper(const char *contents, size_t len, bool shortOpenTag) + : contents(contents), len(len), index(0), shortOpenTag(shortOpenTag) {} + + void strip(std::string &out); + +private: + const char *contents; + size_t len; + size_t index; + bool shortOpenTag; + + /* length of the open tag at `at`, or 0 if there is none */ + size_t openTagLength(size_t at) const + { + if (at + 1 >= len || contents[at] != '<' || contents[at + 1] != '?') { + return 0; + } + if (at + 4 < len && equalsIgnoreCase(contents + at + 2, "php", 3) + && (at + 5 >= len || isSpaceByte((unsigned char) contents[at + 5])) + ) { + return 5; + } + if (at + 2 < len && contents[at + 2] == '=') { + return 3; + } + + return shortOpenTag ? 2 : 0; + } + + /* a // or # comment: ends at a newline (left in place, it is whitespace) + * or at ?>, which the caller then handles as the close tag it is */ + void skipLineComment() + { + while (index < len) { + char c = contents[index]; + if (c == '\n' || c == '\r') { + return; + } + if (c == '?' && index + 1 < len && contents[index + 1] == '>') { + return; + } + index++; + } + } + + void skipBlockComment() + { + index += 2; + while (index + 1 < len) { + if (contents[index] == '*' && contents[index + 1] == '/') { + index += 2; + return; + } + index++; + } + index = len; + } + + /* copies a quoted string verbatim; a backslash escapes the next byte, + * which finds the same closing quote as PHP's own rules do */ + void copyString(char delimiter, std::string &out) + { + size_t start = index; + index++; + while (index < len) { + char c = contents[index]; + if (c == '\\' && index + 1 < len) { + index += 2; + continue; + } + index++; + if (c == delimiter) { + break; + } + } + out.append(contents + start, index - start); + } + + void copyHeredoc(std::string &out); +}; + +inline void CommentStripper::copyHeredoc(std::string &out) +{ + size_t p = index + 3; + while (p < len && (contents[p] == ' ' || contents[p] == '\t')) { + p++; + } + char quote = '\0'; + if (p < len && (contents[p] == '\'' || contents[p] == '"')) { + quote = contents[p]; + p++; + } + if (p >= len || !isLabelStart((unsigned char) contents[p])) { + /* not a heredoc after all — let the caller copy the bytes */ + return; + } + size_t labelStart = p; + p++; + while (p < len && isLabelByte((unsigned char) contents[p])) { + p++; + } + size_t labelLen = p - labelStart; + if (quote != '\0') { + if (p >= len || contents[p] != quote) { + return; + } + p++; + } + if (p < len && contents[p] == '\r') { + p += (p + 1 < len && contents[p + 1] == '\n') ? 2 : 1; + } else if (p < len && contents[p] == '\n') { + p += 1; + } else { + return; + } + + size_t bodyStart = p; + const char *label = contents + labelStart; + while (p < len) { + char c = contents[p]; + if (c == '\t' || c == ' ') { + p++; + continue; + } + if (c == label[0] + && p + labelLen <= len + && memcmp(contents + p, label, labelLen) == 0 + && (p + labelLen >= len || !isLabelByte((unsigned char) contents[p + labelLen])) + ) { + p += labelLen; + break; + } + while (p < len && contents[p] != '\r' && contents[p] != '\n') { + p++; + } + while (p < len && (contents[p] == '\r' || contents[p] == '\n')) { + p++; + } + } + (void) bodyStart; + + out.append(contents + index, p - index); + index = p; +} + +inline void CommentStripper::strip(std::string &out) +{ + out.clear(); + out.reserve(len); + + while (index < len) { + /* inline HTML up to the next opening tag, copied verbatim */ + size_t htmlStart = index; + size_t tagLength = 0; + while (index < len) { + tagLength = openTagLength(index); + if (tagLength != 0) { + break; + } + index++; + } + out.append(contents + htmlStart, index - htmlStart); + if (index >= len) { + return; + } + out.append(contents + index, tagLength); + index += tagLength; + + while (index < len) { + char c = contents[index]; + + if (c == '?' && index + 1 < len && contents[index + 1] == '>') { + out.append("?>", 2); + index += 2; + break; + } + + if (c == '/' && index + 1 < len && contents[index + 1] == '/') { + skipLineComment(); + continue; + } + + if (c == '#') { + if (index + 1 < len && contents[index + 1] == '[') { + out.append("#[", 2); + index += 2; + continue; + } + skipLineComment(); + continue; + } + + if (c == '/' && index + 1 < len && contents[index + 1] == '*') { + skipBlockComment(); + continue; + } + + if (c == '\'' || c == '"' || c == '`') { + copyString(c, out); + continue; + } + + if (c == '<' && index + 2 < len && contents[index + 1] == '<' && contents[index + 2] == '<') { + size_t before = index; + copyHeredoc(out); + if (index != before) { + continue; + } + } + + size_t start = index; + index++; + while (index < len && !pt_strip_table.bytes[(unsigned char) contents[index]]) { + index++; + } + out.append(contents + start, index - start); + } + } +} + +/* }}} */ + + +/* {{{ stage 2a — the prefilter count */ + +/* + * The twin's prefilter, `{\b(?:(?:class|interface|trait|const|function|enum)\s) + * |(?:define\s*\()}i`, whose match count it hands to the cleaner as + * maxMatches. Only "is it exactly one" is ever asked (that is what arms the + * cleaner's early return) and zero means the twin returns no symbols at all, + * so counting stops at two. + * + * It cannot be skipped even though the full scan finds the same declarations: + * $typeConfig always contains `enum`, so on a supportsEnums=false run the + * early return can fire on an enum the symbol regex has no branch for and + * truncate away a function or constant that would otherwise be found. + * + * Note the pattern's shape: the \b applies to the keyword branch only, and + * the keyword must be followed by whitespace, both unlike the symbol regex. + */ +inline size_t prefilterCount(const char *contents, size_t len, bool supportsEnums) +{ + static const char *const keywords[] = { "class", "interface", "trait", "const", "function", "enum" }; + static const size_t keywordLengths[] = { 5, 9, 5, 5, 8, 4 }; + const size_t keywordCount = supportsEnums ? 6 : 5; + + size_t count = 0; + size_t i = 0; + while (i < len && count < 2) { + unsigned char c = (unsigned char) contents[i]; + if (!pt_keyword_start_table.bytes[c]) { + i++; + continue; + } + + if (i == 0 || !isWordByte((unsigned char) contents[i - 1])) { + bool matched = false; + for (size_t k = 0; k < keywordCount; k++) { + size_t length = keywordLengths[k]; + if (i + length < len + && equalsIgnoreCase(contents + i, keywords[k], length) + && isSpaceByte((unsigned char) contents[i + length]) + ) { + count++; + i += length + 1; + matched = true; + break; + } + } + if (matched) { + continue; + } + } + + /* the define branch carries no \b — `mydefine(` counts too */ + if ((c == 'd' || c == 'D') && i + 6 <= len && equalsIgnoreCase(contents + i, "define", 6)) { + size_t p = i + 6; + while (p < len && isSpaceByte((unsigned char) contents[p])) { + p++; + } + if (p < len && contents[p] == '(') { + count++; + i = p + 1; + continue; + } + } + + i++; + } + + return count; +} + +/* }}} */ + +/* {{{ stage 3 — the symbol regex */ + +struct Symbols { + std::vector classes; + std::vector functions; + std::vector constants; + + void clear() + { + classes.clear(); + functions.clear(); + constants.clear(); + } +}; + +/* + * The preg_match_all() over the cleaned text plus the loop that turns its + * captures into symbol names. The pattern is one alternation of five + * branches, all sharing a `\b(?])` prefix, so the walk only has to + * try a branch where that guard holds and the byte can start a keyword. + */ +class SymbolMatcher +{ +public: + SymbolMatcher(const char *contents, size_t len, bool supportsEnums) + : contents(contents), len(len), supportsEnums(supportsEnums) {} + + void match(Symbols &out); + +private: + const char *contents; + size_t len; + bool supportsEnums; + std::string currentNamespace; + + bool guard(size_t at) const + { + if (at == 0) { + return true; + } + unsigned char prev = (unsigned char) contents[at - 1]; + return !isWordByte(prev) && prev != '$' && prev != ':' && prev != '>'; + } + + bool keyword(size_t at, const char *lowercase, size_t length) const + { + return at + length <= len && equalsIgnoreCase(contents + at, lowercase, length) + && (at + length >= len || !isWordByte((unsigned char) contents[at + length])); + } + + size_t skipSpaces(size_t at) const + { + while (at < len && isSpaceByte((unsigned char) contents[at])) { + at++; + } + return at; + } + + /* [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff\-]*+ */ + size_t readName(size_t at) const + { + if (at >= len || !isNameStart((unsigned char) contents[at])) { + return 0; + } + size_t end = at + 1; + while (end < len && isNameByte((unsigned char) contents[end])) { + end++; + } + return end; + } + + /* the define() name: identifiers joined by one or two backslashes */ + size_t readDefineName(size_t at) const + { + if (at >= len || !isNameStart((unsigned char) contents[at])) { + return 0; + } + size_t end = at + 1; + while (end < len && isDefineNameByte((unsigned char) contents[end])) { + end++; + } + for (;;) { + size_t p = end; + size_t slashes = 0; + while (p < len && contents[p] == '\\' && slashes < 2) { + p++; + slashes++; + } + if (slashes == 0 || p >= len || !isNameStart((unsigned char) contents[p])) { + break; + } + p++; + while (p < len && isDefineNameByte((unsigned char) contents[p])) { + p++; + } + end = p; + } + return end; + } + + static void appendLowercase(std::string &out, const char *from, size_t length) + { + for (size_t i = 0; i < length; i++) { + char c = from[i]; + out.push_back(c >= 'A' && c <= 'Z' ? (char) (c - 'A' + 'a') : c); + } + } + + /* strtolower(ltrim($namespace . $name, '\\')) */ + std::string qualified(const char *name, size_t nameLen) const + { + std::string full = currentNamespace; + full.append(name, nameLen); + size_t start = 0; + while (start < full.size() && full[start] == '\\') { + start++; + } + std::string result; + result.reserve(full.size() - start); + appendLowercase(result, full.data() + start, full.size() - start); + return result; + } + + /* self::normalizeConstantName(): the namespace part lowercases, the + * constant's own name keeps its case */ + static std::string normalizeConstantName(const std::string &name) + { + if (name.find('\\') == std::string::npos) { + return name; + } + + std::vector parts; + size_t start = 0; + for (size_t i = 0; i <= name.size(); i++) { + if (i == name.size() || name[i] == '\\') { + if (i > start) { + parts.emplace_back(name, start, i - start); + } + start = i + 1; + } + } + if (parts.empty()) { + return std::string("\\"); + } + + std::string result; + for (size_t i = 0; i + 1 < parts.size(); i++) { + if (i > 0) { + result.push_back('\\'); + } + appendLowercase(result, parts[i].data(), parts[i].size()); + } + result.push_back('\\'); + result.append(parts.back()); + + return result; + } + + /* ltrim($namespace . $name, '\\') without the lowercasing */ + std::string qualifiedConstant(const char *name, size_t nameLen) const + { + std::string full = currentNamespace; + full.append(name, nameLen); + size_t start = 0; + while (start < full.size() && full[start] == '\\') { + start++; + } + return full.substr(start); + } +}; + +inline void SymbolMatcher::match(Symbols &out) +{ + currentNamespace.clear(); + + size_t i = 0; + while (i < len) { + unsigned char c = (unsigned char) contents[i]; + if (!pt_keyword_start_table.bytes[c] || !guard(i)) { + i++; + continue; + } + + /* class|interface|trait[|enum] \s++ NAME */ + static const char *const typeNames[] = { "class", "interface", "trait", "enum" }; + static const size_t typeLengths[] = { 5, 9, 5, 4 }; + bool matched = false; + for (size_t t = 0; t < 4; t++) { + if (t == 3 && !supportsEnums) { + break; + } + if (!keyword(i, typeNames[t], typeLengths[t])) { + continue; + } + size_t after = i + typeLengths[t]; + size_t nameStart = skipSpaces(after); + if (nameStart == after) { + break; + } + size_t nameEnd = readName(nameStart); + if (nameEnd == 0) { + break; + } + size_t nameLen = nameEnd - nameStart; + /* skip anonymous classes: `new class extends X` captures the + * keyword that follows as if it were the name */ + if (!(nameLen == 7 && memcmp(contents + nameStart, "extends", 7) == 0) + && !(nameLen == 10 && memcmp(contents + nameStart, "implements", 10) == 0) + ) { + out.classes.push_back(qualified(contents + nameStart, nameLen)); + } + i = nameEnd; + matched = true; + break; + } + if (matched) { + continue; + } + + /* function \s++ (&\s*)? NAME \s*+ [&(] */ + if (keyword(i, "function", 8)) { + size_t after = i + 8; + size_t p = skipSpaces(after); + if (p != after) { + if (p < len && contents[p] == '&') { + p = skipSpaces(p + 1); + } + size_t nameEnd = readName(p); + if (nameEnd != 0) { + size_t tail = skipSpaces(nameEnd); + if (tail < len && (contents[tail] == '&' || contents[tail] == '(')) { + out.functions.push_back(qualified(contents + p, nameEnd - p)); + i = tail + 1; + continue; + } + } + } + } + + /* const \s++ NAME \s*+ [^;] */ + if (keyword(i, "const", 5)) { + size_t after = i + 5; + size_t p = skipSpaces(after); + if (p != after) { + size_t nameEnd = readName(p); + if (nameEnd != 0) { + size_t tail = skipSpaces(nameEnd); + if (tail < len && contents[tail] != ';') { + out.constants.push_back(normalizeConstantName(qualifiedConstant(contents + p, nameEnd - p))); + i = tail + 1; + continue; + } + } + } + } + + /* define \s*+ \( \s*+ ['"] DNAME */ + if (keyword(i, "define", 6)) { + size_t p = skipSpaces(i + 6); + if (p < len && contents[p] == '(') { + p = skipSpaces(p + 1); + if (p < len && (contents[p] == '\'' || contents[p] == '"')) { + size_t nameStart = p + 1; + size_t nameEnd = readDefineName(nameStart); + if (nameEnd != 0) { + out.constants.push_back(normalizeConstantName(std::string(contents + nameStart, nameEnd - nameStart))); + i = nameEnd; + continue; + } + } + } + } + + /* namespace (\s++ NSNAME)? \s*+ [{;] */ + if (keyword(i, "namespace", 9)) { + size_t after = i + 9; + size_t nameStart = skipSpaces(after); + size_t nameEnd = nameStart; + if (nameStart != after && nameStart < len && isNameStart((unsigned char) contents[nameStart])) { + nameEnd = nameStart + 1; + while (nameEnd < len && isDefineNameByte((unsigned char) contents[nameEnd])) { + nameEnd++; + } + for (;;) { + size_t p = skipSpaces(nameEnd); + if (p >= len || contents[p] != '\\') { + break; + } + p = skipSpaces(p + 1); + if (p >= len || !isNameStart((unsigned char) contents[p])) { + break; + } + p++; + while (p < len && isDefineNameByte((unsigned char) contents[p])) { + p++; + } + nameEnd = p; + } + } else { + nameEnd = after; + nameStart = after; + } + + size_t tail = skipSpaces(nameEnd); + if (tail < len && (contents[tail] == '{' || contents[tail] == ';')) { + currentNamespace.clear(); + for (size_t p = nameStart; p < nameEnd; p++) { + char ch = contents[p]; + if (isSpaceByte((unsigned char) ch)) { + continue; + } + currentNamespace.push_back(ch >= 'A' && ch <= 'Z' ? (char) (ch - 'A' + 'a') : ch); + } + currentNamespace.push_back('\\'); + i = tail + 1; + continue; + } + } + + i++; + } +} + +/* }}} */ + +} // namespace phpstanturbo + +#endif diff --git a/turbo-ext/src/main.cpp b/turbo-ext/src/main.cpp index cdaef0609a3..9840738c1e2 100644 --- a/turbo-ext/src/main.cpp +++ b/turbo-ext/src/main.cpp @@ -97,6 +97,8 @@ static PHP_MINIT_FUNCTION(phpstan_turbo) pt_register_type_combinator_cache(); pt_register_arena_cache(); pt_register_expression_result_storage(); + pt_register_php_file_cleaner(); + pt_register_symbol_finder_in_files(); return SUCCESS; } diff --git a/turbo-ext/src/support.h b/turbo-ext/src/support.h index af8da094021..7359be4a14d 100644 --- a/turbo-ext/src/support.h +++ b/turbo-ext/src/support.h @@ -148,6 +148,8 @@ void pt_register_parser_runner(); void pt_register_type_combinator_cache(); void pt_register_arena_cache(); void pt_register_expression_result_storage(); +void pt_register_php_file_cleaner(); +void pt_register_symbol_finder_in_files(); /* per-request hooks of individual classes */ void pt_node_traverser_rinit(); diff --git a/turbo-ext/tests/php-file-cleaner-corpus.php b/turbo-ext/tests/php-file-cleaner-corpus.php new file mode 100644 index 00000000000..fa776a379f0 --- /dev/null +++ b/turbo-ext/tests/php-file-cleaner-corpus.php @@ -0,0 +1,151 @@ +clean($contents, $maxMatches); + $b = $reference->clean($contents, $maxMatches); + if ($a === $b) { + continue; + } + + $failures++; + if ($failures > 5) { + continue; + } + + $at = 0; + $min = min(strlen($a), strlen($b)); + while ($at < $min && $a[$at] === $b[$at]) { + $at++; + } + printf("FAIL: %s (maxMatches=%d) differs at byte %d (lengths %d/%d)\n", $label, $maxMatches, $at, strlen($a), strlen($b)); + printf(" native: %s\n", var_export(substr($a, max(0, $at - 40), 90), true)); + printf(" php : %s\n", var_export(substr($b, max(0, $at - 40), 90), true)); + } +}; + +// ---- synthetic fixtures ---- +// Constructs whose handling differs between the twin's regexes and the +// hand-rolled native matchers, or which the repo corpus may not contain. +$fixtures = [ + 'heredoc' => " " " " " " " " " " " " " " " " " " "\n

class NotAClass {}

\n "\n\n " "class;\n", + 'function by reference' => " " "just text, no php at all\n", + 'php tag at eof' => " " " " $source) { + $tmp = tempnam(sys_get_temp_dir(), 'pfc'); + file_put_contents($tmp, $source); + $stripped = @php_strip_whitespace($tmp); + unlink($tmp); + if ($stripped === '' || $stripped === false) { + continue; + } + $compare('fixture ' . $label, $stripped); +} +printf("fixtures: %d (file, maxMatches) pairs checked\n", $checked); + +// ---- repo corpus ---- +$corpusStart = $checked; +$files = []; +foreach (['src', 'tests', 'build', 'compiler', 'e2e', 'turbo-ext', 'vendor'] as $dir) { + if (!is_dir($root . '/' . $dir)) { + continue; + } + $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root . '/' . $dir, FilesystemIterator::SKIP_DOTS)); + foreach ($iterator as $file) { + if ($file->isFile() && in_array($file->getExtension(), ['php', 'inc', 'stub'], true)) { + $files[] = $file->getPathname(); + } + } +} +sort($files); + +foreach ($files as $file) { + $contents = @php_strip_whitespace($file); + if ($contents === '' || $contents === false) { + continue; + } + $compare($file, $contents); +} +printf("corpus: %d files, %d (file, maxMatches) pairs checked\n", count($files), $checked - $corpusStart); + +echo $failures === 0 ? "ALL OK\n" : "$failures FAILURES\n"; +exit($failures === 0 ? 0 : 1); diff --git a/turbo-ext/tests/smoke.php b/turbo-ext/tests/smoke.php index 9f3ac3291c8..4fdc3e2e9c7 100644 --- a/turbo-ext/tests/smoke.php +++ b/turbo-ext/tests/smoke.php @@ -630,6 +630,8 @@ public function enterNode(\PhpParser\Node $node) $coveredElsewhere = [ \PHPStan\Cache\ArenaCache::class => 'arena-smoke.php', \PHPStan\Parser\ParserRunner::class => 'parser-corpus.php', + \PHPStan\Reflection\BetterReflection\SourceLocator\PhpFileCleaner::class => 'php-file-cleaner-corpus.php', + \PHPStan\Reflection\BetterReflection\SourceLocator\SymbolFinderInFiles::class => 'symbol-finder-corpus.php', ]; foreach (array_keys($shadowedClasses) as $shadowedClass) { check( diff --git a/turbo-ext/tests/symbol-finder-corpus.php b/turbo-ext/tests/symbol-finder-corpus.php new file mode 100644 index 00000000000..f49b9002115 --- /dev/null +++ b/turbo-ext/tests/symbol-finder-corpus.php @@ -0,0 +1,156 @@ + prefilter -> PhpFileCleaner -> symbol regex) with +// three native passes, so the bar is identical symbol triples — over every +// PHP file in the repository plus fixtures for the constructs where the +// stages disagree with a naive reading. +// +// Run: php -d extension=.../phpstan_turbo.so turbo-ext/tests/symbol-finder-corpus.php + +$root = dirname(__DIR__, 2); +require $root . '/vendor/autoload.php'; + +if (!extension_loaded('phpstan_turbo')) { + fwrite(STDERR, "extension not loaded\n"); + exit(2); +} + +// Declares the stub subclasses before the autoloader can load the twins. +PHPStan\Turbo\TurboExtensionEnabler::enableIfLoaded(); + +use PHPStan\Reflection\BetterReflection\SourceLocator\PhpFileCleaner; +use PHPStan\Reflection\BetterReflection\SourceLocator\SymbolFinderInFiles; + +$native = new SymbolFinderInFiles(new PhpFileCleaner()); +if (get_parent_class($native) !== 'PHPStanTurbo\SymbolFinderInFiles') { + fwrite(STDERR, "the native class is not shadowing the twin — is the extension version current?\n"); + exit(2); +} + +// The references are the twins' own sources with the classes renamed, so they +// cannot drift from the files the port mirrors. +$load = static function (string $file, string $from, string $to) use ($root): void { + $source = file_get_contents($root . '/src/Reflection/BetterReflection/SourceLocator/' . $file); + $source = substr($source, strpos($source, 'final class ' . $from)); + eval(str_replace( + ['final class ' . $from, 'PhpFileCleaner $cleaner'], + ['final class ' . $to, 'ReferencePhpFileCleaner $cleaner'], + $source, + )); +}; +$load('PhpFileCleaner.php', 'PhpFileCleaner', 'ReferencePhpFileCleaner'); +$load('SymbolFinderInFiles.php', 'SymbolFinderInFiles', 'ReferenceSymbolFinderInFiles'); +$reference = new ReferenceSymbolFinderInFiles(new ReferencePhpFileCleaner()); + +$failures = 0; +$checked = 0; + +$compare = static function (array $files, string $label) use ($native, $reference, &$failures, &$checked): void { + foreach ([true, false] as $supportsEnums) { + $checked += count($files); + $a = $native->findSymbols($files, $supportsEnums); + $b = $reference->findSymbols($files, $supportsEnums); + if ($a === $b) { + continue; + } + + foreach ($files as $file) { + if (($a[$file] ?? null) === ($b[$file] ?? null)) { + continue; + } + + $failures++; + if ($failures > 10) { + continue; + } + printf("FAIL: %s%s (supportsEnums=%s)\n", $label, $file, $supportsEnums ? 'true' : 'false'); + printf(" native: %s\n", json_encode($a[$file] ?? null)); + printf(" php : %s\n", json_encode($b[$file] ?? null)); + } + } +}; + +// ---- synthetic fixtures ---- +$fixtures = [ + 'plain class' => " " " " " " " " " " " " " " " " " " " " " "\n\n", + 'heredoc' => " " " " " "\n

class NotAClass {}

\n "\n\n "plain text class NotAClass\n", + 'property named class' => " " " " "", + 'only open tag' => " $source) { + $path = sprintf('%s/fixture-%02d.php', $dir, $i++); + file_put_contents($path, $source); + $fixtureFiles[$path] = $label; +} +foreach ($fixtureFiles as $path => $label) { + $compare([$path], $label . ': '); +} +printf("fixtures: %d checks\n", $checked); +foreach (array_keys($fixtureFiles) as $path) { + @unlink($path); +} +@rmdir($dir); + +// ---- repo corpus ---- +$corpusStart = $checked; +$files = []; +foreach (['src', 'tests', 'build', 'compiler', 'e2e', 'turbo-ext', 'vendor'] as $sub) { + if (!is_dir($root . '/' . $sub)) { + continue; + } + $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root . '/' . $sub, FilesystemIterator::SKIP_DOTS)); + foreach ($iterator as $file) { + if ($file->isFile() && in_array($file->getExtension(), ['php', 'inc', 'stub'], true)) { + $files[] = $file->getPathname(); + } + } +} +sort($files); + +// batched, so the native side exercises its reusable buffers +foreach (array_chunk($files, 400) as $chunk) { + $compare($chunk, ''); +} +printf("corpus: %d files, %d checks\n", count($files), $checked - $corpusStart); + +echo $failures === 0 ? "ALL OK\n" : "$failures FAILURES\n"; +exit($failures === 0 ? 0 : 1);