Native directory symbol scan, and no cache or lock behind it

src/Parallel/ParallelAnalyser.php

@@ -15,6 +15,7 @@
 use PHPStan\DependencyInjection\AutowiredParameter;
 use PHPStan\DependencyInjection\AutowiredService;
 use PHPStan\Process\ProcessHelper;
+use PHPStan\Reflection\BetterReflection\SourceLocator\PreForkDirectorySymbolScanner;
 use React\EventLoop\LoopInterface;
 use React\Promise\Deferred;
 use React\Promise\PromiseInterface;
@@ -54,6 +55,7 @@ public function __construct(
 		#[AutowiredParameter(ref: '%parallel.buffer%')]
 		private int $decoderBufferSize,
 		private ForkParallelChecker $forkParallelChecker,
+		private PreForkDirectorySymbolScanner $preForkDirectorySymbolScanner,
 		private WorkerRunner $workerRunner,
 	)
 	{
@@ -216,6 +218,15 @@ public function analyse(
 
 		$useFork = $this->forkParallelChecker->isSupported();
 
+		if ($useFork && $numberOfProcesses > 1) {
+			// Build the directory symbol indexes here, in the parent, so the
+			// children inherit them copy-on-write instead of each scanning the
+			// same directories (see PreForkDirectorySymbolScanner). With a
+			// single worker there is nothing to share, and doing it here would
+			// only take work off the lazy path that worker might never reach.
+			$this->preForkDirectorySymbolScanner->scanBeforeFork();
+		}
+
 		for ($i = 0; $i < $numberOfProcesses; $i++) {
 			if (count($jobs) === 0) {
 				break;

src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php

@@ -63,10 +63,33 @@ public function __construct(
 		private array $functionToFiles,
 		private array $constantToFile,
 		private ?string $arenaKeyPrefix = null,
+		private bool $awaitingBatchedScan = false,
 	)
 	{
 	}
 
+	/**
+	 * Fills in the symbol maps of a locator created for a batched scan.
+	 *
+	 * The factory hands these out before the scan that produces their contents
+	 * has run, so that one scan can cover every directory at once
+	 * (see OptimizedDirectorySourceLocatorFactory::flushBatchedScan()). Nothing
+	 * may look a symbol up in between - the maps are empty, so a lookup would
+	 * quietly answer "not found" - which is what the flag guards.
+	 *
+	 * @param array<string, string> $classToFile
+	 * @param array<string, array<int, string>> $functionToFiles
+	 * @param array<string, string> $constantToFile
+	 * @internal
+	 */
+	public function fillBatchedScan(array $classToFile, array $functionToFiles, array $constantToFile): void
+	{
+		$this->classToFile = $classToFile;
+		$this->functionToFiles = $functionToFiles;
+		$this->constantToFile = $constantToFile;
+		$this->awaitingBatchedScan = false;
+	}
+
 	/**
 	 * @return array{non-empty-string, string}
 	 */
@@ -86,6 +109,10 @@ private function getCacheKeys(string $file, Identifier $identifier): array
 	#[Override]
 	public function locateIdentifier(Reflector $reflector, Identifier $identifier): ?Reflection
 	{
+		if ($this->awaitingBatchedScan) {
+			throw new ShouldNotHappenException('Symbols were looked up in a directory whose batched scan has not been flushed yet.');
+		}
+
 		if ($identifier->isClass()) {
 			$identifierName = strtolower($identifier->getName());
 			$file = $this->findFileByClass($identifierName);
@@ -334,6 +361,10 @@ private function hydrateSymbolsFromArena(): void
 	#[Override]
 	public function locateIdentifiersByType(Reflector $reflector, IdentifierType $identifierType): array
 	{
+		if ($this->awaitingBatchedScan) {
+			throw new ShouldNotHappenException('Symbols were looked up in a directory whose batched scan has not been flushed yet.');
+		}
+
 		$this->hydrateSymbolsFromArena();
 
 		$reflections = [];

src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php

@@ -10,9 +10,12 @@
 use PHPStan\File\FileFinder;
 use PHPStan\Internal\DirectoryCreator;
 use PHPStan\Internal\DirectoryCreatorException;
+use PHPStan\Parallel\ForkParallelChecker;
 use PHPStan\Php\PhpVersion;
+use PHPStan\Turbo\TurboExtensionEnabler;
 use function array_key_exists;
 use function array_keys;
+use function array_values;
 use function fclose;
 use function flock;
 use function fopen;
@@ -50,6 +53,13 @@ final class OptimizedDirectorySourceLocatorFactory
 	 */
 	private const HASH_LOCK_POLL_INTERVAL_MICROSECONDS = 5_000;
 
+	/**
+	 * Directories collected for a batched scan, null when not batching.
+	 *
+	 * @var list<array{string[], OptimizedDirectorySourceLocator}>|null
+	 */
+	private ?array $batchedScan = null;
+
 	public function __construct(
 		private FileNodesFetcher $fileNodesFetcher,
 		#[AutowiredParameter(ref: '@fileFinderScan')]
@@ -58,6 +68,7 @@ public function __construct(
 		private SymbolFinderInFiles $symbolFinderInFiles,
 		private Cache $cache,
 		private FileContentHasher $fileContentHasher,
+		private ForkParallelChecker $forkParallelChecker,
 		#[AutowiredParameter]
 		private string $tmpDir,
 	)
@@ -66,6 +77,10 @@ public function __construct(
 
 	public function createByDirectory(string $directory): OptimizedDirectorySourceLocator
 	{
+		if ($this->scansFresh()) {
+			return $this->createFreshDirectorySourceLocator($directory);
+		}
+
 		$cacheKey = sprintf('odsl-%s', $directory);
 		$hashesRecordKey = 'odsl-filehashes-' . $directory;
 
@@ -126,6 +141,119 @@ public function createByDirectory(string $directory): OptimizedDirectorySourceLo
 		return $this->createCachedDirectorySourceLocator($fileHashes, $cacheKey);
 	}
 
+	/**
+	 * Whether the symbol index is built outright instead of being cached.
+	 *
+	 * Both halves are needed. The native scan is what makes the cache not
+	 * worth its keep, and forking is what keeps the scan from happening once
+	 * per worker: the parent scans before it forks and the children inherit
+	 * the result (see PreForkDirectorySymbolScanner). Where the extension is
+	 * active but workers are spawned rather than forked - Windows, or OPcache
+	 * left on - there is nothing to inherit, so the cache and its scan lock
+	 * stay in charge.
+	 */
+	private function scansFresh(): bool
+	{
+		return TurboExtensionEnabler::isActive() && $this->forkParallelChecker->isSupported();
+	}
+
+	/**
+	 * With the turbo extension the symbol scan is native and costs about what
+	 * hashing the directory to validate a cache costs, so a cache has nothing
+	 * left to save: the directory is walked and scanned outright, with no file
+	 * hashing, no persisted symbol table, no scan lock and no arena record —
+	 * and therefore no cache that can go stale. PreForkDirectoryScanner runs
+	 * this once in the main process before it forks its workers, so every
+	 * worker inherits the finished locators instead of racing to build them.
+	 */
+	private function createFreshDirectorySourceLocator(string $directory): OptimizedDirectorySourceLocator
+	{
+		return $this->createFreshFileListSourceLocator($this->fileFinder->findFiles([$directory])->getFiles());
+	}
+
+	/**
+	 * Starts collecting the directories asked for instead of scanning each one
+	 * as it comes, so that flushBatchedScan() can cover all of them in a single
+	 * scan: a file reachable from two directories is read once rather than
+	 * twice, and the per-call costs are paid once instead of per directory.
+	 */
+	public function beginBatchedScan(): void
+	{
+		$this->batchedScan = [];
+	}
+
+	/**
+	 * Scans everything collected since beginBatchedScan() at once and fills in
+	 * the locators handed out in the meantime.
+	 */
+	public function flushBatchedScan(): void
+	{
+		$batched = $this->batchedScan;
+		$this->batchedScan = null;
+		if ($batched === null || $batched === []) {
+			return;
+		}
+
+		$allFiles = [];
+		foreach ($batched as [$files]) {
+			foreach ($files as $file) {
+				// a file reachable from two directories is scanned once
+				$allFiles[$file] = $file;
+			}
+		}
+
+		$symbols = $this->symbolFinderInFiles->findSymbols(array_values($allFiles), $this->phpVersion->supportsEnums());
+
+		foreach ($batched as [$files, $locator]) {
+			$directorySymbols = [];
+			foreach ($files as $file) {
+				if (!array_key_exists($file, $symbols)) {
+					continue;
+				}
+
+				$directorySymbols[$file] = $symbols[$file];
+			}
+
+			[$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($directorySymbols);
+			$locator->fillBatchedScan($classToFile, $functionToFiles, $constantToFile);
+		}
+	}
+
+	/**
+	 * @param string[] $files
+	 */
+	private function createFreshFileListSourceLocator(array $files): OptimizedDirectorySourceLocator
+	{
+		if ($this->batchedScan !== null) {
+			$locator = new OptimizedDirectorySourceLocator(
+				$this->fileNodesFetcher,
+				$this->cache,
+				$this->phpVersion,
+				$this->fileContentHasher,
+				[],
+				[],
+				[],
+				awaitingBatchedScan: true,
+			);
+			$this->batchedScan[] = [$files, $locator];
+
+			return $locator;
+		}
+
+		$symbols = $this->symbolFinderInFiles->findSymbols($files, $this->phpVersion->supportsEnums());
+		[$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($symbols);
+
+		return new OptimizedDirectorySourceLocator(
+			$this->fileNodesFetcher,
+			$this->cache,
+			$this->phpVersion,
+			$this->fileContentHasher,
+			$classToFile,
+			$functionToFiles,
+			$constantToFile,
+		);
+	}
+
 	/**
 	 * @param array<string, string> $fileHashes
 	 * @param non-empty-string $cacheKey
@@ -233,7 +361,12 @@ private function createCachedDirectorySourceLocator(array $fileHashes, string $c
 			}
 		}
 
-		[$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($cached);
+		$symbols = [];
+		foreach ($cached as $file => [, $classes, $functions, $constants]) {
+			$symbols[$file] = [$classes, $functions, $constants];
+		}
+
+		[$classToFile, $functionToFiles, $constantToFile] = $this->changeStructure($symbols);
 
 		// Publication order matters: the reader above requires all three
 		// records, so a partially-published index is never consumed.
@@ -322,6 +455,10 @@ private function releaseDirectoryScanLock($lockHandle): void
 	 */
 	public function createByFiles(array $files, string $uniqueCacheIdentifier): OptimizedDirectorySourceLocator
 	{
+		if ($this->scansFresh()) {
+			return $this->createFreshFileListSourceLocator($files);
+		}
+
 		$fileHashes = [];
 		foreach ($files as $file) {
 			$hash = $this->fileContentHasher->hash($file);
@@ -335,15 +472,15 @@ public function createByFiles(array $files, string $uniqueCacheIdentifier): Opti
 	}
 
 	/**
-	 * @param array<string, array{string, string[], string[], string[]}> $symbols
+	 * @param array<string, array{string[], string[], string[]}> $symbols
 	 * @return array{array<string, string>, array<string, array<int, string>>, array<string, string>}
 	 */
 	private function changeStructure(array $symbols): array
 	{
 		$classToFile = [];
 		$constantToFile = [];
 		$functionToFiles = [];
-		foreach ($symbols as $file => [, $classes, $functions, $constants]) {
+		foreach ($symbols as $file => [$classes, $functions, $constants]) {
 			foreach ($classes as $classInFile) {
 				$classToFile[$classInFile] = $file;
 			}

src/Reflection/BetterReflection/SourceLocator/PhpFileCleaner.php

@@ -3,6 +3,7 @@
 namespace PHPStan\Reflection\BetterReflection\SourceLocator;
 
 use PHPStan\DependencyInjection\AutowiredService;
+use PHPStan\Turbo\ShadowedByTurboExtension;
 use function array_keys;
 use function implode;
 use function in_array;
@@ -18,6 +19,7 @@
  * @see https://github.com/composer/composer/pull/10107
  */
 #[AutowiredService]
+#[ShadowedByTurboExtension(turboClass: 'PHPStanTurbo\PhpFileCleaner', implementation: __DIR__ . '/../../../../turbo-ext/src/PhpFileCleaner.cpp')]
 final class PhpFileCleaner
 {