From 403136290e5b7a5cd76b431ca62a2c72b49b5acb Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Fri, 28 Aug 2026 00:12:52 +0200 Subject: [PATCH] Report peak memory per process instead of a summed total The printed number was the sum of every worker's peak plus a snapshot of the main process's current usage, which described a moment that never happens: workers do not peak at the same time, and the main process's own peak comes later still, while it collects the workers' results and saves the result cache. The snapshot understated it threefold (136 MB recorded against a 405 MB peak on a self-analysis run). Fork made it worse: memory_get_peak_usage() carries over into a pcntl_fork()-ed child, so every worker reported the main process's peak before allocating anything of its own. Summed across workers that multiplied a spike the main process took before forking - loading the result cache on an incremental run - by the number of workers, and one project displayed 29 GB while no process ever exceeded 3.2 GB. Report what each process actually reached instead: this process's peak, read at the very end, and the heaviest worker's - which is also the number memory_limit applies to, since it is a per-process limit. The forked child restarts its peak tracking so the figure is its own. Peak memory: 3.24 GB (main process), 1.41 GB (largest of 10 forked workers) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014fMtimQECyyBNKTmFddr8L --- src/Analyser/Analyser.php | 4 +-- src/Analyser/AnalyserResult.php | 10 +++++++ src/Analyser/AnalyserResultFinalizer.php | 3 +++ src/Command/AnalyseApplication.php | 8 ++++-- src/Command/AnalyseCommand.php | 14 +++++----- src/Command/AnalyserRunner.php | 3 +-- src/Command/AnalysisResult.php | 10 +++++++ src/Command/InceptionResult.php | 33 +++++++++++++++++++----- src/Parallel/ForkedProcess.php | 12 +++++++++ src/Parallel/ParallelAnalyser.php | 17 +++++++----- 10 files changed, 89 insertions(+), 25 deletions(-) diff --git a/src/Analyser/Analyser.php b/src/Analyser/Analyser.php index 9b4e4a7460f..fe1716f5d65 100644 --- a/src/Analyser/Analyser.php +++ b/src/Analyser/Analyser.php @@ -12,7 +12,6 @@ use function array_fill_keys; use function array_merge; use function count; -use function memory_get_peak_usage; /** * @phpstan-import-type CollectorData from CollectedData @@ -147,7 +146,8 @@ public function analyse( packageDependencies: $internalErrorsCount === 0 ? $packageDependencies : null, exportedNodes: $exportedNodes, reachedInternalErrorsCountLimit: $reachedInternalErrorsCountLimit, - peakMemoryUsageBytes: memory_get_peak_usage(true), + // analysis ran in this process - its peak is read where the number is printed + peakMemoryUsageBytes: 0, processedFiles: $allProcessedFiles, ); } diff --git a/src/Analyser/AnalyserResult.php b/src/Analyser/AnalyserResult.php index f9cb1bdef66..78a44ae2601 100644 --- a/src/Analyser/AnalyserResult.php +++ b/src/Analyser/AnalyserResult.php @@ -47,10 +47,20 @@ public function __construct( private bool $reachedInternalErrorsCountLimit, private int $peakMemoryUsageBytes, private array $processedFiles, + private int $workerCount = 0, ) { } + /** + * How many parallel workers produced this result; 0 when the analysis ran in + * the main process. + */ + public function getWorkerCount(): int + { + return $this->workerCount; + } + /** * @return list */ diff --git a/src/Analyser/AnalyserResultFinalizer.php b/src/Analyser/AnalyserResultFinalizer.php index 1688b4fd2f3..fd883666f87 100644 --- a/src/Analyser/AnalyserResultFinalizer.php +++ b/src/Analyser/AnalyserResultFinalizer.php @@ -156,6 +156,7 @@ public function finalize(AnalyserResult $analyserResult, bool $onlyFiles, bool $ reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(), peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(), processedFiles: $analyserResult->getProcessedFiles(), + workerCount: $analyserResult->getWorkerCount(), ), $collectorErrors, $locallyIgnoredCollectorErrors); } @@ -177,6 +178,7 @@ private function mergeFilteredPhpErrors(AnalyserResult $analyserResult): Analyse reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(), peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(), processedFiles: $analyserResult->getProcessedFiles(), + workerCount: $analyserResult->getWorkerCount(), ); } @@ -243,6 +245,7 @@ private function addUnmatchedIgnoredErrors( reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(), peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(), processedFiles: $analyserResult->getProcessedFiles(), + workerCount: $analyserResult->getWorkerCount(), ), $collectorErrors, $locallyIgnoredCollectorErrors, diff --git a/src/Command/AnalyseApplication.php b/src/Command/AnalyseApplication.php index 1f437b604d0..a474871a380 100644 --- a/src/Command/AnalyseApplication.php +++ b/src/Command/AnalyseApplication.php @@ -82,7 +82,8 @@ public function analyse( $internalErrors = []; $collectedData = []; $savedResultCache = false; - $memoryUsageBytes = memory_get_peak_usage(true); + $memoryUsageBytes = 0; + $workerCount = 0; $processedFiles = []; if ($errorOutput->isVeryVerbose()) { $errorOutput->writeLineFormatted('Result cache was not saved because of ignoredErrorHelperResult errors.'); @@ -127,6 +128,7 @@ public function analyse( reachedInternalErrorsCountLimit: $intermediateAnalyserResult->hasReachedInternalErrorsCountLimit(), peakMemoryUsageBytes: $intermediateAnalyserResult->getPeakMemoryUsageBytes(), processedFiles: $intermediateAnalyserResult->getProcessedFiles(), + workerCount: $intermediateAnalyserResult->getWorkerCount(), ); } @@ -145,6 +147,7 @@ public function analyse( ); $hasInternalErrors = count($internalErrors) > 0 || $analyserResult->hasReachedInternalErrorsCountLimit(); $memoryUsageBytes = $analyserResult->getPeakMemoryUsageBytes(); + $workerCount = $analyserResult->getWorkerCount(); $isResultCacheUsed = !$resultCache->isFullAnalysis(); $changedProjectExtensionFilesOutsideOfAnalysedPaths = []; @@ -194,6 +197,7 @@ public function analyse( $changedProjectExtensionFilesOutsideOfAnalysedPaths, $processedFiles, $resultCacheExisted, + $workerCount, ); } @@ -255,7 +259,7 @@ private function runAnalyser( packageDependencies: [], exportedNodes: [], reachedInternalErrorsCountLimit: false, - peakMemoryUsageBytes: memory_get_peak_usage(true), + peakMemoryUsageBytes: 0, processedFiles: [], ); } diff --git a/src/Command/AnalyseCommand.php b/src/Command/AnalyseCommand.php index 7bd63baf085..8f984fe98fc 100644 --- a/src/Command/AnalyseCommand.php +++ b/src/Command/AnalyseCommand.php @@ -496,7 +496,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int count($internalErrors) === 1 ? 'An internal error' : 'Internal errors', )); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } return $this->generateBaseline($generateBaselineFile, $inceptionResult, $analysisResult, $output, $allowEmptyBaseline, $baselineExtension, $failWithoutResultCache); @@ -535,6 +535,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, + $analysisResult->getWorkerCount(), ); } @@ -672,7 +673,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $errorOutput->writeLineFormatted(''); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } } @@ -684,6 +685,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, + $analysisResult->getWorkerCount(), ); } @@ -798,7 +800,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult $inceptionResult->getStdOutput()->getStyle()->error('No errors were found during the analysis. Baseline could not be generated.'); $inceptionResult->getStdOutput()->writeLineFormatted('To allow generating empty baselines, pass --allow-empty-baseline option.'); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } $streamOutput = $this->createStreamOutput(); @@ -826,7 +828,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult } catch (DirectoryCreatorException $e) { $inceptionResult->getStdOutput()->writeLineFormatted($e->getMessage()); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } try { @@ -834,7 +836,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult } catch (CouldNotWriteFileException $e) { $inceptionResult->getStdOutput()->writeLineFormatted($e->getMessage()); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } $errorsCount = 0; @@ -874,7 +876,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult $exitCode = 2; } - return $inceptionResult->handleReturn($exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn($exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } /** diff --git a/src/Command/AnalyserRunner.php b/src/Command/AnalyserRunner.php index be77e56245c..c86842aaf70 100644 --- a/src/Command/AnalyserRunner.php +++ b/src/Command/AnalyserRunner.php @@ -19,7 +19,6 @@ use function filesize; use function function_exists; use function is_file; -use function memory_get_peak_usage; #[AutowiredService] final class AnalyserRunner @@ -72,7 +71,7 @@ public function runAnalyser( packageDependencies: [], exportedNodes: [], reachedInternalErrorsCountLimit: false, - peakMemoryUsageBytes: memory_get_peak_usage(true), + peakMemoryUsageBytes: 0, processedFiles: [], ); } diff --git a/src/Command/AnalysisResult.php b/src/Command/AnalysisResult.php index 3a100dbc4de..6b1ecec279a 100644 --- a/src/Command/AnalysisResult.php +++ b/src/Command/AnalysisResult.php @@ -40,6 +40,7 @@ public function __construct( private array $changedProjectExtensionFilesOutsideOfAnalysedPaths, private array $processedFiles = [], private bool $resultCacheExisted = true, + private int $workerCount = 0, ) { usort( @@ -138,6 +139,15 @@ public function getPeakMemoryUsageBytes(): int return $this->peakMemoryUsageBytes; } + /** + * How many parallel workers produced this result; 0 when the analysis ran in + * the main process. + */ + public function getWorkerCount(): int + { + return $this->workerCount; + } + public function isResultCacheUsed(): bool { return $this->isResultCacheUsed; diff --git a/src/Command/InceptionResult.php b/src/Command/InceptionResult.php index a3e6a7afdbd..0999d63329b 100644 --- a/src/Command/InceptionResult.php +++ b/src/Command/InceptionResult.php @@ -5,9 +5,9 @@ use PHPStan\DependencyInjection\Container; use PHPStan\File\PathNotFoundException; use PHPStan\Internal\BytesHelper; +use PHPStan\Parallel\ForkParallelChecker; use function floor; use function implode; -use function max; use function memory_get_peak_usage; use function microtime; use function round; @@ -100,7 +100,11 @@ public function getEditorModeInsteadOfFile(): ?string return $this->editorModeInsteadOfFile; } - public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $analysisStartTime): int + /** + * @param int|null $peakMemoryUsageBytes the heaviest parallel worker's peak, 0 when the + * analysis ran in this process + */ + public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $analysisStartTime, int $workerCount = 0): int { if ($this->getErrorOutput()->isVerbose()) { $elapsedTime = round(microtime(true) - $analysisStartTime, 2); @@ -116,10 +120,27 @@ public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $a } if ($peakMemoryUsageBytes !== null && $this->getErrorOutput()->isVerbose()) { - $this->getErrorOutput()->writeLineFormatted(sprintf( - 'Used memory: %s', - BytesHelper::bytes(max(memory_get_peak_usage(true), $peakMemoryUsageBytes)), - )); + // This process's peak is read here, at the very end, so it covers collecting + // the workers' results and saving the result cache - the part of a parallel + // run where the main process is at its largest. Both numbers are per process, + // which is also how memory_limit applies. + $mainProcessPeak = memory_get_peak_usage(true); + if ($peakMemoryUsageBytes === 0 || $workerCount === 0) { + $this->getErrorOutput()->writeLineFormatted(sprintf( + 'Peak memory: %s', + BytesHelper::bytes($mainProcessPeak), + )); + } else { + $mechanism = $this->container->getByType(ForkParallelChecker::class)->isSupported() ? 'forked' : 'spawned'; + $this->getErrorOutput()->writeLineFormatted(sprintf( + 'Peak memory: %s (main process), %s (%s)', + BytesHelper::bytes($mainProcessPeak), + BytesHelper::bytes($peakMemoryUsageBytes), + $workerCount === 1 + ? sprintf('the %s worker', $mechanism) + : sprintf('largest of %d %s workers', $workerCount, $mechanism), + )); + } } return $exitCode; diff --git a/src/Parallel/ForkedProcess.php b/src/Parallel/ForkedProcess.php index 23895f89b00..d95101d94a6 100644 --- a/src/Parallel/ForkedProcess.php +++ b/src/Parallel/ForkedProcess.php @@ -11,6 +11,7 @@ use Symfony\Component\Console\Output\StreamOutput; use Throwable; use function fclose; +use function function_exists; use function pcntl_fork; use function pcntl_waitpid; use function pcntl_wexitstatus; @@ -97,6 +98,17 @@ public function start(callable $onData, callable $onError, callable $onExit): vo // Child: drop the inherited listening socket immediately, then run // the worker on its own fresh event loop and never return. $this->server->close(); + // memory_get_peak_usage() carries over into the child, so without this a + // worker would report the main process's peak instead of its own - on an + // incremental run, the spike taken while loading the result cache, which + // every worker would then repeat. Restarting the peak here keeps the + // reported number the worker's own high-water usage, inherited memory it + // still holds included. + /** phpcs:disable SlevomatCodingStandard.Namespaces.ReferenceUsedNamesOnly.ReferenceViaFullyQualifiedName */ + if (function_exists('memory_reset_peak_usage')) { + \memory_reset_peak_usage(); + } + /** phpcs:enable */ ForkedChildCrashReporter::install($tmpStdErr); $output = new StreamOutput($tmpStdOut); try { diff --git a/src/Parallel/ParallelAnalyser.php b/src/Parallel/ParallelAnalyser.php index 48be3c07d73..e3aa2ca36c9 100644 --- a/src/Parallel/ParallelAnalyser.php +++ b/src/Parallel/ParallelAnalyser.php @@ -25,14 +25,12 @@ use function array_map; use function array_pop; use function array_reverse; -use function array_sum; use function count; use function defined; use function escapeshellarg; use function getenv; use function ini_get; use function max; -use function memory_get_usage; use function parse_url; use function sprintf; use function str_contains; @@ -110,6 +108,7 @@ public function analyse( $locallyIgnoredErrors = []; $linesToIgnore = []; $unmatchedLineIgnores = []; + /** @var array $peakMemoryUsages */ $peakMemoryUsages = []; $internalErrors = []; $internalErrorsCount = 0; @@ -156,8 +155,12 @@ public function analyse( packageDependencies: $internalErrorsCount === 0 ? $packageDependencies : null, exportedNodes: $exportedNodes, reachedInternalErrorsCountLimit: $reachedInternalErrorsCountLimit, - peakMemoryUsageBytes: array_sum($peakMemoryUsages), // not 100% correct as the peak usages of workers might not have met + // The heaviest single worker. Summing the workers' peaks would describe a + // moment that never happens - they do not peak at the same time - while + // each worker's own peak is what its memory_limit is measured against. + peakMemoryUsageBytes: $peakMemoryUsages === [] ? 0 : max($peakMemoryUsages), processedFiles: $allProcessedFiles, + workerCount: count($peakMemoryUsages), )); }); $server->on('connection', function (ConnectionInterface $connection) use (&$jobs, $arenaName, $expectedWorkerCount, &$helloCount): void { @@ -372,10 +375,10 @@ public function analyse( $job = array_pop($jobs); $process->request(['action' => 'analyse', 'files' => $job]); - }, $handleError, function ($exitCode, string $output) use (&$someChildEnded, &$peakMemoryUsages, &$internalErrors, &$internalErrorsCount, $processIdentifier): void { - if ($someChildEnded === false) { - $peakMemoryUsages['main'] = memory_get_usage(true); - } + }, $handleError, function ($exitCode, string $output) use (&$someChildEnded, &$internalErrors, &$internalErrorsCount, $processIdentifier): void { + // The main process is not sampled here any more: its own peak comes + // later (collecting the workers' results, saving the result cache) and + // is read where the number is printed. Only worker peaks are summed. $someChildEnded = true; if ($exitCode === 0) {