diff --git a/src/Analyser/Analyser.php b/src/Analyser/Analyser.php index 9b4e4a7460..fe1716f5d6 100644 --- a/src/Analyser/Analyser.php +++ b/src/Analyser/Analyser.php @@ -12,7 +12,6 @@ use function array_fill_keys; use function array_merge; use function count; -use function memory_get_peak_usage; /** * @phpstan-import-type CollectorData from CollectedData @@ -147,7 +146,8 @@ public function analyse( packageDependencies: $internalErrorsCount === 0 ? $packageDependencies : null, exportedNodes: $exportedNodes, reachedInternalErrorsCountLimit: $reachedInternalErrorsCountLimit, - peakMemoryUsageBytes: memory_get_peak_usage(true), + // analysis ran in this process - its peak is read where the number is printed + peakMemoryUsageBytes: 0, processedFiles: $allProcessedFiles, ); } diff --git a/src/Analyser/AnalyserResult.php b/src/Analyser/AnalyserResult.php index f9cb1bdef6..78a44ae260 100644 --- a/src/Analyser/AnalyserResult.php +++ b/src/Analyser/AnalyserResult.php @@ -47,10 +47,20 @@ public function __construct( private bool $reachedInternalErrorsCountLimit, private int $peakMemoryUsageBytes, private array $processedFiles, + private int $workerCount = 0, ) { } + /** + * How many parallel workers produced this result; 0 when the analysis ran in + * the main process. + */ + public function getWorkerCount(): int + { + return $this->workerCount; + } + /** * @return list */ diff --git a/src/Analyser/AnalyserResultFinalizer.php b/src/Analyser/AnalyserResultFinalizer.php index 1688b4fd2f..fd883666f8 100644 --- a/src/Analyser/AnalyserResultFinalizer.php +++ b/src/Analyser/AnalyserResultFinalizer.php @@ -156,6 +156,7 @@ public function finalize(AnalyserResult $analyserResult, bool $onlyFiles, bool $ reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(), peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(), processedFiles: $analyserResult->getProcessedFiles(), + workerCount: $analyserResult->getWorkerCount(), ), $collectorErrors, $locallyIgnoredCollectorErrors); } @@ -177,6 +178,7 @@ private function mergeFilteredPhpErrors(AnalyserResult $analyserResult): Analyse reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(), peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(), processedFiles: $analyserResult->getProcessedFiles(), + workerCount: $analyserResult->getWorkerCount(), ); } @@ -243,6 +245,7 @@ private function addUnmatchedIgnoredErrors( reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(), peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(), processedFiles: $analyserResult->getProcessedFiles(), + workerCount: $analyserResult->getWorkerCount(), ), $collectorErrors, $locallyIgnoredCollectorErrors, diff --git a/src/Command/AnalyseApplication.php b/src/Command/AnalyseApplication.php index 1f437b604d..a474871a38 100644 --- a/src/Command/AnalyseApplication.php +++ b/src/Command/AnalyseApplication.php @@ -82,7 +82,8 @@ public function analyse( $internalErrors = []; $collectedData = []; $savedResultCache = false; - $memoryUsageBytes = memory_get_peak_usage(true); + $memoryUsageBytes = 0; + $workerCount = 0; $processedFiles = []; if ($errorOutput->isVeryVerbose()) { $errorOutput->writeLineFormatted('Result cache was not saved because of ignoredErrorHelperResult errors.'); @@ -127,6 +128,7 @@ public function analyse( reachedInternalErrorsCountLimit: $intermediateAnalyserResult->hasReachedInternalErrorsCountLimit(), peakMemoryUsageBytes: $intermediateAnalyserResult->getPeakMemoryUsageBytes(), processedFiles: $intermediateAnalyserResult->getProcessedFiles(), + workerCount: $intermediateAnalyserResult->getWorkerCount(), ); } @@ -145,6 +147,7 @@ public function analyse( ); $hasInternalErrors = count($internalErrors) > 0 || $analyserResult->hasReachedInternalErrorsCountLimit(); $memoryUsageBytes = $analyserResult->getPeakMemoryUsageBytes(); + $workerCount = $analyserResult->getWorkerCount(); $isResultCacheUsed = !$resultCache->isFullAnalysis(); $changedProjectExtensionFilesOutsideOfAnalysedPaths = []; @@ -194,6 +197,7 @@ public function analyse( $changedProjectExtensionFilesOutsideOfAnalysedPaths, $processedFiles, $resultCacheExisted, + $workerCount, ); } @@ -255,7 +259,7 @@ private function runAnalyser( packageDependencies: [], exportedNodes: [], reachedInternalErrorsCountLimit: false, - peakMemoryUsageBytes: memory_get_peak_usage(true), + peakMemoryUsageBytes: 0, processedFiles: [], ); } diff --git a/src/Command/AnalyseCommand.php b/src/Command/AnalyseCommand.php index 7bd63baf08..8f984fe98f 100644 --- a/src/Command/AnalyseCommand.php +++ b/src/Command/AnalyseCommand.php @@ -496,7 +496,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int count($internalErrors) === 1 ? 'An internal error' : 'Internal errors', )); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } return $this->generateBaseline($generateBaselineFile, $inceptionResult, $analysisResult, $output, $allowEmptyBaseline, $baselineExtension, $failWithoutResultCache); @@ -535,6 +535,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, + $analysisResult->getWorkerCount(), ); } @@ -672,7 +673,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $errorOutput->writeLineFormatted(''); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } } @@ -684,6 +685,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, + $analysisResult->getWorkerCount(), ); } @@ -798,7 +800,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult $inceptionResult->getStdOutput()->getStyle()->error('No errors were found during the analysis. Baseline could not be generated.'); $inceptionResult->getStdOutput()->writeLineFormatted('To allow generating empty baselines, pass --allow-empty-baseline option.'); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } $streamOutput = $this->createStreamOutput(); @@ -826,7 +828,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult } catch (DirectoryCreatorException $e) { $inceptionResult->getStdOutput()->writeLineFormatted($e->getMessage()); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } try { @@ -834,7 +836,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult } catch (CouldNotWriteFileException $e) { $inceptionResult->getStdOutput()->writeLineFormatted($e->getMessage()); - return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } $errorsCount = 0; @@ -874,7 +876,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult $exitCode = 2; } - return $inceptionResult->handleReturn($exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime); + return $inceptionResult->handleReturn($exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount()); } /** diff --git a/src/Command/AnalyserRunner.php b/src/Command/AnalyserRunner.php index be77e56245..c86842aaf7 100644 --- a/src/Command/AnalyserRunner.php +++ b/src/Command/AnalyserRunner.php @@ -19,7 +19,6 @@ use function filesize; use function function_exists; use function is_file; -use function memory_get_peak_usage; #[AutowiredService] final class AnalyserRunner @@ -72,7 +71,7 @@ public function runAnalyser( packageDependencies: [], exportedNodes: [], reachedInternalErrorsCountLimit: false, - peakMemoryUsageBytes: memory_get_peak_usage(true), + peakMemoryUsageBytes: 0, processedFiles: [], ); } diff --git a/src/Command/AnalysisResult.php b/src/Command/AnalysisResult.php index 3a100dbc4d..6b1ecec279 100644 --- a/src/Command/AnalysisResult.php +++ b/src/Command/AnalysisResult.php @@ -40,6 +40,7 @@ public function __construct( private array $changedProjectExtensionFilesOutsideOfAnalysedPaths, private array $processedFiles = [], private bool $resultCacheExisted = true, + private int $workerCount = 0, ) { usort( @@ -138,6 +139,15 @@ public function getPeakMemoryUsageBytes(): int return $this->peakMemoryUsageBytes; } + /** + * How many parallel workers produced this result; 0 when the analysis ran in + * the main process. + */ + public function getWorkerCount(): int + { + return $this->workerCount; + } + public function isResultCacheUsed(): bool { return $this->isResultCacheUsed; diff --git a/src/Command/InceptionResult.php b/src/Command/InceptionResult.php index a3e6a7afdb..0999d63329 100644 --- a/src/Command/InceptionResult.php +++ b/src/Command/InceptionResult.php @@ -5,9 +5,9 @@ use PHPStan\DependencyInjection\Container; use PHPStan\File\PathNotFoundException; use PHPStan\Internal\BytesHelper; +use PHPStan\Parallel\ForkParallelChecker; use function floor; use function implode; -use function max; use function memory_get_peak_usage; use function microtime; use function round; @@ -100,7 +100,11 @@ public function getEditorModeInsteadOfFile(): ?string return $this->editorModeInsteadOfFile; } - public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $analysisStartTime): int + /** + * @param int|null $peakMemoryUsageBytes the heaviest parallel worker's peak, 0 when the + * analysis ran in this process + */ + public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $analysisStartTime, int $workerCount = 0): int { if ($this->getErrorOutput()->isVerbose()) { $elapsedTime = round(microtime(true) - $analysisStartTime, 2); @@ -116,10 +120,27 @@ public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $a } if ($peakMemoryUsageBytes !== null && $this->getErrorOutput()->isVerbose()) { - $this->getErrorOutput()->writeLineFormatted(sprintf( - 'Used memory: %s', - BytesHelper::bytes(max(memory_get_peak_usage(true), $peakMemoryUsageBytes)), - )); + // This process's peak is read here, at the very end, so it covers collecting + // the workers' results and saving the result cache - the part of a parallel + // run where the main process is at its largest. Both numbers are per process, + // which is also how memory_limit applies. + $mainProcessPeak = memory_get_peak_usage(true); + if ($peakMemoryUsageBytes === 0 || $workerCount === 0) { + $this->getErrorOutput()->writeLineFormatted(sprintf( + 'Peak memory: %s', + BytesHelper::bytes($mainProcessPeak), + )); + } else { + $mechanism = $this->container->getByType(ForkParallelChecker::class)->isSupported() ? 'forked' : 'spawned'; + $this->getErrorOutput()->writeLineFormatted(sprintf( + 'Peak memory: %s (main process), %s (%s)', + BytesHelper::bytes($mainProcessPeak), + BytesHelper::bytes($peakMemoryUsageBytes), + $workerCount === 1 + ? sprintf('the %s worker', $mechanism) + : sprintf('largest of %d %s workers', $workerCount, $mechanism), + )); + } } return $exitCode; diff --git a/src/Parallel/ForkedProcess.php b/src/Parallel/ForkedProcess.php index 23895f89b0..d95101d94a 100644 --- a/src/Parallel/ForkedProcess.php +++ b/src/Parallel/ForkedProcess.php @@ -11,6 +11,7 @@ use Symfony\Component\Console\Output\StreamOutput; use Throwable; use function fclose; +use function function_exists; use function pcntl_fork; use function pcntl_waitpid; use function pcntl_wexitstatus; @@ -97,6 +98,17 @@ public function start(callable $onData, callable $onError, callable $onExit): vo // Child: drop the inherited listening socket immediately, then run // the worker on its own fresh event loop and never return. $this->server->close(); + // memory_get_peak_usage() carries over into the child, so without this a + // worker would report the main process's peak instead of its own - on an + // incremental run, the spike taken while loading the result cache, which + // every worker would then repeat. Restarting the peak here keeps the + // reported number the worker's own high-water usage, inherited memory it + // still holds included. + /** phpcs:disable SlevomatCodingStandard.Namespaces.ReferenceUsedNamesOnly.ReferenceViaFullyQualifiedName */ + if (function_exists('memory_reset_peak_usage')) { + \memory_reset_peak_usage(); + } + /** phpcs:enable */ ForkedChildCrashReporter::install($tmpStdErr); $output = new StreamOutput($tmpStdOut); try { diff --git a/src/Parallel/ParallelAnalyser.php b/src/Parallel/ParallelAnalyser.php index 48be3c07d7..e3aa2ca36c 100644 --- a/src/Parallel/ParallelAnalyser.php +++ b/src/Parallel/ParallelAnalyser.php @@ -25,14 +25,12 @@ use function array_map; use function array_pop; use function array_reverse; -use function array_sum; use function count; use function defined; use function escapeshellarg; use function getenv; use function ini_get; use function max; -use function memory_get_usage; use function parse_url; use function sprintf; use function str_contains; @@ -110,6 +108,7 @@ public function analyse( $locallyIgnoredErrors = []; $linesToIgnore = []; $unmatchedLineIgnores = []; + /** @var array $peakMemoryUsages */ $peakMemoryUsages = []; $internalErrors = []; $internalErrorsCount = 0; @@ -156,8 +155,12 @@ public function analyse( packageDependencies: $internalErrorsCount === 0 ? $packageDependencies : null, exportedNodes: $exportedNodes, reachedInternalErrorsCountLimit: $reachedInternalErrorsCountLimit, - peakMemoryUsageBytes: array_sum($peakMemoryUsages), // not 100% correct as the peak usages of workers might not have met + // The heaviest single worker. Summing the workers' peaks would describe a + // moment that never happens - they do not peak at the same time - while + // each worker's own peak is what its memory_limit is measured against. + peakMemoryUsageBytes: $peakMemoryUsages === [] ? 0 : max($peakMemoryUsages), processedFiles: $allProcessedFiles, + workerCount: count($peakMemoryUsages), )); }); $server->on('connection', function (ConnectionInterface $connection) use (&$jobs, $arenaName, $expectedWorkerCount, &$helloCount): void { @@ -372,10 +375,10 @@ public function analyse( $job = array_pop($jobs); $process->request(['action' => 'analyse', 'files' => $job]); - }, $handleError, function ($exitCode, string $output) use (&$someChildEnded, &$peakMemoryUsages, &$internalErrors, &$internalErrorsCount, $processIdentifier): void { - if ($someChildEnded === false) { - $peakMemoryUsages['main'] = memory_get_usage(true); - } + }, $handleError, function ($exitCode, string $output) use (&$someChildEnded, &$internalErrors, &$internalErrorsCount, $processIdentifier): void { + // The main process is not sampled here any more: its own peak comes + // later (collecting the workers' results, saving the result cache) and + // is read where the number is printed. Only worker peaks are summed. $someChildEnded = true; if ($exitCode === 0) {