From 73cb4bcf6ec20385cdbc69f29bff9a68904c14d2 Mon Sep 17 00:00:00 2001 From: Anders Jenbo Date: Mon, 24 Aug 2026 17:30:19 +0200 Subject: [PATCH 1/3] Implement worker auto-scaler --- conf/config.neon | 2 +- conf/parametersSchema.neon | 2 +- src/Parallel/Scheduler.php | 35 ++++++++++++++++++++---- tests/PHPStan/Parallel/SchedulerTest.php | 28 +++++++++++++++++++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/conf/config.neon b/conf/config.neon index de093affa36..31869418817 100644 --- a/conf/config.neon +++ b/conf/config.neon @@ -107,7 +107,7 @@ parameters: parallel: jobSize: 20 processTimeout: 600.0 - maximumNumberOfProcesses: 8 + maximumNumberOfProcesses: auto minimumNumberOfJobsPerProcess: 2 buffer: 134217728 # 128 MB loadLimit: 1.0 diff --git a/conf/parametersSchema.neon b/conf/parametersSchema.neon index ecbac473b43..5741d05cd34 100644 --- a/conf/parametersSchema.neon +++ b/conf/parametersSchema.neon @@ -109,7 +109,7 @@ parametersSchema: parallel: structure([ jobSize: int(), processTimeout: float(), - maximumNumberOfProcesses: int(), + maximumNumberOfProcesses: anyOf(int(), 'auto'), minimumNumberOfJobsPerProcess: int(), buffer: int(), loadLimit: schema(float(), nullable()) diff --git a/src/Parallel/Scheduler.php b/src/Parallel/Scheduler.php index 50760e65b92..312ce19d0ea 100644 --- a/src/Parallel/Scheduler.php +++ b/src/Parallel/Scheduler.php @@ -19,19 +19,21 @@ final class Scheduler implements DiagnoseExtension { - /** @var array{int, int, int, int}|null */ + public const AUTO = 'auto'; + + /** @var array{int, int, int, int, string}|null */ private ?array $storedData = null; /** * @param positive-int $jobSize - * @param positive-int $maximumNumberOfProcesses + * @param positive-int|self::AUTO $maximumNumberOfProcesses * @param positive-int $minimumNumberOfJobsPerProcess */ public function __construct( #[AutowiredParameter(ref: '%parallel.jobSize%')] private int $jobSize, #[AutowiredParameter(ref: '%parallel.maximumNumberOfProcesses%')] - private int $maximumNumberOfProcesses, + private int|string $maximumNumberOfProcesses, #[AutowiredParameter(ref: '%parallel.minimumNumberOfJobsPerProcess%')] private int $minimumNumberOfJobsPerProcess, ) @@ -78,25 +80,46 @@ public function scheduleWork( $cpuCores, ); - $usedNumberOfProcesses = min($numberOfProcesses, $this->maximumNumberOfProcesses); - $this->storedData = [$cpuCores, count($files), count($jobs), $usedNumberOfProcesses]; + [$maximumNumberOfProcesses, $decision] = $this->resolveMaximumNumberOfProcesses($cpuCores); + $usedNumberOfProcesses = min($numberOfProcesses, $maximumNumberOfProcesses); + $this->storedData = [$cpuCores, count($files), count($jobs), $usedNumberOfProcesses, $decision]; return new Schedule($usedNumberOfProcesses, $jobs); } + /** + * How many workers may run at once, and a human-readable account of why - which + * `diagnose` prints, because a user who thinks the number is wrong needs to see + * which input produced it. + * + * @return array{positive-int, string} + */ + private function resolveMaximumNumberOfProcesses(int $cpuCores): array + { + if ($this->maximumNumberOfProcesses !== self::AUTO) { + return [$this->maximumNumberOfProcesses, 'configured']; + } + + return [ + max(1, $cpuCores), + sprintf('auto, limited by %d usable CPU cores', $cpuCores), + ]; + } + public function print(Output $output): void { if ($this->storedData === null) { return; } - [$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses] = $this->storedData; + [$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses, $decision] = $this->storedData; $output->writeLineFormatted('Parallel processing scheduler:'); $output->writeLineFormatted(sprintf('# of detected CPU cores: %d', $cpuCores)); $output->writeLineFormatted(sprintf('# of analysed files: %d', $filesCount)); $output->writeLineFormatted(sprintf('# of jobs: %d', $jobsCount)); $output->writeLineFormatted(sprintf('# of spawned processes: %d', $usedNumberOfProcesses)); + $output->writeLineFormatted(sprintf('Process limit: %s', $decision)); $output->writeLineFormatted(''); } diff --git a/tests/PHPStan/Parallel/SchedulerTest.php b/tests/PHPStan/Parallel/SchedulerTest.php index a5fa5b829d6..727d6e0a984 100644 --- a/tests/PHPStan/Parallel/SchedulerTest.php +++ b/tests/PHPStan/Parallel/SchedulerTest.php @@ -165,4 +165,32 @@ public function testEveryFileIsScheduledExactlyOnce(): void } } + public function testAutoUsesAllUsableCores(): void + { + // 12 usable cores, plenty of jobs - auto follows the cores, not the old + // fixed default of 8 + $scheduler = new Scheduler(1, Scheduler::AUTO, 1); + $schedule = $scheduler->scheduleWork(12, array_fill(0, 200, 'file.php'), static fn (string $file): int => 0); + + $this->assertSame(12, $schedule->getNumberOfProcesses()); + } + + public function testAutoIsStillCappedByTheJobCount(): void + { + // 40 files -> 2 jobs at size 20, at least 2 jobs per process -> a single + // worker no matter how many cores the machine has + $scheduler = new Scheduler(20, Scheduler::AUTO, 2); + $schedule = $scheduler->scheduleWork(32, array_fill(0, 40, 'file.php'), static fn (string $file): int => 0); + + $this->assertSame(1, $schedule->getNumberOfProcesses()); + } + + public function testAnExplicitLimitStillWins(): void + { + $scheduler = new Scheduler(1, 20, 1); + $schedule = $scheduler->scheduleWork(32, array_fill(0, 200, 'file.php'), static fn (string $file): int => 0); + + $this->assertSame(20, $schedule->getNumberOfProcesses()); + } + } From e7d6d84319858e16b3175e1b60b43b0e57660f3f Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Mon, 31 Aug 2026 14:22:32 +0200 Subject: [PATCH 2/3] Cap auto worker scaling at 20 processes `auto` had no ceiling: with the scheduler already bounding the count by the usable cores, `max(1, $cpuCores)` was no limit at all, so a 128-thread machine got 128 workers. The measurements behind the feature show where the returns diminish - 8 to 16 workers cut wall time by a third for 20 % more CPU time, 16 to 32 bought 6 % for 73 % more - and memory grows with every worker, so stop at AUTO_PROCESSES_LIMIT and say so in `diagnose`. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PRfhXECP8wuY7qyLqRaXk3 --- src/Parallel/Scheduler.php | 15 +++++++++++++++ tests/PHPStan/Parallel/SchedulerTest.php | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/Parallel/Scheduler.php b/src/Parallel/Scheduler.php index 312ce19d0ea..b5b30b52ee4 100644 --- a/src/Parallel/Scheduler.php +++ b/src/Parallel/Scheduler.php @@ -21,6 +21,14 @@ final class Scheduler implements DiagnoseExtension public const AUTO = 'auto'; + /** + * Where auto scaling stops, because the returns diminish: on a 16c/32t + * workstation going from 8 to 16 workers cut wall time by a third for 20 % + * more CPU time, 16 to 32 bought 6 % for 73 % more, and memory grows with + * every worker (https://github.com/phpstan/phpstan-src/pull/6256). + */ + private const AUTO_PROCESSES_LIMIT = 20; + /** @var array{int, int, int, int, string}|null */ private ?array $storedData = null; @@ -100,6 +108,13 @@ private function resolveMaximumNumberOfProcesses(int $cpuCores): array return [$this->maximumNumberOfProcesses, 'configured']; } + if ($cpuCores > self::AUTO_PROCESSES_LIMIT) { + return [ + self::AUTO_PROCESSES_LIMIT, + sprintf('auto, capped at %d processes (%d usable CPU cores)', self::AUTO_PROCESSES_LIMIT, $cpuCores), + ]; + } + return [ max(1, $cpuCores), sprintf('auto, limited by %d usable CPU cores', $cpuCores), diff --git a/tests/PHPStan/Parallel/SchedulerTest.php b/tests/PHPStan/Parallel/SchedulerTest.php index 727d6e0a984..de2b132980a 100644 --- a/tests/PHPStan/Parallel/SchedulerTest.php +++ b/tests/PHPStan/Parallel/SchedulerTest.php @@ -175,6 +175,16 @@ public function testAutoUsesAllUsableCores(): void $this->assertSame(12, $schedule->getNumberOfProcesses()); } + public function testAutoIsCappedAtTheProcessLimit(): void + { + // 64 usable cores and 5000 files worth of jobs - auto stops where the + // returns diminish rather than spawning a worker per core + $scheduler = new Scheduler(20, Scheduler::AUTO, 2); + $schedule = $scheduler->scheduleWork(64, array_fill(0, 5000, 'file.php'), static fn (string $file): int => 0); + + $this->assertSame(20, $schedule->getNumberOfProcesses()); + } + public function testAutoIsStillCappedByTheJobCount(): void { // 40 files -> 2 jobs at size 20, at least 2 jobs per process -> a single From 5a776f152cdc1278038cf20b2d28d2e466d29fcc Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Mon, 31 Aug 2026 14:22:33 +0200 Subject: [PATCH 3/3] Label the scheduler's core count as usable in diagnose The number handed to the scheduler is what is left after the load limit and the cgroup quota, not what the machine reports; the raw count and each limit are on the "System resources" lines above it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PRfhXECP8wuY7qyLqRaXk3 --- src/Parallel/Scheduler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parallel/Scheduler.php b/src/Parallel/Scheduler.php index b5b30b52ee4..604792a3406 100644 --- a/src/Parallel/Scheduler.php +++ b/src/Parallel/Scheduler.php @@ -130,7 +130,7 @@ public function print(Output $output): void [$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses, $decision] = $this->storedData; $output->writeLineFormatted('Parallel processing scheduler:'); - $output->writeLineFormatted(sprintf('# of detected CPU cores: %d', $cpuCores)); + $output->writeLineFormatted(sprintf('# of usable CPU cores: %d', $cpuCores)); $output->writeLineFormatted(sprintf('# of analysed files: %d', $filesCount)); $output->writeLineFormatted(sprintf('# of jobs: %d', $jobsCount)); $output->writeLineFormatted(sprintf('# of spawned processes: %d', $usedNumberOfProcesses));