diff --git a/conf/config.neon b/conf/config.neon index de093affa36..31869418817 100644 --- a/conf/config.neon +++ b/conf/config.neon @@ -107,7 +107,7 @@ parameters: parallel: jobSize: 20 processTimeout: 600.0 - maximumNumberOfProcesses: 8 + maximumNumberOfProcesses: auto minimumNumberOfJobsPerProcess: 2 buffer: 134217728 # 128 MB loadLimit: 1.0 diff --git a/conf/parametersSchema.neon b/conf/parametersSchema.neon index ecbac473b43..5741d05cd34 100644 --- a/conf/parametersSchema.neon +++ b/conf/parametersSchema.neon @@ -109,7 +109,7 @@ parametersSchema: parallel: structure([ jobSize: int(), processTimeout: float(), - maximumNumberOfProcesses: int(), + maximumNumberOfProcesses: anyOf(int(), 'auto'), minimumNumberOfJobsPerProcess: int(), buffer: int(), loadLimit: schema(float(), nullable()) diff --git a/src/Parallel/Scheduler.php b/src/Parallel/Scheduler.php index 50760e65b92..604792a3406 100644 --- a/src/Parallel/Scheduler.php +++ b/src/Parallel/Scheduler.php @@ -19,19 +19,29 @@ final class Scheduler implements DiagnoseExtension { - /** @var array{int, int, int, int}|null */ + public const AUTO = 'auto'; + + /** + * Where auto scaling stops, because the returns diminish: on a 16c/32t + * workstation going from 8 to 16 workers cut wall time by a third for 20 % + * more CPU time, 16 to 32 bought 6 % for 73 % more, and memory grows with + * every worker (https://github.com/phpstan/phpstan-src/pull/6256). + */ + private const AUTO_PROCESSES_LIMIT = 20; + + /** @var array{int, int, int, int, string}|null */ private ?array $storedData = null; /** * @param positive-int $jobSize - * @param positive-int $maximumNumberOfProcesses + * @param positive-int|self::AUTO $maximumNumberOfProcesses * @param positive-int $minimumNumberOfJobsPerProcess */ public function __construct( #[AutowiredParameter(ref: '%parallel.jobSize%')] private int $jobSize, #[AutowiredParameter(ref: '%parallel.maximumNumberOfProcesses%')] - private int $maximumNumberOfProcesses, + private int|string $maximumNumberOfProcesses, #[AutowiredParameter(ref: '%parallel.minimumNumberOfJobsPerProcess%')] private int $minimumNumberOfJobsPerProcess, ) @@ -78,25 +88,53 @@ public function scheduleWork( $cpuCores, ); - $usedNumberOfProcesses = min($numberOfProcesses, $this->maximumNumberOfProcesses); - $this->storedData = [$cpuCores, count($files), count($jobs), $usedNumberOfProcesses]; + [$maximumNumberOfProcesses, $decision] = $this->resolveMaximumNumberOfProcesses($cpuCores); + $usedNumberOfProcesses = min($numberOfProcesses, $maximumNumberOfProcesses); + $this->storedData = [$cpuCores, count($files), count($jobs), $usedNumberOfProcesses, $decision]; return new Schedule($usedNumberOfProcesses, $jobs); } + /** + * How many workers may run at once, and a human-readable account of why - which + * `diagnose` prints, because a user who thinks the number is wrong needs to see + * which input produced it. + * + * @return array{positive-int, string} + */ + private function resolveMaximumNumberOfProcesses(int $cpuCores): array + { + if ($this->maximumNumberOfProcesses !== self::AUTO) { + return [$this->maximumNumberOfProcesses, 'configured']; + } + + if ($cpuCores > self::AUTO_PROCESSES_LIMIT) { + return [ + self::AUTO_PROCESSES_LIMIT, + sprintf('auto, capped at %d processes (%d usable CPU cores)', self::AUTO_PROCESSES_LIMIT, $cpuCores), + ]; + } + + return [ + max(1, $cpuCores), + sprintf('auto, limited by %d usable CPU cores', $cpuCores), + ]; + } + public function print(Output $output): void { if ($this->storedData === null) { return; } - [$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses] = $this->storedData; + [$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses, $decision] = $this->storedData; $output->writeLineFormatted('Parallel processing scheduler:'); - $output->writeLineFormatted(sprintf('# of detected CPU cores: %d', $cpuCores)); + $output->writeLineFormatted(sprintf('# of usable CPU cores: %d', $cpuCores)); $output->writeLineFormatted(sprintf('# of analysed files: %d', $filesCount)); $output->writeLineFormatted(sprintf('# of jobs: %d', $jobsCount)); $output->writeLineFormatted(sprintf('# of spawned processes: %d', $usedNumberOfProcesses)); + $output->writeLineFormatted(sprintf('Process limit: %s', $decision)); $output->writeLineFormatted(''); } diff --git a/tests/PHPStan/Parallel/SchedulerTest.php b/tests/PHPStan/Parallel/SchedulerTest.php index a5fa5b829d6..de2b132980a 100644 --- a/tests/PHPStan/Parallel/SchedulerTest.php +++ b/tests/PHPStan/Parallel/SchedulerTest.php @@ -165,4 +165,42 @@ public function testEveryFileIsScheduledExactlyOnce(): void } } + public function testAutoUsesAllUsableCores(): void + { + // 12 usable cores, plenty of jobs - auto follows the cores, not the old + // fixed default of 8 + $scheduler = new Scheduler(1, Scheduler::AUTO, 1); + $schedule = $scheduler->scheduleWork(12, array_fill(0, 200, 'file.php'), static fn (string $file): int => 0); + + $this->assertSame(12, $schedule->getNumberOfProcesses()); + } + + public function testAutoIsCappedAtTheProcessLimit(): void + { + // 64 usable cores and 5000 files worth of jobs - auto stops where the + // returns diminish rather than spawning a worker per core + $scheduler = new Scheduler(20, Scheduler::AUTO, 2); + $schedule = $scheduler->scheduleWork(64, array_fill(0, 5000, 'file.php'), static fn (string $file): int => 0); + + $this->assertSame(20, $schedule->getNumberOfProcesses()); + } + + public function testAutoIsStillCappedByTheJobCount(): void + { + // 40 files -> 2 jobs at size 20, at least 2 jobs per process -> a single + // worker no matter how many cores the machine has + $scheduler = new Scheduler(20, Scheduler::AUTO, 2); + $schedule = $scheduler->scheduleWork(32, array_fill(0, 40, 'file.php'), static fn (string $file): int => 0); + + $this->assertSame(1, $schedule->getNumberOfProcesses()); + } + + public function testAnExplicitLimitStillWins(): void + { + $scheduler = new Scheduler(1, 20, 1); + $schedule = $scheduler->scheduleWork(32, array_fill(0, 200, 'file.php'), static fn (string $file): int => 0); + + $this->assertSame(20, $schedule->getNumberOfProcesses()); + } + }