From 11d3e17d17973127ac7967f72aeb9b3fc4f09de1 Mon Sep 17 00:00:00 2001 From: Sander Muller Date: Thu, 11 Jun 2026 09:02:44 +0200 Subject: [PATCH] Pre-warm source locator caches before spawning parallel workers Building the source locator eagerly scans the analysed directories and composer classmap directories and writes the shared file cache. Without this, every parallel worker redoes the same symbol scan against a cold cache: measured 8x duplicated scanning CPU at 8 workers, and removing this pre-warm costs +51% wall / +86% CPU on a 14-core cold run of src/Type (with PHPSTAN_PARALLEL_FORK=1, forked workers additionally inherit the warm in-memory state via copy-on-write). Co-Authored-By: Claude Fable 5 --- src/Command/AnalyserRunner.php | 5 + .../BetterReflectionSourceLocatorFactory.php | 180 ++++++++++-------- ...OptimizedDirectorySourceLocatorFactory.php | 15 ++ 3 files changed, 125 insertions(+), 75 deletions(-) diff --git a/src/Command/AnalyserRunner.php b/src/Command/AnalyserRunner.php index 51149328b96..0d04f3ebb25 100644 --- a/src/Command/AnalyserRunner.php +++ b/src/Command/AnalyserRunner.php @@ -9,6 +9,7 @@ use PHPStan\Parallel\ParallelAnalyser; use PHPStan\Parallel\Scheduler; use PHPStan\Process\CpuCoreCounter; +use PHPStan\Reflection\BetterReflection\BetterReflectionSourceLocatorFactory; use PHPStan\ShouldNotHappenException; use React\EventLoop\StreamSelectLoop; use Symfony\Component\Console\Input\InputInterface; @@ -29,6 +30,7 @@ public function __construct( private Analyser $analyser, private ParallelAnalyser $parallelAnalyser, private CpuCoreCounter $cpuCoreCounter, + private BetterReflectionSourceLocatorFactory $sourceLocatorFactory, ) { } @@ -81,6 +83,9 @@ public function runAnalyser( } if ($mainScript !== null && $schedule->getNumberOfProcesses() > 0) { + // otherwise every worker redoes the same symbol scan in parallel against a cold cache + $this->sourceLocatorFactory->prewarmColdDirectoryCaches(); + $loop = new StreamSelectLoop(); $result = null; $promise = $this->parallelAnalyser->analyse($loop, $schedule, $allAnalysedFiles, $mainScript, $postFileCallback, $projectConfigFile, $tmpFile, $insteadOfFile, $input, null); diff --git a/src/Reflection/BetterReflection/BetterReflectionSourceLocatorFactory.php b/src/Reflection/BetterReflection/BetterReflectionSourceLocatorFactory.php index fa6b0e858b1..a12e0171791 100644 --- a/src/Reflection/BetterReflection/BetterReflectionSourceLocatorFactory.php +++ b/src/Reflection/BetterReflection/BetterReflectionSourceLocatorFactory.php @@ -21,6 +21,7 @@ use PHPStan\Reflection\BetterReflection\SourceLocator\ComposerJsonAndInstalledJsonSourceLocatorMaker; use PHPStan\Reflection\BetterReflection\SourceLocator\FileNodesFetcher; use PHPStan\Reflection\BetterReflection\SourceLocator\LazySourceLocator; +use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedDirectorySourceLocatorFactory; use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedDirectorySourceLocatorRepository; use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedPsrAutoloaderLocatorFactory; use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedSingleFileSourceLocatorRepository; @@ -29,6 +30,7 @@ use PHPStan\Reflection\BetterReflection\SourceLocator\RewriteClassAliasSourceLocator; use PHPStan\Reflection\BetterReflection\SourceLocator\SkipClassAliasSourceLocator; use PHPStan\Reflection\BetterReflection\SourceLocator\SkipPolyfillSourceLocator; +use function array_keys; use function array_merge; use function array_unique; use function count; @@ -58,6 +60,7 @@ public function __construct( private ReflectionSourceStubber $reflectionSourceStubber, private OptimizedSingleFileSourceLocatorRepository $optimizedSingleFileSourceLocatorRepository, private OptimizedDirectorySourceLocatorRepository $optimizedDirectorySourceLocatorRepository, + private OptimizedDirectorySourceLocatorFactory $optimizedDirectorySourceLocatorFactory, private ComposerJsonAndInstalledJsonSourceLocatorMaker $composerJsonAndInstalledJsonSourceLocatorMaker, private OptimizedPsrAutoloaderLocatorFactory $optimizedPsrAutoloaderLocatorFactory, private FileNodesFetcher $fileNodesFetcher, @@ -81,99 +84,126 @@ public function __construct( public function create(): SourceLocator { - $initializer = function () { - $locators = [ - $this->optimizedSingleFileSourceLocatorRepository->getOrCreate( - PHP_VERSION_ID < 80500 - ? __DIR__ . '/../../../stubs/runtime/Attribute84.php' - : __DIR__ . '/../../../stubs/runtime/Attribute85.php', - ), - ]; - - if ($this->singleReflectionFile !== null) { - $locators[] = $this->optimizedSingleFileSourceLocatorRepository->getOrCreate($this->singleReflectionFile); + return new MemoizingSourceLocator(new LazySourceLocator(fn () => $this->buildSourceLocator())); + } + + /** + * Fills the OptimizedDirectorySourceLocator symbol cache for analysed + * directories that have no cache entry yet, so parallel workers read it + * instead of every worker redoing the same scan against a cold cache. + * + * Directories with an existing cache entry are skipped: workers validate + * those cheaply themselves, and the main thread must otherwise stay lazy + * (see https://github.com/phpstan/phpstan-src/pull/5577) - nothing else + * of the source locator is initialized and no identifier is located. + */ + public function prewarmColdDirectoryCaches(): void + { + $directories = []; + foreach (array_merge($this->analysedPaths, $this->analysedPathsFromConfig, $this->scanDirectories) as $path) { + if (!is_dir($path)) { + continue; } - $astLocator = new Locator($this->parser); - $locators[] = new AutoloadFunctionsSourceLocator( - new AutoloadSourceLocator($this->fileNodesFetcher, false), - new ReflectionClassSourceLocator( - $astLocator, - $this->reflectionSourceStubber, - ), - ); - - $analysedDirectories = []; - $analysedFiles = []; - - foreach (array_merge($this->analysedPaths, $this->analysedPathsFromConfig) as $analysedPath) { - if (is_file($analysedPath)) { - $analysedFiles[] = $analysedPath; - continue; - } + $directories[$path] = true; + } - if (!is_dir($analysedPath)) { - continue; - } + foreach (array_keys($directories) as $directory) { + $this->optimizedDirectorySourceLocatorFactory->prewarmIfCold($directory); + } + } - $analysedDirectories[] = $analysedPath; + private function buildSourceLocator(): SourceLocator + { + $locators = [ + $this->optimizedSingleFileSourceLocatorRepository->getOrCreate( + PHP_VERSION_ID < 80500 + ? __DIR__ . '/../../../stubs/runtime/Attribute84.php' + : __DIR__ . '/../../../stubs/runtime/Attribute85.php', + ), + ]; + + if ($this->singleReflectionFile !== null) { + $locators[] = $this->optimizedSingleFileSourceLocatorRepository->getOrCreate($this->singleReflectionFile); + } + + $astLocator = new Locator($this->parser); + $locators[] = new AutoloadFunctionsSourceLocator( + new AutoloadSourceLocator($this->fileNodesFetcher, false), + new ReflectionClassSourceLocator( + $astLocator, + $this->reflectionSourceStubber, + ), + ); + + $analysedDirectories = []; + $analysedFiles = []; + + foreach (array_merge($this->analysedPaths, $this->analysedPathsFromConfig) as $analysedPath) { + if (is_file($analysedPath)) { + $analysedFiles[] = $analysedPath; + continue; } - $fileLocators = []; - $analysedFiles = array_unique(array_merge($analysedFiles, $this->scanFiles)); - foreach ($analysedFiles as $analysedFile) { - $fileLocators[] = $this->optimizedSingleFileSourceLocatorRepository->getOrCreate($analysedFile); + if (!is_dir($analysedPath)) { + continue; } - $directories = array_unique(array_merge($analysedDirectories, $this->scanDirectories)); - foreach ($directories as $directory) { - $fileLocators[] = $this->optimizedDirectorySourceLocatorRepository->getOrCreate($directory); - } + $analysedDirectories[] = $analysedPath; + } - $astPhp8Locator = new Locator($this->php8Parser); + $fileLocators = []; + $analysedFiles = array_unique(array_merge($analysedFiles, $this->scanFiles)); + foreach ($analysedFiles as $analysedFile) { + $fileLocators[] = $this->optimizedSingleFileSourceLocatorRepository->getOrCreate($analysedFile); + } - $composerLocators = []; + $directories = array_unique(array_merge($analysedDirectories, $this->scanDirectories)); + foreach ($directories as $directory) { + $fileLocators[] = $this->optimizedDirectorySourceLocatorRepository->getOrCreate($directory); + } - foreach ($this->composerAutoloaderProjectPaths as $composerAutoloaderProjectPath) { - $locator = $this->composerJsonAndInstalledJsonSourceLocatorMaker->create($composerAutoloaderProjectPath); - if ($locator === null) { - continue; - } - $composerLocators[] = $locator; - } + $astPhp8Locator = new Locator($this->php8Parser); - if (count($composerLocators) > 0) { - $fileLocators[] = new SkipPolyfillSourceLocator(new AggregateSourceLocator($composerLocators), $this->phpVersion); - } + $composerLocators = []; - if (extension_loaded('phar')) { - $pharProtocolPath = Phar::running(); - if ($pharProtocolPath !== '') { - $mappings = [ - 'PHPStan\\BetterReflection\\' => [$pharProtocolPath . '/vendor/ondrejmirtes/better-reflection/src/'], - ]; - if ($this->playgroundMode) { - $mappings['PHPStan\\'] = [$pharProtocolPath . '/src/']; - } else { - $mappings['PHPStan\\Testing\\'] = [$pharProtocolPath . '/src/Testing/']; - } - $fileLocators[] = $this->optimizedPsrAutoloaderLocatorFactory->create( - Psr4Mapping::fromArrayMappings($mappings), - ); + foreach ($this->composerAutoloaderProjectPaths as $composerAutoloaderProjectPath) { + $locator = $this->composerJsonAndInstalledJsonSourceLocatorMaker->create($composerAutoloaderProjectPath); + if ($locator === null) { + continue; + } + $composerLocators[] = $locator; + } + + if (count($composerLocators) > 0) { + $fileLocators[] = new SkipPolyfillSourceLocator(new AggregateSourceLocator($composerLocators), $this->phpVersion); + } + + if (extension_loaded('phar')) { + $pharProtocolPath = Phar::running(); + if ($pharProtocolPath !== '') { + $mappings = [ + 'PHPStan\\BetterReflection\\' => [$pharProtocolPath . '/vendor/ondrejmirtes/better-reflection/src/'], + ]; + if ($this->playgroundMode) { + $mappings['PHPStan\\'] = [$pharProtocolPath . '/src/']; + } else { + $mappings['PHPStan\\Testing\\'] = [$pharProtocolPath . '/src/Testing/']; } + $fileLocators[] = $this->optimizedPsrAutoloaderLocatorFactory->create( + Psr4Mapping::fromArrayMappings($mappings), + ); } + } - $locators[] = new RewriteClassAliasSourceLocator(new AggregateSourceLocator($fileLocators)); - $locators[] = new SkipClassAliasSourceLocator(new PhpInternalSourceLocator($astPhp8Locator, $this->phpstormStubsSourceStubber)); - - $locators[] = new AutoloadSourceLocator($this->fileNodesFetcher, true); - $locators[] = new PhpVersionBlacklistSourceLocator(new PhpInternalSourceLocator($astLocator, $this->reflectionSourceStubber), $this->phpstormStubsSourceStubber); - $locators[] = new PhpVersionBlacklistSourceLocator(new EvaledCodeSourceLocator($astLocator, $this->reflectionSourceStubber), $this->phpstormStubsSourceStubber); + $locators[] = new RewriteClassAliasSourceLocator(new AggregateSourceLocator($fileLocators)); + $locators[] = new SkipClassAliasSourceLocator(new PhpInternalSourceLocator($astPhp8Locator, $this->phpstormStubsSourceStubber)); - return new AggregateSourceLocator($locators); - }; + $locators[] = new AutoloadSourceLocator($this->fileNodesFetcher, true); + $locators[] = new PhpVersionBlacklistSourceLocator(new PhpInternalSourceLocator($astLocator, $this->reflectionSourceStubber), $this->phpstormStubsSourceStubber); + $locators[] = new PhpVersionBlacklistSourceLocator(new EvaledCodeSourceLocator($astLocator, $this->reflectionSourceStubber), $this->phpstormStubsSourceStubber); - return new MemoizingSourceLocator(new LazySourceLocator($initializer)); + return new AggregateSourceLocator($locators); } } diff --git a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php index 48b1404ee9e..90c77e58e9b 100644 --- a/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php +++ b/src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php @@ -27,6 +27,21 @@ public function __construct( { } + /** + * Builds the symbol cache for the directory only when no cache entry + * exists yet. With an existing entry this is a single cache read - the + * per-file validation is left to whoever actually uses the locator. + */ + public function prewarmIfCold(string $directory): void + { + $variableCacheKey = sprintf('v1-%s', $this->phpVersion->supportsEnums() ? 'enums' : 'no-enums'); + if ($this->cache->load(sprintf('odsl-%s', $directory), $variableCacheKey) !== null) { + return; + } + + $this->createByDirectory($directory); + } + public function createByDirectory(string $directory): OptimizedDirectorySourceLocator { $files = $this->fileFinder->findFiles([$directory])->getFiles();