Skip to content

Pre-warm cold OptimizedDirectorySourceLocator caches before spawning workers #5846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SanderMuller wants to merge 1 commit into
base: 2.2.x
Choose a base branch
from
+125 −75
Open

Pre-warm cold OptimizedDirectorySourceLocator caches before spawning workers #5846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/Command/AnalyserRunner.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
use PHPStan\Parallel\ParallelAnalyser;
use PHPStan\Parallel\Scheduler;
use PHPStan\Process\CpuCoreCounter;
use PHPStan\Reflection\BetterReflection\BetterReflectionSourceLocatorFactory;
use PHPStan\ShouldNotHappenException;
use React\EventLoop\StreamSelectLoop;
use Symfony\Component\Console\Input\InputInterface;
Expand All @@ -29,6 +30,7 @@ public function __construct(
private Analyser $analyser,
private ParallelAnalyser $parallelAnalyser,
private CpuCoreCounter $cpuCoreCounter,
private BetterReflectionSourceLocatorFactory $sourceLocatorFactory,
)
{
}
Expand Down Expand Up @@ -81,6 +83,9 @@ public function runAnalyser(
}

if ($mainScript !== null && $schedule->getNumberOfProcesses() > 0) {
// otherwise every worker redoes the same symbol scan in parallel against a cold cache
$this->sourceLocatorFactory->prewarmColdDirectoryCaches();

$loop = new StreamSelectLoop();
$result = null;
$promise = $this->parallelAnalyser->analyse($loop, $schedule, $allAnalysedFiles, $mainScript, $postFileCallback, $projectConfigFile, $tmpFile, $insteadOfFile, $input, null);
Expand Down
180 changes: 105 additions & 75 deletions src/Reflection/BetterReflection/BetterReflectionSourceLocatorFactory.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
use PHPStan\Reflection\BetterReflection\SourceLocator\ComposerJsonAndInstalledJsonSourceLocatorMaker;
use PHPStan\Reflection\BetterReflection\SourceLocator\FileNodesFetcher;
use PHPStan\Reflection\BetterReflection\SourceLocator\LazySourceLocator;
use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedDirectorySourceLocatorFactory;
use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedDirectorySourceLocatorRepository;
use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedPsrAutoloaderLocatorFactory;
use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedSingleFileSourceLocatorRepository;
Expand All @@ -29,6 +30,7 @@
use PHPStan\Reflection\BetterReflection\SourceLocator\RewriteClassAliasSourceLocator;
use PHPStan\Reflection\BetterReflection\SourceLocator\SkipClassAliasSourceLocator;
use PHPStan\Reflection\BetterReflection\SourceLocator\SkipPolyfillSourceLocator;
use function array_keys;
use function array_merge;
use function array_unique;
use function count;
Expand Down Expand Up @@ -58,6 +60,7 @@ public function __construct(
private ReflectionSourceStubber $reflectionSourceStubber,
private OptimizedSingleFileSourceLocatorRepository $optimizedSingleFileSourceLocatorRepository,
private OptimizedDirectorySourceLocatorRepository $optimizedDirectorySourceLocatorRepository,
private OptimizedDirectorySourceLocatorFactory $optimizedDirectorySourceLocatorFactory,
private ComposerJsonAndInstalledJsonSourceLocatorMaker $composerJsonAndInstalledJsonSourceLocatorMaker,
private OptimizedPsrAutoloaderLocatorFactory $optimizedPsrAutoloaderLocatorFactory,
private FileNodesFetcher $fileNodesFetcher,
Expand All @@ -81,99 +84,126 @@ public function __construct(

public function create(): SourceLocator
{
$initializer = function () {
$locators = [
$this->optimizedSingleFileSourceLocatorRepository->getOrCreate(
PHP_VERSION_ID < 80500
? __DIR__ . '/../../../stubs/runtime/Attribute84.php'
: __DIR__ . '/../../../stubs/runtime/Attribute85.php',
),
];

if ($this->singleReflectionFile !== null) {
$locators[] = $this->optimizedSingleFileSourceLocatorRepository->getOrCreate($this->singleReflectionFile);
return new MemoizingSourceLocator(new LazySourceLocator(fn () => $this->buildSourceLocator()));
}

/**
* Fills the OptimizedDirectorySourceLocator symbol cache for analysed
* directories that have no cache entry yet, so parallel workers read it
* instead of every worker redoing the same scan against a cold cache.
*
* Directories with an existing cache entry are skipped: workers validate
* those cheaply themselves, and the main thread must otherwise stay lazy
* (see https://github.com/phpstan/phpstan-src/pull/5577) - nothing else
* of the source locator is initialized and no identifier is located.
*/
public function prewarmColdDirectoryCaches(): void
{
$directories = [];
foreach (array_merge($this->analysedPaths, $this->analysedPathsFromConfig, $this->scanDirectories) as $path) {
if (!is_dir($path)) {
continue;
}

$astLocator = new Locator($this->parser);
$locators[] = new AutoloadFunctionsSourceLocator(
new AutoloadSourceLocator($this->fileNodesFetcher, false),
new ReflectionClassSourceLocator(
$astLocator,
$this->reflectionSourceStubber,
),
);

$analysedDirectories = [];
$analysedFiles = [];

foreach (array_merge($this->analysedPaths, $this->analysedPathsFromConfig) as $analysedPath) {
if (is_file($analysedPath)) {
$analysedFiles[] = $analysedPath;
continue;
}
$directories[$path] = true;
}

if (!is_dir($analysedPath)) {
continue;
}
foreach (array_keys($directories) as $directory) {
$this->optimizedDirectorySourceLocatorFactory->prewarmIfCold($directory);
}
}

$analysedDirectories[] = $analysedPath;
private function buildSourceLocator(): SourceLocator
{
$locators = [
$this->optimizedSingleFileSourceLocatorRepository->getOrCreate(
PHP_VERSION_ID < 80500
? __DIR__ . '/../../../stubs/runtime/Attribute84.php'
: __DIR__ . '/../../../stubs/runtime/Attribute85.php',
),
];

if ($this->singleReflectionFile !== null) {
$locators[] = $this->optimizedSingleFileSourceLocatorRepository->getOrCreate($this->singleReflectionFile);
}

$astLocator = new Locator($this->parser);
$locators[] = new AutoloadFunctionsSourceLocator(
new AutoloadSourceLocator($this->fileNodesFetcher, false),
new ReflectionClassSourceLocator(
$astLocator,
$this->reflectionSourceStubber,
),
);

$analysedDirectories = [];
$analysedFiles = [];

foreach (array_merge($this->analysedPaths, $this->analysedPathsFromConfig) as $analysedPath) {
if (is_file($analysedPath)) {
$analysedFiles[] = $analysedPath;
continue;
}

$fileLocators = [];
$analysedFiles = array_unique(array_merge($analysedFiles, $this->scanFiles));
foreach ($analysedFiles as $analysedFile) {
$fileLocators[] = $this->optimizedSingleFileSourceLocatorRepository->getOrCreate($analysedFile);
if (!is_dir($analysedPath)) {
continue;
}

$directories = array_unique(array_merge($analysedDirectories, $this->scanDirectories));
foreach ($directories as $directory) {
$fileLocators[] = $this->optimizedDirectorySourceLocatorRepository->getOrCreate($directory);
}
$analysedDirectories[] = $analysedPath;
}

$astPhp8Locator = new Locator($this->php8Parser);
$fileLocators = [];
$analysedFiles = array_unique(array_merge($analysedFiles, $this->scanFiles));
foreach ($analysedFiles as $analysedFile) {
$fileLocators[] = $this->optimizedSingleFileSourceLocatorRepository->getOrCreate($analysedFile);
}

$composerLocators = [];
$directories = array_unique(array_merge($analysedDirectories, $this->scanDirectories));
foreach ($directories as $directory) {
$fileLocators[] = $this->optimizedDirectorySourceLocatorRepository->getOrCreate($directory);
}

foreach ($this->composerAutoloaderProjectPaths as $composerAutoloaderProjectPath) {
$locator = $this->composerJsonAndInstalledJsonSourceLocatorMaker->create($composerAutoloaderProjectPath);
if ($locator === null) {
continue;
}
$composerLocators[] = $locator;
}
$astPhp8Locator = new Locator($this->php8Parser);

if (count($composerLocators) > 0) {
$fileLocators[] = new SkipPolyfillSourceLocator(new AggregateSourceLocator($composerLocators), $this->phpVersion);
}
$composerLocators = [];

if (extension_loaded('phar')) {
$pharProtocolPath = Phar::running();
if ($pharProtocolPath !== '') {
$mappings = [
'PHPStan\\BetterReflection\\' => [$pharProtocolPath . '/vendor/ondrejmirtes/better-reflection/src/'],
];
if ($this->playgroundMode) {
$mappings['PHPStan\\'] = [$pharProtocolPath . '/src/'];
} else {
$mappings['PHPStan\\Testing\\'] = [$pharProtocolPath . '/src/Testing/'];
}
$fileLocators[] = $this->optimizedPsrAutoloaderLocatorFactory->create(
Psr4Mapping::fromArrayMappings($mappings),
);
foreach ($this->composerAutoloaderProjectPaths as $composerAutoloaderProjectPath) {
$locator = $this->composerJsonAndInstalledJsonSourceLocatorMaker->create($composerAutoloaderProjectPath);
if ($locator === null) {
continue;
}
$composerLocators[] = $locator;
}

if (count($composerLocators) > 0) {
$fileLocators[] = new SkipPolyfillSourceLocator(new AggregateSourceLocator($composerLocators), $this->phpVersion);
}

if (extension_loaded('phar')) {
$pharProtocolPath = Phar::running();
if ($pharProtocolPath !== '') {
$mappings = [
'PHPStan\\BetterReflection\\' => [$pharProtocolPath . '/vendor/ondrejmirtes/better-reflection/src/'],
];
if ($this->playgroundMode) {
$mappings['PHPStan\\'] = [$pharProtocolPath . '/src/'];
} else {
$mappings['PHPStan\\Testing\\'] = [$pharProtocolPath . '/src/Testing/'];
}
$fileLocators[] = $this->optimizedPsrAutoloaderLocatorFactory->create(
Psr4Mapping::fromArrayMappings($mappings),
);
}
}

$locators[] = new RewriteClassAliasSourceLocator(new AggregateSourceLocator($fileLocators));
$locators[] = new SkipClassAliasSourceLocator(new PhpInternalSourceLocator($astPhp8Locator, $this->phpstormStubsSourceStubber));

$locators[] = new AutoloadSourceLocator($this->fileNodesFetcher, true);
$locators[] = new PhpVersionBlacklistSourceLocator(new PhpInternalSourceLocator($astLocator, $this->reflectionSourceStubber), $this->phpstormStubsSourceStubber);
$locators[] = new PhpVersionBlacklistSourceLocator(new EvaledCodeSourceLocator($astLocator, $this->reflectionSourceStubber), $this->phpstormStubsSourceStubber);
$locators[] = new RewriteClassAliasSourceLocator(new AggregateSourceLocator($fileLocators));
$locators[] = new SkipClassAliasSourceLocator(new PhpInternalSourceLocator($astPhp8Locator, $this->phpstormStubsSourceStubber));

return new AggregateSourceLocator($locators);
};
$locators[] = new AutoloadSourceLocator($this->fileNodesFetcher, true);
$locators[] = new PhpVersionBlacklistSourceLocator(new PhpInternalSourceLocator($astLocator, $this->reflectionSourceStubber), $this->phpstormStubsSourceStubber);
$locators[] = new PhpVersionBlacklistSourceLocator(new EvaledCodeSourceLocator($astLocator, $this->reflectionSourceStubber), $this->phpstormStubsSourceStubber);

return new MemoizingSourceLocator(new LazySourceLocator($initializer));
return new AggregateSourceLocator($locators);
}

}
15 changes: 15 additions & 0 deletions src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,21 @@ public function __construct(
{
}

/**
* Builds the symbol cache for the directory only when no cache entry
* exists yet. With an existing entry this is a single cache read - the
* per-file validation is left to whoever actually uses the locator.
*/
public function prewarmIfCold(string $directory): void
{
$variableCacheKey = sprintf('v1-%s', $this->phpVersion->supportsEnums() ? 'enums' : 'no-enums');
if ($this->cache->load(sprintf('odsl-%s', $directory), $variableCacheKey) !== null) {
return;
}

$this->createByDirectory($directory);
}

public function createByDirectory(string $directory): OptimizedDirectorySourceLocator
{
$files = $this->fileFinder->findFiles([$directory])->getFiles();
Expand Down
Loading