From d54463d472aa20c2430e0e75fa6d5bdcd26c264b Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 12 Jul 2026 20:29:54 +0200 Subject: [PATCH] [v3-3-test] Fix generate-providers-metadata hang by using spawn pools (#69763) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multiprocessing pools in the providers-metadata generation flow used the platform-default start method (fork on Linux). Before the pools are created the parent process has already used GitPython — which opens persistent `git cat-file --batch` subprocesses and is not fork-safe — and holds open network sockets from the version and constraints downloads. Forking that state into the workers left them with broken inherited file descriptors, so the pool deadlocked and the command hung forever, most reliably when --refresh-constraints-and-airflow-releases forces the parent through git and the network before forking. Switching these pools to the spawn start method gives each worker a clean interpreter with no inherited git subprocesses or sockets. (cherry picked from commit 583369271f5ae2098e9d800a0a427cf2eeb4edd4) Co-authored-by: Jarek Potiuk --- .../commands/release_management_commands.py | 9 ++++++--- .../src/airflow_breeze/utils/provider_dependencies.py | 9 +++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 146b3ba180c36..a553bc97916e8 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -34,7 +34,7 @@ from datetime import datetime from enum import Enum from functools import partial -from multiprocessing import Pool +from multiprocessing import get_context from pathlib import Path from subprocess import DEVNULL from typing import IO, TYPE_CHECKING, Any, Literal, NamedTuple @@ -3598,7 +3598,10 @@ def generate_providers_metadata( ) console_print("\n[info]Checking provider.yaml versions[1:] against PyPI for stale entries...[/]\n") - with Pool() as pypi_pool: + # "spawn" (not the platform-default fork): the parent has already used GitPython and + # opened network sockets before reaching here, and forking that state into workers + # deadlocks. See get_all_constraint_files_and_airflow_releases for the same reasoning. + with get_context("spawn").Pool() as pypi_pool: pruned_per_provider = pypi_pool.map(prune_unreleased_versions_from_provider_yaml, package_ids) total_pruned = 0 for pid, pruned in zip(package_ids, pruned_per_provider): @@ -3623,7 +3626,7 @@ def generate_providers_metadata( airflow_release_dates=airflow_release_dates, current_metadata=current_metadata, ) - with Pool() as pool: + with get_context("spawn").Pool() as pool: results = pool.map( partial_generate_providers_metadata, package_ids, diff --git a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py index a8d489b05cacc..963f96cff1716 100644 --- a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py +++ b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py @@ -26,7 +26,7 @@ import urllib.request from collections.abc import Generator from functools import cache, partial -from multiprocessing import Pool +from multiprocessing import get_context from pathlib import Path from threading import Lock from typing import NamedTuple @@ -274,7 +274,12 @@ def get_all_constraint_files_and_airflow_releases( airflow_release_dates_path.write_text(json.dumps(airflow_release_dates, indent=2)) console_print(f"[info]Airflow release dates saved in: {airflow_release_dates_path}[/]") with ci_group("Downloading constraints for all Airflow versions for all historical Python versions"): - with Pool() as pool: + # Use the "spawn" start method rather than the platform default: GitPython + # (used in the workers via get_tag_date) opens persistent `git cat-file --batch` + # subprocesses and is not fork-safe, and the parent already holds open network + # sockets from the version/constraints downloads. Forking that state into workers + # deadlocks; spawn gives each worker a clean interpreter. + with get_context("spawn").Pool() as pool: # We use partial to pass the common parameters to the function get_constraints_for_python_version_partial = partial( get_constraints_for_python_version,