From f5bf8a831cd0783d21e9b9c94c4ed5e6467c3e45 Mon Sep 17 00:00:00 2001
From: Jarek Potiuk <jarek@potiuk.com>
Date: Sat, 11 Jul 2026 18:24:03 +0200
Subject: [PATCH] Fix generate-providers-metadata hang by using spawn pools
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The multiprocessing pools in the providers-metadata generation flow used the
platform-default start method (fork on Linux). Before the pools are created the
parent process has already used GitPython — which opens persistent
`git cat-file --batch` subprocesses and is not fork-safe — and holds open
network sockets from the version and constraints downloads. Forking that state
into the workers left them with broken inherited file descriptors, so the pool
deadlocked and the command hung forever, most reliably when
--refresh-constraints-and-airflow-releases forces the parent through git and the
network before forking.

Switching these pools to the spawn start method gives each worker a clean
interpreter with no inherited git subprocesses or sockets.
---
 .../commands/release_management_commands.py              | 9 ++++++---
 .../src/airflow_breeze/utils/provider_dependencies.py    | 9 +++++++--
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index aed5d0e1ec91d..5859b545e47cc 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -34,7 +34,7 @@
 from datetime import datetime
 from enum import Enum
 from functools import partial
-from multiprocessing import Pool
+from multiprocessing import get_context
 from pathlib import Path
 from subprocess import DEVNULL
 from typing import IO, TYPE_CHECKING, Any, Literal, NamedTuple
@@ -3611,7 +3611,10 @@ def generate_providers_metadata(
     )
 
     console_print("\n[info]Checking provider.yaml versions[1:] against PyPI for stale entries...[/]\n")
-    with Pool() as pypi_pool:
+    # "spawn" (not the platform-default fork): the parent has already used GitPython and
+    # opened network sockets before reaching here, and forking that state into workers
+    # deadlocks. See get_all_constraint_files_and_airflow_releases for the same reasoning.
+    with get_context("spawn").Pool() as pypi_pool:
         pruned_per_provider = pypi_pool.map(prune_unreleased_versions_from_provider_yaml, package_ids)
     total_pruned = 0
     for pid, pruned in zip(package_ids, pruned_per_provider):
@@ -3636,7 +3639,7 @@ def generate_providers_metadata(
         airflow_release_dates=airflow_release_dates,
         current_metadata=current_metadata,
     )
-    with Pool() as pool:
+    with get_context("spawn").Pool() as pool:
         results = pool.map(
             partial_generate_providers_metadata,
             package_ids,
diff --git a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
index a985a65b55de5..d67e0ec9bfc1b 100644
--- a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
+++ b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
@@ -26,7 +26,7 @@
 import urllib.request
 from collections.abc import Generator
 from functools import cache, partial
-from multiprocessing import Pool
+from multiprocessing import get_context
 from pathlib import Path
 from threading import Lock
 from typing import NamedTuple
@@ -276,7 +276,12 @@ def get_all_constraint_files_and_airflow_releases(
         airflow_release_dates_path.write_text(json.dumps(airflow_release_dates, indent=2))
         console_print(f"[info]Airflow release dates saved in: {airflow_release_dates_path}[/]")
         with ci_group("Downloading constraints for all Airflow versions for all historical Python versions"):
-            with Pool() as pool:
+            # Use the "spawn" start method rather than the platform default: GitPython
+            # (used in the workers via get_tag_date) opens persistent `git cat-file --batch`
+            # subprocesses and is not fork-safe, and the parent already holds open network
+            # sockets from the version/constraints downloads. Forking that state into workers
+            # deadlocks; spawn gives each worker a clean interpreter.
+            with get_context("spawn").Pool() as pool:
                 # We use partial to pass the common parameters to the function
                 get_constraints_for_python_version_partial = partial(
                     get_constraints_for_python_version,