[v3-3-test] Fix generate-providers-metadata hang by using spawn pools (#69763)

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -34,7 +34,7 @@
 from datetime import datetime
 from enum import Enum
 from functools import partial
-from multiprocessing import Pool
+from multiprocessing import get_context
 from pathlib import Path
 from subprocess import DEVNULL
 from typing import IO, TYPE_CHECKING, Any, Literal, NamedTuple
@@ -3598,7 +3598,10 @@ def generate_providers_metadata(
     )
 
     console_print("\n[info]Checking provider.yaml versions[1:] against PyPI for stale entries...[/]\n")
-    with Pool() as pypi_pool:
+    # "spawn" (not the platform-default fork): the parent has already used GitPython and
+    # opened network sockets before reaching here, and forking that state into workers
+    # deadlocks. See get_all_constraint_files_and_airflow_releases for the same reasoning.
+    with get_context("spawn").Pool() as pypi_pool:
         pruned_per_provider = pypi_pool.map(prune_unreleased_versions_from_provider_yaml, package_ids)
     total_pruned = 0
     for pid, pruned in zip(package_ids, pruned_per_provider):
@@ -3623,7 +3626,7 @@ def generate_providers_metadata(
         airflow_release_dates=airflow_release_dates,
         current_metadata=current_metadata,
     )
-    with Pool() as pool:
+    with get_context("spawn").Pool() as pool:
         results = pool.map(
             partial_generate_providers_metadata,
             package_ids,

dev/breeze/src/airflow_breeze/utils/provider_dependencies.py

@@ -26,7 +26,7 @@
 import urllib.request
 from collections.abc import Generator
 from functools import cache, partial
-from multiprocessing import Pool
+from multiprocessing import get_context
 from pathlib import Path
 from threading import Lock
 from typing import NamedTuple
@@ -274,7 +274,12 @@ def get_all_constraint_files_and_airflow_releases(
         airflow_release_dates_path.write_text(json.dumps(airflow_release_dates, indent=2))
         console_print(f"[info]Airflow release dates saved in: {airflow_release_dates_path}[/]")
         with ci_group("Downloading constraints for all Airflow versions for all historical Python versions"):
-            with Pool() as pool:
+            # Use the "spawn" start method rather than the platform default: GitPython
+            # (used in the workers via get_tag_date) opens persistent `git cat-file --batch`
+            # subprocesses and is not fork-safe, and the parent already holds open network
+            # sockets from the version/constraints downloads. Forking that state into workers
+            # deadlocks; spawn gives each worker a clean interpreter.
+            with get_context("spawn").Pool() as pool:
                 # We use partial to pass the common parameters to the function
                 get_constraints_for_python_version_partial = partial(
                     get_constraints_for_python_version,