Windows shutdown may race triggering crashes or hangs on workloads that oversaturate the CPU

[ngoldbaum]

I used an AI model to debug this and help with the issue description. I take responsibility for everything here though.

I observed a crash in OpenBLAS while trying to reproduce a user report of a NumPy issue:

numpy/numpy#32168 (comment)

blas_shutdown takes alloc_lock, then calls .func on every entry below release_pos:

OpenBLAS/driver/others/memory.c

Lines 3243 to 3254 in e016600

	#ifdef SMP
	BLASFUNC(blas_thread_shutdown)();
	#endif
	LOCK_COMMAND(&alloc_lock);
	for (pos = 0; pos < release_pos; pos ++) {
	if (likely(pos < NUM_BUFFERS))
	release_info[pos].func(&release_info[pos]);
	else
	new_release_info[pos-NUM_BUFFERS].func(&new_release_info[pos-NUM_BUFFERS]);
	}

alloc_mmap holds that lock across the counter bump and the two stores, so it is mutually exclusive with that loop:

OpenBLAS/driver/others/memory.c

Lines 2152 to 2166 in e016600

	if (map_address != (void *)-1) {
	#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
	LOCK_COMMAND(&alloc_lock);
	#endif
	int rpos = release_pos++;
	if (likely(rpos < NUM_BUFFERS)) {
	release_info[rpos].address = map_address;
	release_info[rpos].func = alloc_mmap_free;
	} else {
	new_release_info[rpos-NUM_BUFFERS].address = map_address;
	new_release_info[rpos-NUM_BUFFERS].func = alloc_mmap_free;
	}
	#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
	UNLOCK_COMMAND(&alloc_lock);
	#endif

alloc_windows does the identical sequence with no lock:

OpenBLAS/driver/others/memory.c

Lines 2435 to 2444 in e016600

	if (map_address != (void *)-1) {
	int rpos = release_pos++;
	if (likely(rpos < NUM_BUFFERS)) {
	release_info[rpos].address = map_address;
	release_info[rpos].func = alloc_windows_free;
	} else {
	new_release_info[rpos-NUM_BUFFERS].address = map_address;
	new_release_info[rpos-NUM_BUFFERS].func = alloc_windows_free;
	}
	}

release_pos is _Atomic, so the bump publishes immediately while the two stores after it are unordered. A slot can be counted in release_pos with func not yet written, and blas_shutdown will call it.

alloc_mmap is the only back-end that takes the lock. The others do not:
alloc_malloc,
alloc_qalloc,
alloc_windows,
alloc_devicedirver,
alloc_shm,
alloc_hugetlb,
alloc_hugetlbfile.

Windows uses alloc_windows, Linux uses alloc_mmap — which is why this only bites on Windows.

Reproducer

This originally happened on a system that was seeing lots of contention due to CPU oversaturation in a Python script using NumPy. I asked an AI model to make a pure-C reproducer and it came up with this.

ob_race.c:

Details

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdatomic.h>
#include <windows.h>
#include <cblas.h>

enum { M = 200, K = 120, N = 90 };

static atomic_int started, warmed;
static int warmup = 2;
static volatile int keep_going = 1;

static DWORD WINAPI worker(LPVOID p)
{
	double *A = malloc(sizeof *A * M * K);
	double *B = malloc(sizeof *B * K * N);
	double *C = malloc(sizeof *C * M * N);
	for (int i = 0; i < M * K; i++) A[i] = (i % 1000) / 1000.0;
	for (int i = 0; i < K * N; i++) B[i] = (i % 997) / 997.0;

	atomic_fetch_add(&started, 1);

	for (int i = 0; keep_going; i++) {
		cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
		            M, N, K, 1.0, A, K, B, N, 0.0, C, N);
		if (i == warmup)
			atomic_fetch_add(&warmed, 1);   /* past the buffer-table overflow */
	}

	free(A); free(B); free(C);
	(void)p;
	return 0;
}

int main(int argc, char **argv)
{
	int n = argc > 1 ? atoi(argv[1]) : 96;
	if (argc > 2) warmup = atoi(argv[2]);
	int join = 0, pin = 1;
	for (int i = 3; i < argc; i++) {
		if (!strcmp(argv[i], "--join"))   join = 1;
		if (!strcmp(argv[i], "--no-pin")) pin  = 0;
	}

	printf("%s\n%d caller threads, exit %s joining, affinity %s\n",
	       openblas_get_config(), n, join ? "after" : "WITHOUT",
	       pin ? "pinned to 1 CPU" : "unrestricted");
	fflush(stdout);

	/* Build the worker pool BEFORE restricting affinity -- otherwise OpenBLAS
	   sees a single CPU, creates no workers, and blas_thread_shutdown's
	   terminate loop never runs. */
	openblas_set_num_threads(4);
	{
		double *a = calloc(M * K, sizeof *a), *b = calloc(K * N, sizeof *b),
		       *c = calloc(M * N, sizeof *c);
		cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
		            M, N, K, 1.0, a, K, b, N, 0.0, c, N);
		free(a); free(b); free(c);
	}

	HANDLE *t = malloc(n * sizeof *t);
	for (int i = 0; i < n; i++)
		t[i] = CreateThread(NULL, 0, worker, NULL, 0, NULL);

	/* Wait until every thread is past the overflow threshold, so release_pos
	   has run past NUM_BUFFERS and the auxiliary array is in use. Bounded, so
	   this can never hang -- the race does not need all of them. */
	for (int ms = 0; atomic_load(&warmed) < n && ms < 20000; ms += 5)
		Sleep(5);
	printf("%d/%d threads warmed\n", atomic_load(&warmed), n);
	fflush(stdout);

	if (join) {
		keep_going = 0;
		for (int i = 0; i < n; i++) WaitForSingleObject(t[i], INFINITE);
		puts("joined, exiting cleanly");
		return 0;
	}

	/* Every thread is still inside cblas_dgemm right now.
	   Collapse the whole process onto one CPU *here*, at the point of exit --
	   not earlier, or the run itself livelocks with every caller waiting on
	   workers that cannot be scheduled. With this many runnable threads on one
	   CPU, a worker cannot make blas_thread_shutdown's 50 ms deadline, so
	   TerminateThread fires. Running the exit path at top priority widens it
	   further, and it stays wide even under a debugger. */
	puts("exiting with all threads live");
	fflush(stdout);
	if (pin)
		SetProcessAffinityMask(GetCurrentProcess(), 1);
	SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL);
	return 0;
}

clang ob_race.c -o ob_race.exe -O1 -I <openblas include> <openblas .lib>
./ob_race.exe 96 2

For example, using the version of openblas NumPy bundles in a powershell prompt:

$ uv pip install scipy-openblas64
$ $SP = python -c "import scipy_openblas64 as o; print(o.__path__[0])"
$ clang ob_race.c -o ob_race.exe -O1 -I "$SP\include" `
>>   "-Dcblas_dgemm=scipy_cblas_dgemm64_" `
>>   "-Dopenblas_get_config=scipy_openblas_get_config64_" `
>>   "-Dopenblas_set_num_threads=scipy_openblas_set_num_threads64_" `
>>   "$SP\lib\libscipy_openblas64_.lib"
$ $t = @{ ok = 0; crash = 0; hang = 0 }
1..5 | ForEach-Object {
  $p = Start-Process .\ob_race.exe "96 2" -PassThru -NoNewWindow; $null = $p.Handle
  if ($p.WaitForExit(30000)) {
    if ($p.ExitCode -eq 0) { $t.ok++ }
    else { $t.crash++; "run $($_): exit 0x$('{0:X8}' -f $p.ExitCode)" }
  } else {
    $t.hang++; "run $($_): hang, killed"; $p.Kill(); $null = $p.WaitForExit(5000)
  }
}
$ "$($t.ok) ok, $($t.crash) crash, $($t.hang) hang"

On my i6-8500k with 16 GB of RAM, I get 1 ok, 3 crash, 1 hang after running the above powershell harness.

Warning

This allocates a lot of memory and oversaturates my 6-core CPU and may cause crashes. It crashed Claude Code! https://gist.github.com/ngoldbaum/3b3c8809b907a9ffe5b0dd4495c40d23

Ping @kumaraditya303 @itamarst since you two have been thinking about similar problems.