Skip to content

Commit 138a841

Browse files
committed
FIX #294: make OpenBLAS thread-pool resilient to fork via pthread_atfork
1 parent 046e401 commit 138a841

6 files changed

Lines changed: 177 additions & 2 deletions

File tree

driver/others/blas_server.c

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
7474
#include <sys/resource.h>
7575
#endif
7676

77+
#ifndef likely
78+
#ifdef __GNUC__
79+
#define likely(x) __builtin_expect(!!(x), 1)
80+
#else
81+
#define likely(x) (x)
82+
#endif
83+
#endif
84+
#ifndef unlikely
85+
#ifdef __GNUC__
86+
#define unlikely(x) __builtin_expect(!!(x), 0)
87+
#else
88+
#define unlikely(x) (x)
89+
#endif
90+
#endif
91+
7792
#ifdef SMP_SERVER
7893

7994
#undef MONITOR
@@ -584,6 +599,10 @@ static BLASULONG exec_queue_lock = 0;
584599

585600
int exec_blas_async(BLASLONG pos, blas_queue_t *queue){
586601

602+
#ifdef SMP_SERVER
603+
// Handle lazy re-init of the thread-pool after a POSIX fork
604+
if (unlikely(blas_server_avail == 0)) blas_thread_init();
605+
#endif
587606
BLASLONG i = 0;
588607
blas_queue_t *current = queue;
589608
#if defined(OS_LINUX) && !defined(NO_AFFINITY) && !defined(PARAMTEST)
@@ -708,7 +727,11 @@ int exec_blas_async_wait(BLASLONG num, blas_queue_t *queue){
708727
/* Execute Threads */
709728
int exec_blas(BLASLONG num, blas_queue_t *queue){
710729

711-
int (*routine)(blas_arg_t *, void *, void *, double *, double *, BLASLONG);
730+
#ifdef SMP_SERVER
731+
// Handle lazy re-init of the thread-pool after a POSIX fork
732+
if (unlikely(blas_server_avail == 0)) blas_thread_init();
733+
#endif
734+
int (*routine)(blas_arg_t *, void *, void *, double *, double *, BLASLONG);
712735

713736
#ifdef TIMING_DEBUG
714737
BLASULONG start, stop;

driver/others/memory.c

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
143143
gotoblas_t *gotoblas = NULL;
144144
#endif
145145

146+
extern void openblas_warning(int verbose, const char * msg);
147+
146148
#ifndef SMP
147149

148150
#define blas_cpu_number 1
@@ -253,6 +255,21 @@ int goto_get_num_procs (void) {
253255
return blas_cpu_number;
254256
}
255257

258+
void openblas_fork_handler()
259+
{
260+
// This handler shuts down the OpenBLAS-managed PTHREAD pool when OpenBLAS is
261+
// built with "make USE_OPENMP=0".
262+
// Hanging can still happen when OpenBLAS is built against the libgomp
263+
// implementation of OpenMP. The problem is tracked at:
264+
// http://gcc.gnu.org/bugzilla/show_bug.cgi?id=60035
265+
// In the mean time build with USE_OPENMP=0 or link against another
266+
// implementation of OpenMP.
267+
int err;
268+
err = pthread_atfork (BLASFUNC(blas_thread_shutdown), NULL, NULL);
269+
if(err != 0)
270+
openblas_warning(0, "OpenBLAS Warning ... cannot install fork handler. You may meet hang after fork.\n");
271+
}
272+
256273
int blas_get_cpu_number(void){
257274
char *p;
258275
#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_DARWIN)
@@ -1268,6 +1285,9 @@ void CONSTRUCTOR gotoblas_init(void) {
12681285

12691286
if (gotoblas_initialized) return;
12701287

1288+
#ifdef SMP
1289+
openblas_fork_handler();
1290+
#endif
12711291

12721292
#ifdef PROFILE
12731293
moncontrol (0);

utest/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ CUNIT_LIB=$(CUNIT_DIR)/lib/libcunit.a
1111

1212
CFLAGS+=-I$(CUNIT_DIR)/include
1313

14-
OBJS=main.o test_rot.o test_swap.o test_axpy.o test_dotu.o test_rotmg.o test_dsdot.o test_amax.o
14+
OBJS=main.o test_rot.o test_swap.o test_axpy.o test_dotu.o test_rotmg.o test_dsdot.o test_amax.o test_fork.o
1515

1616
all : run_test
1717

utest/common_utest.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,4 +63,6 @@ void test_dsdot_n_1(void);
6363

6464
void test_samax(void);
6565

66+
void test_fork_safety(void);
67+
6668
#endif

utest/main.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ CU_TestInfo test_level1[]={
6060
{"Testing dsdot with n == 1",test_dsdot_n_1},
6161

6262
{"Testing samax", test_samax},
63+
64+
#if !defined(USE_OPENMP) && !defined(OS_WINDOWS)
65+
// The GNU OpenMP implementation libgomp is not fork-safe (as of 4.8.2):
66+
// http://gcc.gnu.org/bugzilla/show_bug.cgi?id=60035
67+
// Hence skip this test when OpenBLAS is built with OpenMP.
68+
{"Testing fork safety", test_fork_safety},
69+
#endif
70+
6371
CU_TEST_INFO_NULL,
6472
};
6573

utest/test_fork.c

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*****************************************************************************
2+
Copyright (c) 2014, Lab of Parallel Software and Computational Science,ICSAS
3+
All rights reserved.
4+
5+
Redistribution and use in source and binary forms, with or without
6+
modification, are permitted provided that the following conditions are
7+
met:
8+
9+
1. Redistributions of source code must retain the above copyright
10+
notice, this list of conditions and the following disclaimer.
11+
12+
2. Redistributions in binary form must reproduce the above copyright
13+
notice, this list of conditions and the following disclaimer in
14+
the documentation and/or other materials provided with the
15+
distribution.
16+
3. Neither the name of the ISCAS nor the names of its contributors may
17+
be used to endorse or promote products derived from this software
18+
without specific prior written permission.
19+
20+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
29+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30+
31+
**********************************************************************************/
32+
33+
#include "common_utest.h"
34+
#include <sys/wait.h>
35+
#include <cblas.h>
36+
37+
void* xmalloc(size_t n)
38+
{
39+
void* tmp;
40+
tmp = malloc(n);
41+
if (tmp == NULL) {
42+
fprintf(stderr, "You are about to die\n");
43+
exit(1);
44+
} else {
45+
return tmp;
46+
}
47+
}
48+
49+
void check_dgemm(double *a, double *b, double *result, double *expected, int n)
50+
{
51+
int i;
52+
cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, n, n, n,
53+
1.0, a, n, b, n, 0.0, result, n);
54+
for(i = 0; i < n * n; ++i) {
55+
CU_ASSERT_DOUBLE_EQUAL(expected[i], result[i], CHECK_EPS);
56+
}
57+
}
58+
59+
60+
void test_fork_safety(void)
61+
{
62+
int n = 1000;
63+
int i;
64+
65+
double *a, *b, *c, *d;
66+
size_t n_bytes;
67+
68+
pid_t fork_pid;
69+
pid_t fork_pid_nested;
70+
71+
n_bytes = sizeof(*a) * n * n;
72+
73+
a = xmalloc(n_bytes);
74+
b = xmalloc(n_bytes);
75+
c = xmalloc(n_bytes);
76+
d = xmalloc(n_bytes);
77+
78+
// Put ones in a and b
79+
for(i = 0; i < n * n; ++i) {
80+
a[i] = 1;
81+
b[i] = 1;
82+
}
83+
84+
// Compute a DGEMM product in the parent process prior to forking to
85+
// ensure that the OpenBLAS thread pool is initialized.
86+
cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, n, n, n,
87+
1.0, a, n, b, n, 0.0, c, n);
88+
89+
fork_pid = fork();
90+
if (fork_pid == -1) {
91+
CU_FAIL("Failed to fork process.");
92+
} else if (fork_pid == 0) {
93+
// Compute a DGEMM product in the child process to check that the
94+
// thread pool as been properly been reinitialized after the fork.
95+
check_dgemm(a, b, d, c, n);
96+
97+
// Nested fork to check that the pthread_atfork protection can work
98+
// recursively
99+
fork_pid_nested = fork();
100+
if (fork_pid_nested == -1) {
101+
CU_FAIL("Failed to fork process.");
102+
exit(1);
103+
} else if (fork_pid_nested == 0) {
104+
check_dgemm(a, b, d, c, n);
105+
exit(0);
106+
} else {
107+
check_dgemm(a, b, d, c, n);
108+
int child_status = 0;
109+
pid_t wait_pid = wait(&child_status);
110+
CU_ASSERT(wait_pid == fork_pid_nested);
111+
CU_ASSERT(WEXITSTATUS (child_status) == 0);
112+
exit(0);
113+
}
114+
} else {
115+
check_dgemm(a, b, d, c, n);
116+
// Wait for the child to finish and check the exit code.
117+
int child_status = 0;
118+
pid_t wait_pid = wait(&child_status);
119+
CU_ASSERT(wait_pid == fork_pid);
120+
CU_ASSERT(WEXITSTATUS (child_status) == 0);
121+
}
122+
}

0 commit comments

Comments
 (0)