Initialize mechanism list before initializing OpenSSL in icainit

In libica's library constructor OpenSSL is initialized since commit
05c4e36 "Ensure OpenSSL config is loaded
before creating libica's own library context". This will also cause OpenSSL
to evaluate the OpenSSL config file and load providers configured there.

Even without this commit, it can happen when libica is in FIPS mode, that
OpenSSL gets implicitly initialized by OpenSSL usage within the FIPS power
on tests, which run as part of the library constructor.

If OpenSSL has been initialized by the application already, then the attempt
to initialize OpenSSL again by libica is a no-operation. However, if the libica
constructor is the very first to initialize OpenSSL this can cause problems.
For applications that link against libica (via -lica), the libica library
constructor is run before the applications main function, and thus it will
most likely be the very first OpenSSL initialization attempt.

If the IBMCA provider is configured in the OpenSSL config file, then it
is loaded and initialized during OpenSSL initialization triggered by the libica
constructor. During IBMCA provider initialization, it loads libica and queries
libica's mechanism list.

The problem is that this happens while we are still in the middle of the
libica library constructor, and haven't fully initialized libica.
Especially the mechanism list has not yet been set up, and thus the IBMCA
provider will get a mechanism list where all algorithms are marked as being
not supported. The IBMCA provider will thus not register any of its algorithms,
and thus can not be used subsequentially by the application.

Unfortunately OpenSSL evaluates the config file only once, and also the IBMCA
provider queries the libica mechanism list only once, so the situation will not
change for the lifetime of the calling application.

To fix this, make sure that the mechanism list is initialized before the libica
constructor initializes OpenSSL. That way, the IBMCA provider loaded due to
the OpenSL initialization attempt from within the libica library constructor
will get a valid mechanism list.

Since some mechanisms are dependent on the FIPS flag of libica, it must
evaluate the system's FIPS state before it initializes OpenSSL, but finalize
the FIPS setup (loading the FIPS provider, etc) only after OpenSSL has been
initialized. For that the fips_init() function has been split into two parts.

Signed-off-by: Ingo Franzki <ifranzki@linux.ibm.com>

Author: ifranzki

src/fips.c

@@ -249,7 +249,7 @@ SHA3_KAT(512, 512);
 #endif
 
 void
-fips_init(void)
+fips_get_indicator(void)
 {
 	FILE *fd;
 	char fips_flag;
@@ -271,44 +271,46 @@ fips_init(void)
 		fips_flag = '1';
 
 	if (fips_flag - '0') {
-#if !OPENSSL_VERSION_PREREQ(3, 0)
 		/* Set libica into FIPS mode. */
 		fips |= ICA_FIPS_MODE;
+	}
+}
 
+void
+fips_init(void)
+{
+	if (fips & ICA_FIPS_MODE) {
+#if !OPENSSL_VERSION_PREREQ(3, 0)
 		/* Try to set OpenSSL into FIPS mode. If this is not possible,
 		 * all software fallbacks (including RSA key generation) will
 		 * be disabled. OpenSSL FIPS mode can be queried using the
 		 * FIPS_mode() function. */
 		FIPS_mode_set(1);
 #else
-		fips = 0;
-
 #ifndef NO_FIPS_CONFIG_LOAD
 		/* Allow to skip reading the openssl 3.x fips config. Tests showed
 		 * that this step must be skipped on RHEL9 systems. But on other
 		 * systems, or with a locally built openssl, this step is necessary. */
 		if (!OSSL_LIB_CTX_load_config(openssl_libctx, LIBICA_FIPS_CONFIG)) {
 			syslog(LOG_ERR, "Libica failed to load openssl fips config %s\n",
 					LIBICA_FIPS_CONFIG);
-			fips |= ICA_FIPS_INTEGRITY;
+			fips = ICA_FIPS_INTEGRITY;
 			return;
 		}
 #endif
 
 		openssl_provider = OSSL_PROVIDER_load(openssl_libctx, "fips");
 		if (openssl_provider == NULL) {
 			syslog(LOG_ERR, "Libica failed to load fips provider.\n");
-			fips |= ICA_FIPS_INTEGRITY;
+			fips = ICA_FIPS_INTEGRITY;
 			return;
 		}
 
 		if (!EVP_set_default_properties(openssl_libctx, "fips=yes")) {
 			syslog(LOG_ERR, "Libica failed to set default properties 'fips=yes'\n");
-			fips |= ICA_FIPS_INTEGRITY;
+			fips = ICA_FIPS_INTEGRITY;
 			return;
 		}
-
-		fips |= ICA_FIPS_MODE;
 #endif
 	} else {
 		/* kernel fips flag == 0, load default provider in case we are

src/include/fips.h

@@ -25,6 +25,7 @@ int openssl_in_fips_mode(void);
  * Initialize global fips var to 1 resp. 0 when FIPS_FLAG is 1 resp. 0 (or not
  * present).
  */
+void fips_get_indicator(void);
 void fips_init(void);
 
 /*

src/init.c

@@ -113,6 +113,16 @@ void __attribute__ ((constructor)) icainit(void)
 	if (ptr && sscanf(ptr, "%i", &value) == 1)
 		ica_set_stats_mode(value);
 
+#ifdef ICA_FIPS
+	fips_get_indicator();
+#endif
+
+	rng_init();
+
+	s390_prng_init();
+
+	s390_initialize_functionlist();
+
 #if OPENSSL_VERSION_PREREQ(3, 0)
 	/*
 	 * OpenSSL >= 3.0:
@@ -124,6 +134,18 @@ void __attribute__ ((constructor)) icainit(void)
 	 * is loaded later, which may cause that all configured providers
 	 * are also loaded into the library context. We need to make sure that
 	 * only the default or fips provider is loaded in the library context.
+	 *
+	 * Also make sure that OpenSSL initialization happens AFTER the
+	 * mechanism list has been initialized and the fips indicator has been
+	 * obtained. OPENSSL_init_crypto may load configured providers, and a
+	 * provider might use libica in its initialization function. It must
+	 * be ensured that libica has been initialized that far before OpenSSL
+	 * is initialized, so that such libica calls from providers can be
+	 * fulfilled and return correct information.
+	 *
+	 * The remaining FIPS initialization (fips provider load, power on self
+	 * tests, etc) must still happen after OpenSSL initialization, because
+	 * that relies on OpenSSL being initialized.
 	 */
 	OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL);
 
@@ -135,18 +157,13 @@ void __attribute__ ((constructor)) icainit(void)
 #endif
 
 #ifdef ICA_FIPS
-	fips_init(); /* before powerup tests and initialize functionlist */
+	fips_init();
 #endif
 
 #if OPENSSL_VERSION_PREREQ(3, 0)
 	openssl3_initialized = 1;
 #endif
 
-	rng_init();
-
-	s390_prng_init();
-
-	s390_initialize_functionlist();
 
 #ifdef ICA_FIPS
 	if (fips & ICA_FIPS_MODE)