[WIP] [211] Initialize module search paths via PyConfig for 3.8+

Author: SwooshyCueb

src/main.cpp

@@ -103,7 +103,7 @@ static std::recursive_mutex python_mutex;
 
 namespace
 {
-	const char* const rule_engine_name = "python";
+	static inline constexpr const char* const rule_engine_name = "python";
 	using log_re = irods::experimental::log::rule_engine;
 
 	// Python thread states and other data from the interpreter
@@ -118,6 +118,13 @@ namespace
 		static thread_local PyThreadState* ts_thread_old;
 		// Reference counter for nested python operations
 		static thread_local uint64_t ts_thread_refct = 0;
+
+#if PY_VERSION_HEX >= 0x03080000
+		// List of default module search paths
+		static std::vector<std::wstring> default_module_search_paths;
+		// Whether or not default_module_search_paths is populated
+		static bool default_module_search_paths_set = false;
+#endif
 	} //namespace python_state
 }
 
@@ -378,6 +385,52 @@ namespace
 		bp::class_<CallbackWrapper>("CallbackWrapper", bp::no_init)
 			.def("__getattribute__", &CallbackWrapper::getAttribute);
 	}
+
+	static inline void initialize_python(const std::string& _instance_name)
+	{
+		auto etc_irods_path = irods::get_irods_config_directory();
+
+#if PY_VERSION_HEX >= 0x03080000
+		if (python_state::default_module_search_paths_set) {
+			PyConfig py_config;
+			PyConfig_InitPythonConfig(&py_config);
+
+			py_config.install_signal_handlers = 0;
+
+			for (auto&& module_search_path : python_state::default_module_search_paths) {
+				PyWideStringList_Append(&py_config.module_search_paths, module_search_path.c_str());
+			}
+			PyWideStringList_Append(&py_config.module_search_paths, etc_irods_path.generic_wstring().c_str());
+			py_config.module_search_paths_set = 1;
+
+			PyStatus py_status = Py_InitializeFromConfig(&py_config);
+
+			if (!PyStatus_Exception(py_status)) {
+				return;
+			}
+
+			// clang-format off
+			log_re::error({
+				{"rule_engine_plugin", rule_engine_name},
+				{"instance_name", _instance_name},
+				{"log_message", "failed to initialize interpreter with PyConfig; falling back to legacy initialization"},
+				{"PyStatus.exitcode", fmt::to_string(py_status.exitcode)},
+				{"PyStatus.err_msg", py_status.err_msg},
+				{"PyStatus.func", py_status.func},
+			});
+			// clang-format on
+		}
+#endif
+
+		Py_InitializeEx(0);
+#if PY_VERSION_HEX < 0x03070000
+		PyEval_InitThreads();
+#endif
+		bp::object mod_sys = bp::import("sys");
+		bp::list sys_path = bp::extract<bp::list>(mod_sys.attr("path"));
+		sys_path.append(bp::str(etc_irods_path.generic_string().c_str()));
+	}
+
 } // anonymous namespace
 
 static irods::error start(irods::default_re_ctx&, const std::string& _instance_name)
@@ -391,16 +444,8 @@ static irods::error start(irods::default_re_ctx&, const std::string& _instance_n
 			PyImport_AppendInittab("plugin_wrappers", &PyInit_plugin_wrappers);
 			PyImport_AppendInittab("irods_types", &PyInit_irods_types);
 			PyImport_AppendInittab("irods_errors", &PyInit_irods_errors);
-			Py_InitializeEx(0);
-#if PY_VERSION_HEX < 0x03070000
-			PyEval_InitThreads();
-#endif
-			boost::filesystem::path etc_irods_path = irods::get_irods_config_directory();
-			std::string exec_str = "import sys\nsys.path.append('" + etc_irods_path.generic_string() + "')";
 
-			bp::object main_module = bp::import("__main__");
-			bp::object main_namespace = main_module.attr("__dict__");
-			bp::exec(exec_str.c_str(), main_namespace);
+			initialize_python(_instance_name);
 
 			bp::object plugin_wrappers = bp::import("plugin_wrappers");
 			bp::object irods_types = bp::import("irods_types");
@@ -1022,5 +1067,34 @@ extern "C" irods::pluggable_rule_engine<irods::default_re_ctx>* plugin_factory(c
 		std::function<irods::error(irods::default_re_ctx&, const std::string&, msParamArray_t*, irods::callback)>(
 			exec_rule_expression));
 
+#if PY_VERSION_HEX >= 0x03080000
+	Py_InitializeEx(0);
+	try {
+		bp::object mod_sys = bp::import("sys");
+		bp::list sys_path = bp::extract<bp::list>(mod_sys.attr("path"));
+		std::size_t sys_path_len = bp::extract<std::size_t>(sys_path.attr("__len__")());
+		python_state::default_module_search_paths.reserve(sys_path_len);
+		for (std::size_t i = 0; i < sys_path_len; ++i) {
+			python_state::default_module_search_paths.push_back(bp::extract<std::wstring>(sys_path[i]));
+		}
+		python_state::default_module_search_paths_set = true;
+	}
+	catch (const bp::error_already_set&) {
+		const std::string formatted_python_exception = extract_python_exception();
+		// clang-format off
+		log_re::error({
+			{"rule_engine_plugin", rule_engine_name},
+			{"instance_name", _inst_name},
+			{"log_message", "caught python exception in plugin factory; will use legacy module search path initialization"},
+			{"python_exception", formatted_python_exception},
+		});
+		// clang-format on
+		python_state::default_module_search_paths.clear();
+		python_state::default_module_search_paths.shrink_to_fit();
+		python_state::default_module_search_paths_set = false;
+	}
+	Py_Finalize(); // We're not supposed to do this, but let's try it anyway.
+#endif
+
 	return re;
 }