basic functionality for modules

Author: lchladek

docs/user/gourd.toml.5.tex

@@ -139,6 +139,11 @@
                 For example one account available on DelftBlue is "Education-EEMCS-MSc-CS".
                 To get a list of available accounts on your cluster,
                 you can use Slurm's \Prog{sacctmgr}~\Arg{show}~\Arg{account} command
+              \item[\Opt{modules} = list of strings]
+                Modules to load when running through Slurm using the "module" utility.
+                These are loaded in sequential order.
+                For example, to use R in the DelftBlue cluster, \texttt{["2024r1", "r/3.4.0"]}.
+                By default, no modules are loaded.
               \item[\Opt{mail\_type} = string]        
                 Choose one of Slurm's options for sending emails when a run's status changes.
                 Default is "NONE". Valid options are: 

src/gourd/init/interactive.rs

@@ -7,6 +7,7 @@ use gourd_lib::bailc;
 use gourd_lib::config::slurm::SlurmConfig;
 use gourd_lib::config::Config;
 use gourd_lib::constants::CMD_STYLE;
+use gourd_lib::constants::EMPTY_MODULES;
 use gourd_lib::constants::WRAPPER_DEFAULT;
 use gourd_lib::ctx;
 use gourd_lib::file_system::FileOperations;
@@ -103,6 +104,7 @@ pub fn init_interactive(
             experiment_name: "my-experiment".to_string(),
             output_folder: PathBuf::from("./slurmout/"),
             partition: "".to_string(),
+            modules: EMPTY_MODULES(),
             array_size_limit: None,
             max_submit: None,
             account: "".to_string(),

src/gourd/slurm/handler.rs

@@ -143,6 +143,17 @@ pub fn parse_optional_args(slurm_config: &SlurmConfig) -> String {
     result
 }
 
+/// Helper function to create string with modules for slurm
+pub fn parse_modules(slurm_config: &SlurmConfig) -> String {
+    let mut result = "".to_string();
+
+    for module in &slurm_config.modules {
+        result.push_str(&format!("module load {}\n", module))
+    }
+
+    result
+}
+
 #[cfg(test)]
 #[path = "tests/handler.rs"]
 mod tests;

src/gourd/slurm/interactor.rs

@@ -21,6 +21,7 @@ use log::debug;
 use log::info;
 use log::trace;
 
+use super::handler::parse_modules;
 use super::handler::parse_optional_args;
 use super::SacctOutput;
 use crate::chunks::Chunk;
@@ -215,6 +216,7 @@ impl SlurmInteractor for SlurmCli {
         let chunk_index = experiment.register_runs(&chunk.runs);
 
         let optional_args = parse_optional_args(slurm_config);
+        let modules = parse_modules(slurm_config);
 
         // `%A` gets replaced with array *job* id, `%a` with the array *task* id
         // this is read in `src/gourd/status/slurm_files.rs` to get the output.
@@ -239,6 +241,7 @@ impl SlurmInteractor for SlurmCli {
 #SBATCH --error={:?}
 {}
 set -x
+{}
 
 {} {} {} $SLURM_ARRAY_TASK_ID
 ",
@@ -253,6 +256,7 @@ set -x
             slurm_out,
             slurm_err,
             optional_args,
+            modules,
             experiment.wrapper,
             exp_path.display(),
             chunk_index

src/gourd/slurm/tests/handler.rs

@@ -11,6 +11,7 @@ fn parse_optional_args_test_all() {
         experiment_name: "test experiment".to_string(),
         output_folder: Default::default(),
         partition: "memory".to_string(),
+        modules: Default::default(),
         array_size_limit: None,
         max_submit: None,
         account: "test-account".to_string(),
@@ -34,6 +35,7 @@ fn parse_optional_args_test_only_begin() {
         experiment_name: "test experiment".to_string(),
         output_folder: Default::default(),
         partition: "memory".to_string(),
+        modules: Default::default(),
         array_size_limit: None,
         max_submit: None,
         account: "test-account".to_string(),
@@ -69,6 +71,7 @@ fn parse_optional_args_test_custom_args() {
         experiment_name: "test experiment".to_string(),
         output_folder: Default::default(),
         partition: "memory".to_string(),
+        modules: Default::default(),
         array_size_limit: None,
         max_submit: None,
         account: "test-account".to_string(),
@@ -86,3 +89,47 @@ fn parse_optional_args_test_custom_args() {
 
     assert_eq!(output, desired_output)
 }
+
+#[test]
+fn parse_modules_test_empty() {
+    let config = SlurmConfig {
+        experiment_name: "test experiment".to_string(),
+        output_folder: Default::default(),
+        partition: "memory".to_string(),
+        modules: Vec::new(),
+        array_size_limit: None,
+        max_submit: None,
+        account: "test-account".to_string(),
+        begin: None,
+        mail_type: Some("ALL".to_string()),
+        mail_user: Some("testUSER".to_string()),
+        additional_args: None,
+    };
+    let output = parse_modules(&config);
+    let desired_output = "";
+
+    assert_eq!(output, desired_output)
+}
+
+#[test]
+fn parse_modules_test_not_empty() {
+    let config = SlurmConfig {
+        experiment_name: "test experiment".to_string(),
+        output_folder: Default::default(),
+        partition: "memory".to_string(),
+        modules: vec!["2024r1".to_string(), "r/3.4.0".to_string()],
+        array_size_limit: None,
+        max_submit: None,
+        account: "test-account".to_string(),
+        begin: None,
+        mail_type: Some("ALL".to_string()),
+        mail_user: Some("testUSER".to_string()),
+        additional_args: None,
+    };
+    let output = parse_modules(&config);
+    let desired_output = "module load 2024r1
+module load r/3.4.0
+";
+
+    assert_eq!(output, desired_output)
+}

src/gourd_lib/config/slurm.rs

@@ -6,6 +6,8 @@ use std::time::Duration;
 use serde::Deserialize;
 use serde::Serialize;
 
+use crate::constants::EMPTY_MODULES;
+
 /// The config options when running through Slurm
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -27,6 +29,10 @@ pub struct SlurmConfig {
     /// - "visual"
     pub partition: String,
 
+    /// Which modules to load using the 'modules' command
+    #[serde(default = "EMPTY_MODULES")]
+    pub modules: Vec<String>,
+
     /// Override the maximum number of jobs to schedule in a Slurm array.
     ///
     /// If left `None`, a value fetched directly from slurm will be used.

src/gourd_lib/constants.rs

@@ -36,6 +36,9 @@ pub const LABEL_OVERLAP_DEFAULT: fn() -> bool = || false;
 /// The default arguments for an input.
 pub const EMPTY_ARGS: fn() -> Vec<String> = Vec::new;
 
+/// The default modules for a Slurm job.
+pub const EMPTY_MODULES: fn() -> Vec<String> = Vec::new;
+
 /// The prefix which will cause an argument to be interpreted as a glob.
 /// Ensure matches:
 /// - docs/user/gourd.toml.5