Merge branch 'main' into add-optional-decommission-year-for-assets

Aurashk · web-flow · commit 78b9c1076676 · 2025-10-27T10:48:50.000Z
diff --git a/docs/glossary.md b/docs/glossary.md
@@ -16,7 +16,7 @@ becomes an *Asset* that they own and operate. An *Asset* is an instance of a *Pr
 specific capacity, and a decommissioning year. A set of *Assets* must exist in the base year
 sufficient to serve base year demands (i.e. a calibrated base year, based on user input data).
 
-**Availability:** The maximum, minimum or fixed percentage of maximum output (or input) that an
+**Availability:** The maximum, minimum or fixed percentage of maximum output (or input) that a
 *Process* delivers over a period. The time period could be a single time slice, a season, or a year.
 
 **Base Year:** The starting year of a model run. The base year is typically calibrated to known
diff --git a/docs/templates/input_files.md.jinja b/docs/templates/input_files.md.jinja
@@ -6,6 +6,7 @@
 <!-- markdownlint-disable MD033 -->
 
 This file contains information about the file formats for MUSE2 input files.
+It is not required to supply every input file - empty files can be omitted.
 
 {{ toml_info }}
 
diff --git a/examples/missing_commodity/agent_search_space.csv b/examples/missing_commodity/agent_search_space.csv
diff --git a/examples/missing_commodity/process_availabilities.csv b/examples/missing_commodity/process_availabilities.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,time_slice,limit_type,value
+process_id,regions,commission_years,time_slice,limit_type,value
 GASDRV,all,all,annual,up,0.9
 GASPRC,all,all,annual,up,0.9
 BIOPRO,all,all,annual,up,1.0
diff --git a/examples/missing_commodity/process_flows.csv b/examples/missing_commodity/process_flows.csv
@@ -1,4 +1,4 @@
-process_id,commodity_id,regions,years,coeff,type,cost
+process_id,commodity_id,regions,commission_years,coeff,type,cost
 GASDRV,GASPRD,all,all,1.0,fixed,
 GASPRC,GASPRD,all,all,-1.05,fixed,
 GASPRC,GASNAT,all,all,1.0,fixed,
diff --git a/examples/missing_commodity/process_parameters.csv b/examples/missing_commodity/process_parameters.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
+process_id,regions,commission_years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
 GASDRV,all,all,10.0,0.3,2.0,25,0.1
 GASPRC,all,all,7.0,0.21,0.5,25,0.1
 BIOPRO,all,all,1.0,0.2,0.25,20,0.09
diff --git a/examples/muse1_default/agent_cost_limits.csv b/examples/muse1_default/agent_cost_limits.csv
diff --git a/examples/muse1_default/agent_search_space.csv b/examples/muse1_default/agent_search_space.csv
diff --git a/examples/muse1_default/process_availabilities.csv b/examples/muse1_default/process_availabilities.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,time_slice,limit_type,value
+process_id,regions,commission_years,time_slice,limit_type,value
 gassupply1,R1,all,annual,up,0.9
 gasCCGT,R1,all,annual,up,0.9
 windturbine,R1,all,annual,up,0.4
diff --git a/examples/muse1_default/process_flows.csv b/examples/muse1_default/process_flows.csv
@@ -1,4 +1,4 @@
-process_id,commodity_id,regions,years,coeff,type,cost
+process_id,commodity_id,regions,commission_years,coeff,type,cost
 gassupply1,gas,R1,all,1,fixed,
 gasCCGT,gas,R1,all,-1.67,fixed,
 gasCCGT,electricity,R1,all,1,fixed,
diff --git a/examples/muse1_default/process_parameters.csv b/examples/muse1_default/process_parameters.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
+process_id,regions,commission_years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
 gassupply1,R1,all,0,0,2.55,60,0.1
 gasCCGT,R1,all,23.78234399,0,0,60,0.1
 windturbine,R1,all,36.30771182,0,0,60,0.1
diff --git a/examples/simple/agent_search_space.csv b/examples/simple/agent_search_space.csv
diff --git a/examples/simple/process_availabilities.csv b/examples/simple/process_availabilities.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,time_slice,limit_type,value
+process_id,regions,commission_years,time_slice,limit_type,value
 GASDRV,all,all,annual,up,0.9
 GASPRC,all,all,annual,up,0.9
 GASCGT,all,all,annual,up,0.9
diff --git a/examples/simple/process_flows.csv b/examples/simple/process_flows.csv
@@ -1,4 +1,4 @@
-process_id,commodity_id,regions,years,coeff,type,cost
+process_id,commodity_id,regions,commission_years,coeff,type,cost
 GASDRV,GASPRD,all,all,1.0,fixed,
 GASPRC,GASPRD,all,all,-1.05,fixed,
 GASPRC,GASNAT,all,all,1.0,fixed,
diff --git a/examples/simple/process_parameters.csv b/examples/simple/process_parameters.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
+process_id,regions,commission_years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
 GASDRV,all,all,10.0,0.3,2.0,25,0.1
 GASPRC,all,all,7.0,0.21,0.5,25,0.1
 WNDFRM,all,all,1000.0,30.0,0.4,25,0.1
diff --git a/examples/two_outputs/agent_search_space.csv b/examples/two_outputs/agent_search_space.csv
diff --git a/examples/two_outputs/process_availabilities.csv b/examples/two_outputs/process_availabilities.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,time_slice,limit_type,value
+process_id,regions,commission_years,time_slice,limit_type,value
 GASDRV,all,all,annual,up,0.9
 OAGRSV,all,all,annual,up,0.9
 GASPRC,all,all,annual,up,0.9
diff --git a/examples/two_outputs/process_flows.csv b/examples/two_outputs/process_flows.csv
@@ -1,4 +1,4 @@
-process_id,commodity_id,regions,years,coeff,type,cost
+process_id,commodity_id,regions,commission_years,coeff,type,cost
 GASDRV,GASPRD,all,all,1,fixed,
 OAGRSV,OILCRD,all,all,1,fixed,
 OAGRSV,GASPRD,all,all,0.1,fixed,
diff --git a/examples/two_outputs/process_parameters.csv b/examples/two_outputs/process_parameters.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
+process_id,regions,commission_years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
 GASDRV,GBR,all,10,0.3,2,25,0.1
 OAGRSV,GBR,all,15,0.45,3,25,0.1
 GASPRC,GBR,all,7,0.21,0.5,25,0.1
diff --git a/examples/two_regions/agent_cost_limits.csv b/examples/two_regions/agent_cost_limits.csv
diff --git a/examples/two_regions/agent_search_space.csv b/examples/two_regions/agent_search_space.csv
diff --git a/examples/two_regions/process_availabilities.csv b/examples/two_regions/process_availabilities.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,time_slice,limit_type,value
+process_id,regions,commission_years,time_slice,limit_type,value
 gassupply1,R1;R2,all,annual,up,0.9
 gasCCGT,R1;R2,all,annual,up,0.9
 windturbine,R1;R2,all,annual,up,0.4
diff --git a/examples/two_regions/process_flows.csv b/examples/two_regions/process_flows.csv
@@ -1,4 +1,4 @@
-process_id,commodity_id,regions,years,coeff
+process_id,commodity_id,regions,commission_years,coeff
 gassupply1,gas,R1;R2,all,1
 gasCCGT,gas,R1;R2,all,-1.67
 gasCCGT,electricity,R1;R2,all,1
diff --git a/examples/two_regions/process_parameters.csv b/examples/two_regions/process_parameters.csv
@@ -1,4 +1,4 @@
-process_id,regions,years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
+process_id,regions,commission_years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate
 gassupply1,R1;R2,all,0,0,2.55,60,0.1
 gasCCGT,R1;R2,all,23.78234399,0,0,60,0.1
 windturbine,R1;R2,all,36.30771182,0,0,60,0.1
diff --git a/src/graph.rs b/src/graph.rs
@@ -198,11 +198,10 @@ fn prepare_commodities_graph_for_validation(
     // NOTE: we only do this for commodities with the same time_slice_level as the selection
     let demand_node_index = filtered_graph.add_node(GraphNode::Demand);
     for (commodity_id, commodity) in commodities {
-        if time_slice_selection.level() == commodity.time_slice_level
-            && commodity
-                .demand
-                .get(&(region_id.clone(), year, time_slice_selection.clone()))
-                .is_some_and(|&v| v > Flow(0.0))
+        if commodity
+            .demand
+            .get(&(region_id.clone(), year, time_slice_selection.clone()))
+            .is_some_and(|&v| v > Flow(0.0))
         {
             let commodity_node = GraphNode::Commodity(commodity_id.clone());
             let commodity_node_index = filtered_graph
diff --git a/src/input.rs b/src/input.rs
@@ -56,6 +56,10 @@ pub fn read_csv<'a, T: DeserializeOwned + 'a>(
 pub fn read_csv_optional<'a, T: DeserializeOwned + 'a>(
     file_path: &'a Path,
 ) -> Result<impl Iterator<Item = T> + 'a> {
+    if !file_path.exists() {
+        return Ok(Vec::new().into_iter());
+    }
+
     let vec = read_csv_internal(file_path)?;
     Ok(vec.into_iter())
 }
@@ -347,6 +351,19 @@ mod tests {
                 .next()
                 .is_none()
         );
+
+        // Missing file
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("a_missing_file.csv");
+        assert!(!file_path.exists());
+        assert!(read_csv::<Record>(&file_path).is_err());
+        // optional csv's should return empty iterator
+        assert!(
+            read_csv_optional::<Record>(&file_path)
+                .unwrap()
+                .next()
+                .is_none()
+        );
     }
 
     #[test]
diff --git a/src/input/commodity/demand.rs b/src/input/commodity/demand.rs
@@ -135,8 +135,8 @@ where
         );
 
         ensure!(
-            demand.demand.is_normal() && demand.demand > Flow(0.0),
-            "Demand must be a valid number greater than zero"
+            demand.demand.is_finite() && demand.demand >= Flow(0.0),
+            "Demand must be a finite number greater than or equal to zero"
         );
 
         ensure!(
@@ -270,6 +270,12 @@ mod tests {
                 commodity_id: "commodity1".to_string(),
                 demand: Flow(11.0),
             },
+            Demand {
+                year: 2020,
+                region_id: "Spain".to_string(),
+                commodity_id: "commodity3".to_string(),
+                demand: Flow(0.0),
+            },
         ];
         assert_error!(
             read_demand_from_iter(demand.into_iter(), &svd_commodities, &region_ids, &[2020]),
@@ -334,7 +340,6 @@ mod tests {
 
     #[rstest]
     #[case(-1.0)]
-    #[case(0.0)]
     #[case(f64::NAN)]
     #[case(f64::NEG_INFINITY)]
     #[case(f64::INFINITY)]
@@ -353,7 +358,7 @@ mod tests {
         }];
         assert_error!(
             read_demand_from_iter(demand.into_iter(), &svd_commodities, &region_ids, &[2020],),
-            "Demand must be a valid number greater than zero"
+            "Demand must be a finite number greater than or equal to zero"
         );
     }
 
diff --git a/src/input/commodity/demand_slicing.rs b/src/input/commodity/demand_slicing.rs
@@ -1,7 +1,5 @@
 //! Demand slicing determines how annual demand is distributed across the year.
-use super::super::{
-    check_values_sum_to_one_approx, deserialise_proportion_nonzero, input_err_msg, read_csv,
-};
+use super::super::{check_values_sum_to_one_approx, input_err_msg, read_csv};
 use crate::commodity::CommodityID;
 use crate::id::IDCollection;
 use crate::input::commodity::demand::BorrowedCommodityMap;
@@ -22,7 +20,6 @@ struct DemandSlice {
     commodity_id: String,
     region_id: String,
     time_slice: String,
-    #[serde(deserialize_with = "deserialise_proportion_nonzero")]
     fraction: Dimensionless,
 }
 
diff --git a/src/input/process/availability.rs b/src/input/process/availability.rs
@@ -21,7 +21,7 @@ const PROCESS_AVAILABILITIES_FILE_NAME: &str = "process_availabilities.csv";
 struct ProcessAvailabilityRaw {
     process_id: String,
     regions: String,
-    years: String,
+    commission_years: String,
     time_slice: String,
     limit_type: LimitType,
     value: Dimensionless,
@@ -138,9 +138,10 @@ where
 
         // Get years
         let process_years = &process.years;
-        let record_years = parse_year_str(&record.years, process_years).with_context(|| {
-            format!("Invalid year for process {id}. Valid years are {process_years:?}")
-        })?;
+        let record_years =
+            parse_year_str(&record.commission_years, process_years).with_context(|| {
+                format!("Invalid year for process {id}. Valid years are {process_years:?}")
+            })?;
 
         // Get time slices
         let ts_selection = time_slice_info.get_selection(&record.time_slice)?;
@@ -261,7 +262,7 @@ mod tests {
         ProcessAvailabilityRaw {
             process_id: "process".into(),
             regions: "region".into(),
-            years: "2010".into(),
+            commission_years: "2010".into(),
             time_slice: "day".into(),
             limit_type,
             value,
diff --git a/src/input/process/flow.rs b/src/input/process/flow.rs
@@ -2,7 +2,7 @@
 use super::super::{input_err_msg, read_csv};
 use crate::commodity::{CommodityID, CommodityMap};
 use crate::process::{FlowType, ProcessFlow, ProcessFlowsMap, ProcessID, ProcessMap};
-use crate::region::parse_region_str;
+use crate::region::{RegionID, parse_region_str};
 use crate::units::{FlowPerActivity, MoneyPerFlow};
 use crate::year::parse_year_str;
 use anyhow::{Context, Result, ensure};
@@ -19,7 +19,7 @@ const PROCESS_FLOWS_FILE_NAME: &str = "process_flows.csv";
 struct ProcessFlowRaw {
     process_id: String,
     commodity_id: String,
-    years: String,
+    commission_years: String,
     regions: String,
     coeff: FlowPerActivity,
     #[serde(default)]
@@ -46,7 +46,7 @@ impl ProcessFlowRaw {
         // Check that flow cost is non-negative
         if let Some(cost) = self.cost {
             ensure!(
-                (0.0..f64::INFINITY).contains(&cost.value()),
+                (cost.value() >= 0.0),
                 "Invalid value for flow cost ({cost}). Must be >=0."
             );
         }
@@ -94,9 +94,10 @@ where
 
         // Get years
         let process_years = &process.years;
-        let record_years = parse_year_str(&record.years, process_years).with_context(|| {
-            format!("Invalid year for process {id}. Valid years are {process_years:?}")
-        })?;
+        let record_years =
+            parse_year_str(&record.commission_years, process_years).with_context(|| {
+                format!("Invalid year for process {id}. Valid years are {process_years:?}")
+            })?;
 
         // Get commodity
         let commodity = commodities
@@ -132,6 +133,7 @@ where
     }
 
     validate_flows_and_update_primary_output(processes, &flows_map)?;
+    validate_secondary_flows(processes, &flows_map)?;
 
     Ok(flows_map)
 }
@@ -229,6 +231,53 @@ fn check_flows_primary_output(
     Ok(())
 }
 
+/// Checks that non-primary io are defined for all years (within a region) and that
+/// they are only inputs or only outputs in all years.
+fn validate_secondary_flows(
+    processes: &mut ProcessMap,
+    flows_map: &HashMap<ProcessID, ProcessFlowsMap>,
+) -> Result<()> {
+    for (process_id, process) in processes.iter() {
+        // Get the flows for this process - there should be no error, as was checked already
+        let map = flows_map
+            .get(process_id)
+            .with_context(|| format!("Missing flows map for process {process_id}"))?;
+
+        // Get the non-primary io flows for all years, if any, arranged by (commodity, region)
+        let iter = iproduct!(process.years.iter(), process.regions.iter());
+        let mut flows: HashMap<(CommodityID, RegionID), Vec<bool>> = HashMap::new();
+        for (&year, region_id) in iter {
+            let flow = map[&(region_id.clone(), year)]
+                .iter()
+                .filter_map(|(commodity_id, flow)| {
+                    (Some(commodity_id) != process.primary_output.as_ref())
+                        .then_some(((commodity_id.clone(), region_id.clone()), flow.is_input()))
+                });
+
+            for (key, value) in flow {
+                flows.entry(key).or_default().push(value);
+            }
+        }
+
+        // Finally we check that the flows for a given commodity and region are defined for all
+        // years and that they are all inputs or all outputs
+        for ((commodity_id, region_id), value) in &flows {
+            ensure!(
+                value.len() == process.years.len(),
+                "Flow of commodity {commodity_id} in region {region_id} for process {process_id} \
+                does not cover all years"
+            );
+            ensure!(
+                value.iter().all(|&x| x == value[0]),
+                "Flow of commodity {commodity_id} in region {region_id} for process {process_id} \
+                behaves as input or output in different years."
+            );
+        }
+    }
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/input/process/parameter.rs b/src/input/process/parameter.rs
@@ -17,7 +17,7 @@ const PROCESS_PARAMETERS_FILE_NAME: &str = "process_parameters.csv";
 struct ProcessParameterRaw {
     process_id: String,
     regions: String,
-    years: String,
+    commission_years: String,
     capital_cost: MoneyPerCapacity,
     fixed_operating_cost: MoneyPerCapacityPerYear,
     variable_operating_cost: MoneyPerActivity,
@@ -111,8 +111,8 @@ where
 
         // Get years
         let process_years = &process.years;
-        let parameter_years =
-            parse_year_str(&param_raw.years, process_years).with_context(|| {
+        let parameter_years = parse_year_str(&param_raw.commission_years, process_years)
+            .with_context(|| {
                 format!("Invalid year for process {id}. Valid years are {process_years:?}")
             })?;
 
@@ -193,7 +193,7 @@ mod tests {
             variable_operating_cost: MoneyPerActivity(0.0),
             lifetime,
             discount_rate,
-            years: "all".to_string(),
+            commission_years: "all".to_string(),
             regions: "all".to_string(),
         }
     }

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-process_id,regions,years,time_slice,limit_type,value`
	`1`	`+process_id,regions,commission_years,time_slice,limit_type,value`
`2`	`2`	`GASDRV,all,all,annual,up,0.9`
`3`	`3`	`GASPRC,all,all,annual,up,0.9`
`4`	`4`	`BIOPRO,all,all,annual,up,1.0`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-process_id,commodity_id,regions,years,coeff,type,cost`
	`1`	`+process_id,commodity_id,regions,commission_years,coeff,type,cost`
`2`	`2`	`GASDRV,GASPRD,all,all,1.0,fixed,`
`3`	`3`	`GASPRC,GASPRD,all,all,-1.05,fixed,`
`4`	`4`	`GASPRC,GASNAT,all,all,1.0,fixed,`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-process_id,regions,years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate`
	`1`	`+process_id,regions,commission_years,capital_cost,fixed_operating_cost,variable_operating_cost,lifetime,discount_rate`
`2`	`2`	`GASDRV,all,all,10.0,0.3,2.0,25,0.1`
`3`	`3`	`GASPRC,all,all,7.0,0.21,0.5,25,0.1`
`4`	`4`	`BIOPRO,all,all,1.0,0.2,0.25,20,0.09`