diff --git a/.gitignore b/.gitignore index 010ef540..b63c733a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ !temoa/ !tests/ !docs/ -!data_files/ !notebooks/ !output_files/ !.github/ diff --git a/docs/source/computational_implementation.rst b/docs/source/computational_implementation.rst index bc21a124..023964f3 100644 --- a/docs/source/computational_implementation.rst +++ b/docs/source/computational_implementation.rst @@ -593,7 +593,7 @@ You can also explore the various development branches in the repository: .. code:: $ ls - data_files stochastic temoa_model create_archive.sh README.txt + stochastic temoa tests pyproject.toml README.md $ git branch -a * energysystem diff --git a/docs/source/quick_start.rst b/docs/source/quick_start.rst index fb929a8b..12272620 100644 --- a/docs/source/quick_start.rst +++ b/docs/source/quick_start.rst @@ -98,8 +98,8 @@ Running Temoa ------------- To run the model, a configuration (``config``) file is needed. An -example config file called :code:`config_sample.toml` resides within the -:code:`data_files/my_configs` folder. Running the model with a config file allows +example config file called :code:`config_sample.toml` is available as a package resource +in :code:`temoa/tutorial_assets`. Running the model with a config file allows the user to (1) use a sqlite database for storing input and output data, (2) create a formatted Excel output file, (2) specify the solver to use, (3) return the log file produced during model execution, (4) return the lp file utilized by diff --git a/temoa/cli.py b/temoa/cli.py index 66b1d498..fa8c8587 100644 --- a/temoa/cli.py +++ b/temoa/cli.py @@ -443,7 +443,8 @@ def migrate( if input_path.is_dir(): if output_path is not None: rich.print( - '[yellow]Warning: --output is ignored when migrating a directory. Originals are overwritten after backup.[/yellow]' + '[yellow]Warning: --output is ignored when migrating a directory. Originals are ' + 'overwritten after backup.[/yellow]' ) migration_script = Path(__file__).parent / 'utilities' / 'master_migration.py' @@ -570,6 +571,7 @@ def _copy_tutorial_resources(target_config: Path, target_database: Path) -> None base = resources.files('temoa') / 'tutorial_assets' config_resource = base / 'config_sample.toml' sql_resource = base / 'utopia.sql' + mc_settings_resource = base / 'mc_settings.csv' # Copy configuration file with config_resource.open('rb') as source: @@ -585,11 +587,18 @@ def _copy_tutorial_resources(target_config: Path, target_database: Path) -> None with sqlite3.connect(target_database) as conn: conn.executescript(sql_content) + # Copy Monte Carlo settings + with mc_settings_resource.open('rb') as source: + target_mc = target_config.parent / 'mc_settings.csv' + with open(target_mc, 'wb') as target: + shutil.copyfileobj(source, target) + except (ModuleNotFoundError, FileNotFoundError, AttributeError) as e: logger.exception('Failed to load tutorial resources from package') # Fallback to development paths (for development environments) fallback_config = Path(__file__).parent / 'tutorial_assets' / 'config_sample.toml' fallback_sql = Path(__file__).parent / 'tutorial_assets' / 'utopia.sql' + fallback_mc = Path(__file__).parent / 'tutorial_assets' / 'mc_settings.csv' if not fallback_config.exists(): raise FileNotFoundError( @@ -614,6 +623,9 @@ def _copy_tutorial_resources(target_config: Path, target_database: Path) -> None with sqlite3.connect(target_database) as conn: conn.executescript(fallback_sql.read_text(encoding='utf-8')) + # Copy mc_settings from fallback + shutil.copy2(fallback_mc, target_config.parent / 'mc_settings.csv') + def _update_toml_database_paths(config_path: Path, new_database_name: str) -> None: """ diff --git a/temoa/extensions/method_of_morris/MM_README.md b/temoa/extensions/method_of_morris/MM_README.md index c539352d..74f30a70 100644 --- a/temoa/extensions/method_of_morris/MM_README.md +++ b/temoa/extensions/method_of_morris/MM_README.md @@ -34,12 +34,14 @@ relevant `config.toml` file which can then be run. ### Example: `morris_utopia` -1. Convert the `.sql` file in the `example_dbs` folder back to a database: +1. Convert the `.sql` source file to a database. If you have the `morris_utopia.sql` file: -`data_files/example_dbs % sqlite3 morris_utopia.sqlite < morris_utopia.sql` + ```bash + sqlite3 morris_utopia.sqlite < morris_utopia.sql + ``` 2. Observe the markings (3 groups) in the `MMAnalysis` columns in `cost_variable` and `efficiency`. -3. Observe the `morris` configuration comments in the corresponding `morris_utopia.toml` file in `my_configs`. +3. Observe the `morris` configuration comments in the relevant config file (e.g., `morris_utopia.toml`). 4. Run the config as normal. 5. MM analysis is reported on screen and in 2 csv files for the objective and `co2` in the Outputs folder 6. The DB will contain updated values (tagged by scenario name and "dash run") in `output_objective` and `output_emissions` diff --git a/temoa/extensions/method_of_morris/Method_of_Morris_README.txt b/temoa/extensions/method_of_morris/Method_of_Morris_README.txt index 9c495761..c1fb1d7d 100644 --- a/temoa/extensions/method_of_morris/Method_of_Morris_README.txt +++ b/temoa/extensions/method_of_morris/Method_of_Morris_README.txt @@ -1,7 +1,6 @@ - NOTE: The below README is OUTDATED but it describes the history of MM application. Current users should refer to the MM_README.md file in this directory for how-to and -walk-through on the example provided in data_files! +walk-through on the provided example files! ----------------------- Method of Morris README diff --git a/temoa/extensions/method_of_morris/morris.py b/temoa/extensions/method_of_morris/morris.py index 6af977a5..7619303b 100644 --- a/temoa/extensions/method_of_morris/morris.py +++ b/temoa/extensions/method_of_morris/morris.py @@ -67,11 +67,17 @@ def evaluate(param_names: dict[int, list[Any]], param_values: Any, perturbation_coefficient = 0.2 # minus plus 10% of the baseline values param_file = morris_root / 'm_params.txt' -db_resource = resources.files('data_files.untracked_data.morris') / 'morris_utopia.sqlite' -config_resource = resources.files('data_files.untracked_data.morris') / 'morris_utopia.toml' -with resources.as_file(db_resource) as db_file, \ - resources.as_file(config_resource) as config_path, \ - sqlite3.connect(str(db_file)) as con: +db_file = Path(__file__).parent / 'morris_utopia.sqlite' +config_path = Path(__file__).parent / 'morris_utopia.toml' + +if not db_file.exists() or not config_path.exists(): + raise FileNotFoundError( + "Example Morris data files not found in current directory. " + "Please see MM_README.md for instructions on how to generate " + "or provide the required 'morris_utopia.sqlite' and 'morris_utopia.toml'." + ) + +with sqlite3.connect(str(db_file)) as con: with open(param_file, 'w') as file: param_names = {} cur = con.cursor() diff --git a/temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py b/temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py index d2d179ca..2593ca8a 100644 --- a/temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py +++ b/temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py @@ -3,13 +3,23 @@ """ from importlib import resources from math import sqrt +from pathlib import Path from sqlite3 import Connection from matplotlib import pyplot as plt scenario_name = 'Purple Onion' # must match config file -db_resource = resources.files('data_files.example_dbs') / 'utopia.sqlite' -with resources.as_file(db_resource) as db_path, Connection(str(db_path)) as conn: +# To run this example, ensure tutorial_database.sqlite is in your current directory. +# You can generate it using: temoa tutorial +db_resource = 'tutorial_database.sqlite' + +if not Path(db_resource).exists(): + raise FileNotFoundError( + f"Database file '{db_resource}' not found. " + "Please run 'temoa tutorial' to create the required files." + ) + +with Connection(db_resource) as conn: cur = conn.cursor() obj_values = cur.execute( f"SELECT total_system_cost FROM output_objective WHERE scenario LIKE '{scenario_name}-%'" diff --git a/temoa/tutorial_assets/config_sample.toml b/temoa/tutorial_assets/config_sample.toml index 3943bf18..755787e2 100644 --- a/temoa/tutorial_assets/config_sample.toml +++ b/temoa/tutorial_assets/config_sample.toml @@ -19,13 +19,13 @@ scenario = "zulu" scenario_mode = "perfect_foresight" # Input database (Mandatory) -input_database = "data_files/example_dbs/utopia.sqlite" +input_database = "utopia.sqlite" # Output file (Mandatory) # The output file must be an existing .sqlite file # For Perfect Foresight, the user may target the same input file or a separate / # copied sqlite file in a different location. Myopic, MGA require that input_database = output_database -output_database = "data_files/example_dbs/utopia.sqlite" +output_database = "utopia.sqlite" # ------------------------------------ # DATABASE CONFIGURATION @@ -211,4 +211,4 @@ activity_labels = [] [monte_carlo] # a path from the PROJECT ROOT to the settings file that contains the run data. -run_settings = 'data_files/monte_carlo/run_settings_1.csv' +run_settings = 'mc_settings.csv' diff --git a/temoa/tutorial_assets/mc_settings.csv b/temoa/tutorial_assets/mc_settings.csv new file mode 100644 index 00000000..f4b6b6eb --- /dev/null +++ b/temoa/tutorial_assets/mc_settings.csv @@ -0,0 +1,3 @@ +run,param,index,mod,value,notes +1,cost_invest,utopia|TXD|2010,a,-44.0,reduce invest cost to 1000 +2,demand,utopia|2010|RH,r,0.1,increase demand by 10% diff --git a/temoa/utilities/db_migration_to_v3.py b/temoa/utilities/db_migration_to_v3.py index 19c7999e..40baafb8 100644 --- a/temoa/utilities/db_migration_to_v3.py +++ b/temoa/utilities/db_migration_to_v3.py @@ -11,6 +11,7 @@ import sqlite3 import sys from collections import defaultdict +from importlib import resources from pathlib import Path parser = argparse.ArgumentParser() @@ -23,14 +24,28 @@ ) parser.add_argument( '--schema', - help='Path to schema file (default=data_files/temoa_schema_v3.sql)', + help='Path to v3 schema file (defaults to bundled package resource)', required=False, dest='schema', - default='data_files/temoa_schema_v3.sql', + default=None, ) options = parser.parse_args() legacy_db: Path = Path(options.source_db) -schema_file = Path(options.schema) + +# Resolve schema path +if options.schema: + schema_file = Path(options.schema) +else: + try: + # Try package resources first + schema_file = Path(str(resources.files('temoa.db_schema') / 'temoa_schema_v3.sql')) + except (ModuleNotFoundError, FileNotFoundError, AttributeError): + # Fallback to local path relative to this script + schema_file = Path(__file__).parent.parent / 'db_schema' / 'temoa_schema_v3.sql' + +if not schema_file.exists(): + print(f'Error: Schema file not found: {schema_file}') + sys.exit(1) new_db_name = legacy_db.stem + '_v3.sqlite'