removing and mitigating all remaining references to data_files

.gitignore

@@ -5,7 +5,6 @@
 !temoa/
 !tests/
 !docs/
-!data_files/
 !notebooks/
 !output_files/
 !.github/

docs/source/computational_implementation.rst

@@ -593,7 +593,7 @@ You can also explore the various development branches in the repository:
 .. code::
 
    $ ls
-   data_files  stochastic  temoa_model  create_archive.sh  README.txt
+   stochastic  temoa  tests  pyproject.toml  README.md
 
    $ git branch -a
    * energysystem

docs/source/quick_start.rst

@@ -98,8 +98,8 @@ Running Temoa
 -------------
 
 To run the model, a configuration (``config``) file is needed. An
-example config file called :code:`config_sample.toml` resides within the
-:code:`data_files/my_configs` folder. Running the model with a config file allows
+example config file called :code:`config_sample.toml` is available as a package resource
+in :code:`temoa/tutorial_assets`. Running the model with a config file allows
 the user to (1) use a sqlite database for storing input and output data, (2)
 create a formatted Excel output file, (2) specify the solver to use, (3) return
 the log file produced during model execution, (4) return the lp file utilized by

temoa/cli.py

@@ -443,7 +443,8 @@ def migrate(
     if input_path.is_dir():
         if output_path is not None:
             rich.print(
-                '[yellow]Warning: --output is ignored when migrating a directory. Originals are overwritten after backup.[/yellow]'
+                '[yellow]Warning: --output is ignored when migrating a directory. Originals are '
+                'overwritten after backup.[/yellow]'
             )
 
         migration_script = Path(__file__).parent / 'utilities' / 'master_migration.py'
@@ -570,6 +571,7 @@ def _copy_tutorial_resources(target_config: Path, target_database: Path) -> None
         base = resources.files('temoa') / 'tutorial_assets'
         config_resource = base / 'config_sample.toml'
         sql_resource = base / 'utopia.sql'
+        mc_settings_resource = base / 'mc_settings.csv'
 
         # Copy configuration file
         with config_resource.open('rb') as source:
@@ -585,11 +587,18 @@ def _copy_tutorial_resources(target_config: Path, target_database: Path) -> None
         with sqlite3.connect(target_database) as conn:
             conn.executescript(sql_content)
 
+        # Copy Monte Carlo settings
+        with mc_settings_resource.open('rb') as source:
+            target_mc = target_config.parent / 'mc_settings.csv'
+            with open(target_mc, 'wb') as target:
+                shutil.copyfileobj(source, target)
+
     except (ModuleNotFoundError, FileNotFoundError, AttributeError) as e:
         logger.exception('Failed to load tutorial resources from package')
         # Fallback to development paths (for development environments)
         fallback_config = Path(__file__).parent / 'tutorial_assets' / 'config_sample.toml'
         fallback_sql = Path(__file__).parent / 'tutorial_assets' / 'utopia.sql'
+        fallback_mc = Path(__file__).parent / 'tutorial_assets' / 'mc_settings.csv'
 
         if not fallback_config.exists():
             raise FileNotFoundError(
@@ -614,6 +623,9 @@ def _copy_tutorial_resources(target_config: Path, target_database: Path) -> None
         with sqlite3.connect(target_database) as conn:
             conn.executescript(fallback_sql.read_text(encoding='utf-8'))
 
+        # Copy mc_settings from fallback
+        shutil.copy2(fallback_mc, target_config.parent / 'mc_settings.csv')
+
 
 def _update_toml_database_paths(config_path: Path, new_database_name: str) -> None:
     """

temoa/extensions/method_of_morris/MM_README.md

@@ -34,12 +34,14 @@ relevant `config.toml` file which can then be run.
 
 ### Example:  `morris_utopia`
 
-1. Convert the `.sql` file in the `example_dbs` folder back to a database:
+1. Convert the `.sql` source file to a database. If you have the `morris_utopia.sql` file:
 
-`data_files/example_dbs % sqlite3 morris_utopia.sqlite < morris_utopia.sql`
+   ```bash
+   sqlite3 morris_utopia.sqlite < morris_utopia.sql
+   ```
 
 2. Observe the markings (3 groups) in the `MMAnalysis` columns in `cost_variable` and `efficiency`.
-3. Observe the `morris` configuration comments in the corresponding `morris_utopia.toml` file in `my_configs`.
+3. Observe the `morris` configuration comments in the relevant config file (e.g., `morris_utopia.toml`).
 4. Run the config as normal.
 5. MM analysis is reported on screen and in 2 csv files for the objective and `co2` in the Outputs folder
 6. The DB will contain updated values (tagged by scenario name and "dash run") in `output_objective` and `output_emissions`

temoa/extensions/method_of_morris/Method_of_Morris_README.txt

@@ -1,7 +1,6 @@
-
 NOTE:  The below README is OUTDATED but it describes the history of MM application.
 Current users should refer to the MM_README.md file in this directory for how-to and
-walk-through on the example provided in data_files!
+walk-through on the provided example files!
 
 -----------------------
 Method of Morris README

temoa/extensions/method_of_morris/morris.py

@@ -67,11 +67,17 @@ def evaluate(param_names: dict[int, list[Any]], param_values: Any,
 perturbation_coefficient = 0.2  # minus plus 10% of the baseline values
 param_file = morris_root / 'm_params.txt'
 
-db_resource = resources.files('data_files.untracked_data.morris') / 'morris_utopia.sqlite'
-config_resource = resources.files('data_files.untracked_data.morris') / 'morris_utopia.toml'
-with resources.as_file(db_resource) as db_file, \
-     resources.as_file(config_resource) as config_path, \
-     sqlite3.connect(str(db_file)) as con:
+db_file = Path(__file__).parent / 'morris_utopia.sqlite'
+config_path = Path(__file__).parent / 'morris_utopia.toml'
+
+if not db_file.exists() or not config_path.exists():
+    raise FileNotFoundError(
+        "Example Morris data files not found in current directory. "
+        "Please see MM_README.md for instructions on how to generate "
+        "or provide the required 'morris_utopia.sqlite' and 'morris_utopia.toml'."
+    )
+
+with sqlite3.connect(str(db_file)) as con:
     with open(param_file, 'w') as file:
         param_names = {}
         cur = con.cursor()

temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py

@@ -3,13 +3,23 @@
 """
 from importlib import resources
 from math import sqrt
+from pathlib import Path
 from sqlite3 import Connection
 
 from matplotlib import pyplot as plt
 
 scenario_name = 'Purple Onion'  # must match config file
-db_resource = resources.files('data_files.example_dbs') / 'utopia.sqlite'
-with resources.as_file(db_resource) as db_path, Connection(str(db_path)) as conn:
+# To run this example, ensure tutorial_database.sqlite is in your current directory.
+# You can generate it using: temoa tutorial
+db_resource = 'tutorial_database.sqlite'
+
+if not Path(db_resource).exists():
+    raise FileNotFoundError(
+        f"Database file '{db_resource}' not found. "
+        "Please run 'temoa tutorial' to create the required files."
+    )
+
+with Connection(db_resource) as conn:
     cur = conn.cursor()
     obj_values = cur.execute(
         f"SELECT total_system_cost FROM output_objective WHERE scenario LIKE '{scenario_name}-%'"

temoa/tutorial_assets/config_sample.toml

@@ -19,13 +19,13 @@ scenario = "zulu"
 scenario_mode = "perfect_foresight"
 
 # Input database (Mandatory)
-input_database = "data_files/example_dbs/utopia.sqlite"
+input_database = "utopia.sqlite"
 
 # Output file (Mandatory)
 # The output file must be an existing .sqlite file
 # For Perfect Foresight, the user may target the same input file or a separate /
 # copied sqlite file in a different location.  Myopic, MGA require that input_database = output_database
-output_database = "data_files/example_dbs/utopia.sqlite"
+output_database = "utopia.sqlite"
 
 # ------------------------------------
 #        DATABASE CONFIGURATION
@@ -211,4 +211,4 @@ activity_labels = []
 
 [monte_carlo]
 # a path from the PROJECT ROOT to the settings file that contains the run data.
-run_settings = 'data_files/monte_carlo/run_settings_1.csv'
+run_settings = 'mc_settings.csv'

temoa/tutorial_assets/mc_settings.csv

@@ -0,0 +1,3 @@
+run,param,index,mod,value,notes
+1,cost_invest,utopia|TXD|2010,a,-44.0,reduce invest cost to 1000
+2,demand,utopia|2010|RH,r,0.1,increase demand by 10%

temoa/utilities/db_migration_to_v3.py

@@ -11,6 +11,7 @@
 import sqlite3
 import sys
 from collections import defaultdict
+from importlib import resources
 from pathlib import Path
 
 parser = argparse.ArgumentParser()
@@ -23,14 +24,28 @@
 )
 parser.add_argument(
     '--schema',
-    help='Path to schema file (default=data_files/temoa_schema_v3.sql)',
+    help='Path to v3 schema file (defaults to bundled package resource)',
     required=False,
     dest='schema',
-    default='data_files/temoa_schema_v3.sql',
+    default=None,
 )
 options = parser.parse_args()
 legacy_db: Path = Path(options.source_db)
-schema_file = Path(options.schema)
+
+# Resolve schema path
+if options.schema:
+    schema_file = Path(options.schema)
+else:
+    try:
+        # Try package resources first
+        schema_file = Path(str(resources.files('temoa.db_schema') / 'temoa_schema_v3.sql'))
+    except (ModuleNotFoundError, FileNotFoundError, AttributeError):
+        # Fallback to local path relative to this script
+        schema_file = Path(__file__).parent.parent / 'db_schema' / 'temoa_schema_v3.sql'
+
+if not schema_file.exists():
+    print(f'Error: Schema file not found: {schema_file}')
+    sys.exit(1)
 
 
 new_db_name = legacy_db.stem + '_v3.sqlite'