Merge pull request #588 from EnergySystemsModellingLab/document-input-format

Document input format

Author: alexdewar

.github/actions/generate-docs/action.yml

@@ -11,6 +11,12 @@ runs:
     - name: Update CLI documentation
       shell: bash
       run: cargo run -- --markdown-help > docs/command_line_help.md
+    - name: Install Python dependencies
+      shell: bash
+      run: pip install -r doc-requirements.txt
+    - name: Generate input format documentation
+      shell: bash
+      run: python docs/generate_input_format_doc.py
     - name: Build API docs
       shell: bash
       run: |

.gitignore

@@ -19,7 +19,165 @@ target/
 book/
 index.html
 
-# Ignore virtual environment directories
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
 venv/
-.venv/
-.env/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

doc-requirements.txt

@@ -0,0 +1 @@
+table2md

docs/.gitignore

@@ -0,0 +1,3 @@
+# Generated documentation files
+command_line_help.md
+input_format.md

docs/SUMMARY.md

@@ -3,6 +3,7 @@
 [Introduction](./introduction.md)
 
 - [User Guide](./user_guide.md)
+- [Input Format](./input_format.md)
 - [Command Line Help](./command_line_help.md)
 - [Model Description](./model_description.md)
 - [Dispatch Optimisation](./dispatch_optimisation.md)

docs/command_line_help.md

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+#
+# A script to generate markdown documentation from table schemas.
+
+from table2md import MarkdownTable
+import yaml
+from pathlib import Path
+
+_DOCS_DIR = Path(__file__).parent
+_SCHEMA_DIR = _DOCS_DIR.parent / "schemas" / "input"
+_FILE_ORDER = {
+    "Time slices": ["time_slices"],
+    "Regions": ["regions"],
+    "Agents": ["agents", "agent_*"],
+    "Assets": ["assets"],
+    "Commodities": ["commodities", "commodity_costs", "demand", "demand_slicing"],
+    "Processes": ["processes", "process_*"],
+}
+
+
+def generate_markdown() -> str:
+    out = (
+        "# Input file format\n"
+        f"<!-- Automatically generated by {Path(__file__).name}. Do not edit manually. -->\n"
+        "<!-- markdownlint-disable MD013 -->\n"
+        "<!-- markdownlint-disable MD033 -->\n"
+    )
+
+    for title, patterns in _FILE_ORDER.items():
+        out += f"\n## {title}\n"
+
+        for pattern in patterns:
+            paths = map(str, _SCHEMA_DIR.glob(f"{pattern}.yaml"))
+            for path in map(Path, sorted(paths)):
+                out += process_file(path)
+
+    return out
+
+
+def process_file(path: Path) -> str:
+    out = f"\n### `{path.stem}.csv`\n\n"
+    with path.open() as f:
+        data = yaml.safe_load(f)
+
+    out += f"{add_full_stop(data['title'])}\n\n"
+
+    try:
+        table_str, notes_str = fields2table(data["fields"])
+        out += table_str
+    except KeyError:
+        print(f"MISSING VALUE IN {path}")
+        raise
+
+    desc = data.get("description", "")
+    if not desc and not notes_str:
+        return out
+
+    out += "\n#### Notes\n\n"
+
+    if desc:
+        out += f"{add_full_stop(desc)}\n\n"
+
+    if notes_str:
+        out += notes_str
+
+    return out
+
+
+def add_full_stop(s: str) -> str:
+    s = s.rstrip()
+    if s == "" or s.endswith("."):
+        return s
+    else:
+        return f"{s}."
+
+
+def fields2table(fields: list[dict[str, str]]) -> tuple[str, str]:
+    data = []
+    notes = []
+    for f in fields:
+        row = {"Field": f"`{f['name']}`", "Description": f["title"]}
+        data.append(row)
+
+        if desc := f.get("description", ""):
+            # MarkdownTable can't handle newlines, so replace with HTML equivalent
+            desc = desc.replace("\n\n", "<br /><br />").replace("\n", " ")
+            row = {"Field": f"`{f['name']}`", "Notes": desc}
+            notes.append(row)
+
+    data = [
+        {
+            "Field": f"`{f['name']}`",
+            "Description": f["title"],
+        }
+        for f in fields
+    ]
+
+    table_str = str(MarkdownTable.from_dicts(data))
+    notes_str = str(MarkdownTable.from_dicts(notes)) if notes else ""
+    return table_str, notes_str
+
+
+if __name__ == "__main__":
+    output_path = _DOCS_DIR / "input_format.md"
+    output_path.write_text(generate_markdown(), encoding="utf-8")

docs/generate_input_format_doc.py

@@ -1,5 +1,10 @@
 # User Guide
 
+## Running MUSE 2.0
+
+Once you have installed MUSE 2.0, you should be able to run it via the `muse2` command-line program.
+For details of the command-line interface, [see here](./command_line_help.md).
+
 ## Setting the log level
 
 MUSE uses the [`fern`] crate for logging. The default log level is `info`, though this can be

docs/user_guide.md

@@ -0,0 +1,24 @@
+title: Commodity demand portions for agents
+description: |
+  Portions of commodity demand for which agents are responsible.
+
+  If an entry is specified for one agent and commodity, there must be entries covering all milestone
+  years. For each agent listed in this file, the total portions for each region/commodity/year
+  combination must sum to one. In addition, there must be entries for every SVD and SED commodity
+  for all regions and milestone years.
+
+fields:
+  - name: agent_id
+    type: string
+    title: The agent to apply these values to
+  - name: commodity_id
+    type: string
+    title: The commodity for which the agent is responsible
+  - name: years
+    type: string
+    title: The year(s) to which this entry applies
+    description: One or more milestone years separated by semicolons or `all`
+  - name: commodity_portion
+    type: number
+    title: Portion of commodity demand
+    description: Value must be >0 and <=1. The portion applies only to the specified years.

schemas/input/agent_commodity_portions.yaml

@@ -0,0 +1,24 @@
+title: Agent cost limits
+description: |
+  Limits on expenditure for agents.
+
+  If cost limits are provided for an agent, they must be present for all years.
+
+fields:
+  - name: agent_id
+    type: string
+    title: The agent to apply these values to
+  - name: years
+    type: string
+    title: The year(s) to which this entry applies
+    description: One or more milestone years separated by semicolons or `all`
+  - name: capex_limit
+    type: number
+    title: Maximum capital cost the agent will pay
+    description: Must be >0. Optional (defaults to infinity).
+  - name: annual_cost_limit
+    type: number
+    title: Maximum annual operating cost
+    description: |
+      The maximum annual operating cost (fuel plus variable operating cost etc.) that the agent will
+      pay. Must be >0. Optional (defaults to infinity).