Merge pull request #618 from EnergySystemsModellingLab/jinja

Change input format doc script to use jinja templates

Author: alexdewar

.pre-commit-config.yaml

@@ -26,6 +26,14 @@ repos:
     rev: v1.0
     hooks:
       - id: clippy
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.6
+    hooks:
+      - id: ruff
+        types_or: [python]
+        args: [--fix]
+      - id: ruff-format
+        types_or: [python]
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:

.vscode/extensions.json

@@ -4,6 +4,7 @@
         "vadimcn.vscode-lldb",
         "editorconfig.editorconfig",
         "esbenp.prettier-vscode",
-        "davidanson.vscode-markdownlint"
+        "davidanson.vscode-markdownlint",
+        "samuelcolvin.jinjahtml"
     ]
 }

doc-requirements.txt

@@ -1 +1,3 @@
+pyyaml
 table2md
+jinja2

docs/generate_input_format_doc.py

@@ -2,9 +2,12 @@
 #
 # A script to generate markdown documentation from table schemas.
 
+from typing import Iterable
 from table2md import MarkdownTable
 import yaml
 from pathlib import Path
+from dataclasses import dataclass
+from jinja2 import Environment, FileSystemLoader
 
 _DOCS_DIR = Path(__file__).parent
 _SCHEMA_DIR = _DOCS_DIR.parent / "schemas" / "input"
@@ -18,52 +21,57 @@
 }
 
 
+@dataclass
+class Notes:
+    description: str | None
+    table: str | None
+
+
+@dataclass
+class File:
+    name: str
+    description: str
+    table: str
+    notes: Notes | None
+
+
+@dataclass
+class Section:
+    title: str
+    files: Iterable[File]
+
+
 def generate_markdown() -> str:
-    out = (
-        "# Input file format\n"
-        f"<!-- Automatically generated by {Path(__file__).name}. Do not edit manually. -->\n"
-        "<!-- markdownlint-disable MD013 -->\n"
-        "<!-- markdownlint-disable MD033 -->\n"
-    )
+    """Generate markdown from Jinja template using metadata in schemas."""
+    env = Environment(loader=FileSystemLoader(_DOCS_DIR))
+    template = env.get_template("input_format.md.jinja")
+    return template.render(script_name=Path(__file__).name, sections=load_sections())
 
-    for title, patterns in _FILE_ORDER.items():
-        out += f"\n## {title}\n"
 
+def load_sections() -> Iterable[Section]:
+    for title, patterns in _FILE_ORDER.items():
         for pattern in patterns:
             paths = map(str, _SCHEMA_DIR.glob(f"{pattern}.yaml"))
-            for path in map(Path, sorted(paths)):
-                out += process_file(path)
-
-    return out
+            files = (load_file(Path(path)) for path in sorted(paths))
+            yield Section(title, files)
 
 
-def process_file(path: Path) -> str:
-    out = f"\n### `{path.stem}.csv`\n\n"
+def load_file(path: Path) -> File:
     with path.open() as f:
         data = yaml.safe_load(f)
 
-    out += f"{add_full_stop(data['title'])}\n\n"
-
     try:
-        table_str, notes_str = fields2table(data["fields"])
-        out += table_str
+        table, notes_table = fields2table(data["fields"])
     except KeyError:
         print(f"MISSING VALUE IN {path}")
         raise
 
-    desc = data.get("description", "")
-    if not desc and not notes_str:
-        return out
-
-    out += "\n#### Notes\n\n"
-
-    if desc:
-        out += f"{add_full_stop(desc)}\n\n"
-
-    if notes_str:
-        out += notes_str
-
-    return out
+    name = f"{path.stem}.csv"
+    title = add_full_stop(data["title"])
+    if desc := data.get("description", None):
+        desc = add_full_stop(desc)
+    notes = Notes(desc, notes_table) if desc or notes_table else None
+    return File(name, title, table, notes)
 
 
 def add_full_stop(s: str) -> str:
@@ -74,7 +82,7 @@ def add_full_stop(s: str) -> str:
         return f"{s}."
 
 
-def fields2table(fields: list[dict[str, str]]) -> tuple[str, str]:
+def fields2table(fields: list[dict[str, str]]) -> tuple[str, str | None]:
     data = []
     notes = []
     for f in fields:
@@ -95,9 +103,9 @@ def fields2table(fields: list[dict[str, str]]) -> tuple[str, str]:
         for f in fields
     ]
 
-    table_str = str(MarkdownTable.from_dicts(data))
-    notes_str = str(MarkdownTable.from_dicts(notes)) if notes else ""
-    return table_str, notes_str
+    table = str(MarkdownTable.from_dicts(data))
+    notes_table = str(MarkdownTable.from_dicts(notes)) if notes else None
+    return table, notes_table
 
 
 if __name__ == "__main__":

docs/input_format.md.jinja

@@ -0,0 +1,35 @@
+# Input file format
+
+<!-- Automatically generated by {{ script_name }}. Do not edit manually. -->
+<!-- markdownlint-disable MD013 -->
+<!-- markdownlint-disable MD024 -->
+<!-- markdownlint-disable MD033 -->
+
+This file contains information about the input file format for MUSE 2.0.
+
+{# "Sections" are broad categories representing multiple files (e.g. "commodities") -#}
+{% for section in sections -%}
+## {{ section.title }}
+
+{# Documentation for individual CSV files -#}
+{% for file in section.files -%}
+### `{{ file.name }}`
+
+{# Short description of file -#}
+{{ file.description }}
+
+{# Table of fields with a short description -#}
+{{ file.table }}
+
+{#- Optional notes section. This can include a longer description of the file, including validation
+    checks that are performed, as well as a table containing extra information about CSV fields
+    (such as a requirement that the value is >=0 etc.). -#}
+{%- if file.notes %}
+#### Notes
+{% if file.notes.description %}
+{{ file.notes.description }}{% endif %}
+{% if file.notes.table %}
+{{ file.notes.table }}{% endif -%}
+{%- endif %}
+{% endfor -%}
+{% endfor %}