From 846ced54ed58693c4d97da9dd7ec094034fb11b1 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Tue, 19 May 2026 14:56:13 +0000 Subject: [PATCH] fist draft --- docs/internals/requirements/requirements.rst | 28 ++ scripts_bazel/BUILD | 8 + scripts_bazel/tests/BUILD | 9 + .../verification_schema_sync_check_test.py | 122 ++++++++ scripts_bazel/traceability_gate.py | 6 +- .../traceability_metrics_schema.json | 52 +++- .../verification_coverage_schema.json | 163 +++++++++++ .../verification_evidence_schema.json | 275 ++++++++++++++++++ scripts_bazel/verification_report_schema.json | 97 ++++++ .../verification_schema_sync_check.py | 135 +++++++++ src/extensions/score_metamodel/__init__.py | 5 +- ...st_traceability_metrics_json_generation.py | 15 +- src/helper_lib/__init__.py | 12 +- uv.lock | 3 + 14 files changed, 921 insertions(+), 9 deletions(-) create mode 100644 scripts_bazel/tests/verification_schema_sync_check_test.py create mode 100644 scripts_bazel/verification_coverage_schema.json create mode 100644 scripts_bazel/verification_evidence_schema.json create mode 100644 scripts_bazel/verification_report_schema.json create mode 100644 scripts_bazel/verification_schema_sync_check.py create mode 100644 uv.lock diff --git a/docs/internals/requirements/requirements.rst b/docs/internals/requirements/requirements.rst index 382cb8f03..0e169dbd3 100644 --- a/docs/internals/requirements/requirements.rst +++ b/docs/internals/requirements/requirements.rst @@ -944,6 +944,34 @@ Testing * LOW * HIGH +.. tool_req:: Verification report schema contract + :id: tool_req__docs_tvr_report_schema_contract + :tags: Tool Verification Reports + :implemented: PARTIAL + :version: 1 + :parent_covered: NO + :satisfies: gd_req__verification_reporting, gd_req__verification_report_archiving + :source_code_link: scripts_bazel/verification_report_schema.json:1; scripts_bazel/verification_coverage_schema.json:1; scripts_bazel/verification_evidence_schema.json:1 + + Docs-as-Code shall provide machine-readable schema contracts for module verification + reporting and report archiving. + + .. note:: Schema files are in place; generation and enforcement logic is not yet implemented. + +.. tool_req:: Verification section schema checks + :id: tool_req__docs_tvr_section_schema_checks + :tags: Tool Verification Reports + :implemented: PARTIAL + :version: 1 + :parent_covered: NO + :satisfies: gd_req__verification_checks_extended + :source_code_link: scripts_bazel/verification_coverage_schema.json:1; scripts_bazel/verification_evidence_schema.json:1 + + Docs-as-Code shall define machine-readable section-level schema contracts for + verification coverage and evidence to support extended verification checks. + + .. note:: Schema files are in place; check execution and report generation are not yet implemented. + ⚙️ Process / Other ################### diff --git a/scripts_bazel/BUILD b/scripts_bazel/BUILD index e2d0402d2..a3c3168d7 100644 --- a/scripts_bazel/BUILD +++ b/scripts_bazel/BUILD @@ -45,3 +45,11 @@ py_binary( visibility = ["//visibility:public"], deps = [], ) + +py_binary( + name = "verification_schema_sync_check", + srcs = ["verification_schema_sync_check.py"], + main = "verification_schema_sync_check.py", + visibility = ["//visibility:public"], + deps = [], +) diff --git a/scripts_bazel/tests/BUILD b/scripts_bazel/tests/BUILD index e6fab5f1c..33c7dc42c 100644 --- a/scripts_bazel/tests/BUILD +++ b/scripts_bazel/tests/BUILD @@ -41,3 +41,12 @@ score_pytest( ] + all_requirements, pytest_config = "//:pyproject.toml", ) + +score_pytest( + name = "verification_schema_sync_check_test", + srcs = ["verification_schema_sync_check_test.py"], + deps = [ + "//scripts_bazel:verification_schema_sync_check", + ] + all_requirements, + pytest_config = "//:pyproject.toml", +) diff --git a/scripts_bazel/tests/verification_schema_sync_check_test.py b/scripts_bazel/tests/verification_schema_sync_check_test.py new file mode 100644 index 000000000..2dc2141eb --- /dev/null +++ b/scripts_bazel/tests/verification_schema_sync_check_test.py @@ -0,0 +1,122 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Tests for verification_schema_sync_check.py.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +_MY_PATH = Path(__file__).parent +_CHECK_SCRIPT = _MY_PATH.parent / "verification_schema_sync_check.py" + + +def _write_template(tmp_path: Path, description: str) -> Path: + path = tmp_path / "module_verification_report.rst" + path.write_text( + """ +Verification Report contains: + +.. list-table:: Verification report section contract fields + :header-rows: 1 + :widths: 1 2 5 + + * - section_index + - section_key + - section_description + * - 1 + - verification_coverage + - DESCRIPTION_PLACEHOLDER +""".replace("DESCRIPTION_PLACEHOLDER", description), + encoding="utf-8", + ) + return path + + +def _write_schema(tmp_path: Path, description: str | None) -> Path: + schema = { + "properties": { + "verification_coverage": { + "$ref": "./verification_coverage_schema.json", + } + } + } + if description is not None: + schema["properties"]["verification_coverage"]["description"] = description + + path = tmp_path / "verification_report_schema.json" + path.write_text(json.dumps(schema), encoding="utf-8") + return path + + +def _run_check(template_path: Path, schema_path: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [ + sys.executable, + _CHECK_SCRIPT, + "--process-template", + str(template_path), + "--report-schema", + str(schema_path), + "--section-key", + "verification_coverage", + ], + capture_output=True, + text=True, + ) + + +def test_sync_check_passes_on_matching_description(tmp_path: Path) -> None: + description = ( + "Coverage on requirements, architecture, and detailed design including " + "test and inspection results." + ) + template_path = _write_template(tmp_path, description) + schema_path = _write_schema(tmp_path, description) + + result = _run_check(template_path, schema_path) + + assert result.returncode == 0 + assert "Schema sync check passed." in result.stdout + + +def test_sync_check_fails_on_description_drift(tmp_path: Path) -> None: + template_path = _write_template( + tmp_path, + "Coverage on requirements, architecture, and detailed design including test and inspection results.", + ) + schema_path = _write_schema( + tmp_path, + "Coverage-focused sections of the S-CORE process verification report.", + ) + + result = _run_check(template_path, schema_path) + + assert result.returncode == 2 + assert "Schema sync check failed:" in result.stdout + + +def test_sync_check_fails_on_missing_description(tmp_path: Path) -> None: + template_path = _write_template( + tmp_path, + "Coverage on requirements, architecture, and detailed design including test and inspection results.", + ) + schema_path = _write_schema(tmp_path, None) + + result = _run_check(template_path, schema_path) + + assert result.returncode == 2 + assert "schema description missing" in result.stderr diff --git a/scripts_bazel/traceability_gate.py b/scripts_bazel/traceability_gate.py index 399f43f80..2cbddc414 100644 --- a/scripts_bazel/traceability_gate.py +++ b/scripts_bazel/traceability_gate.py @@ -22,7 +22,7 @@ CI gate → traceability_gate --metrics-json metrics.json [--min-* ...] The gate never parses needs.json itself; it only reads the pre-computed -schema-v1 metrics file produced by the docs build. +schema-v2 metrics file produced by the docs build. """ from __future__ import annotations @@ -34,7 +34,7 @@ from pathlib import Path from typing import Any -_SUPPORTED_SCHEMA_VERSION = "1" +_SUPPORTED_SCHEMA_VERSION = "2" def _print_type_summary(need_type: str, metrics: dict[str, Any]) -> None: @@ -125,7 +125,7 @@ def _check_type_thresholds( def main() -> int: parser = argparse.ArgumentParser( description=( - "Read a traceability metrics JSON (schema v1) and enforce coverage " + "Read a traceability metrics JSON (schema v2) and enforce coverage " "thresholds. Exits 0 on pass, 2 on threshold failure, 1 on input error." ) ) diff --git a/scripts_bazel/traceability_metrics_schema.json b/scripts_bazel/traceability_metrics_schema.json index c89bc7182..79408ec05 100644 --- a/scripts_bazel/traceability_metrics_schema.json +++ b/scripts_bazel/traceability_metrics_schema.json @@ -9,7 +9,7 @@ "properties": { "schema_version": { "type": "string", - "const": "1", + "const": "2", "description": "Schema version. Bump when the shape changes incompatibly." }, "generated_by": { @@ -28,7 +28,13 @@ "$defs": { "TypeMetrics": { "type": "object", - "required": ["include_not_implemented", "include_external", "requirements", "tests"], + "required": [ + "include_not_implemented", + "include_external", + "requirements", + "tests", + "process_requirements" + ], "additionalProperties": false, "properties": { "include_not_implemented": { @@ -44,6 +50,9 @@ }, "tests": { "$ref": "#/$defs/TestMetrics" + }, + "process_requirements": { + "$ref": "#/$defs/ProcessRequirementMetrics" } } }, @@ -157,6 +166,45 @@ } } }, + "ProcessRequirementMetrics": { + "type": "object", + "required": [ + "total", + "linked", + "linked_by_tool_requirements", + "linked_by_tool_requirements_pct", + "unlinked_ids" + ], + "additionalProperties": false, + "properties": { + "total": { + "type": "integer", + "minimum": 0, + "description": "Total number of process requirements in scope." + }, + "linked": { + "type": "integer", + "minimum": 0, + "description": "Process requirements linked by at least one tool requirement." + }, + "linked_by_tool_requirements": { + "type": "integer", + "minimum": 0, + "description": "Process requirements linked by tool requirements." + }, + "linked_by_tool_requirements_pct": { + "type": "number", + "minimum": 0, + "maximum": 100, + "description": "linked_by_tool_requirements / total * 100, or 100 when total == 0." + }, + "unlinked_ids": { + "type": "array", + "items": { "type": "string" }, + "description": "Sorted IDs of process requirements without a matching tool requirement." + } + } + }, "BrokenReference": { "type": "object", "required": ["testcase", "missing_need"], diff --git a/scripts_bazel/verification_coverage_schema.json b/scripts_bazel/verification_coverage_schema.json new file mode 100644 index 000000000..9a2de13ae --- /dev/null +++ b/scripts_bazel/verification_coverage_schema.json @@ -0,0 +1,163 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://eclipse-score.github.io/docs-as-code/verification-coverage-schema.json", + "title": "Verification Coverage Section", + "description": "Coverage-focused sections of the S-CORE process verification report.", + "type": "object", + "required": [ + "schema_version", + "on_requirements", + "on_architecture", + "on_detailed_design" + ], + "additionalProperties": false, + "properties": { + "schema_version": { + "type": "string", + "const": "1", + "description": "Verification coverage schema version." + }, + "on_requirements": { + "$ref": "#/$defs/RequirementCoverage" + }, + "on_architecture": { + "$ref": "#/$defs/DomainCoverage" + }, + "on_detailed_design": { + "$ref": "#/$defs/DomainCoverage" + }, + "methods": { + "type": "array", + "items": { + "type": "string", + "enum": ["test", "analysis", "inspection"] + }, + "description": "Verification methods used to produce the listed evidence." + } + }, + "$defs": { + "RequirementCoverage": { + "type": "object", + "required": [ + "qm_tests", + "asil_tests", + "inspection_checklists", + "external_aou_coverage", + "progress" + ], + "additionalProperties": false, + "properties": { + "qm_tests": { + "$ref": "#/$defs/TestCoverageList" + }, + "asil_tests": { + "$ref": "#/$defs/TestCoverageList" + }, + "inspection_checklists": { + "$ref": "#/$defs/InspectionCoverageList" + }, + "external_aou_coverage": { + "$ref": "#/$defs/TestCoverageList" + }, + "progress": { + "$ref": "#/$defs/ProgressSummary" + } + } + }, + "DomainCoverage": { + "type": "object", + "required": ["qm_tests", "asil_tests", "inspection_checklists", "progress"], + "additionalProperties": false, + "properties": { + "qm_tests": { + "$ref": "#/$defs/TestCoverageList" + }, + "asil_tests": { + "$ref": "#/$defs/TestCoverageList" + }, + "inspection_checklists": { + "$ref": "#/$defs/InspectionCoverageList" + }, + "progress": { + "$ref": "#/$defs/ProgressSummary" + } + } + }, + "TestCoverageList": { + "type": "array", + "items": { + "$ref": "#/$defs/TestCoverageEntry" + } + }, + "InspectionCoverageList": { + "type": "array", + "items": { + "$ref": "#/$defs/InspectionCoverageEntry" + } + }, + "TestCoverageEntry": { + "type": "object", + "required": ["item_id", "testcase_id", "status", "completeness_verdict"], + "additionalProperties": false, + "properties": { + "item_id": { + "type": "string" + }, + "testcase_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": ["passed", "failed", "not_run"] + }, + "completeness_verdict": { + "type": "string", + "enum": ["complete", "incomplete"] + } + } + }, + "InspectionCoverageEntry": { + "type": "object", + "required": ["item_id", "checklist_id", "verdict"], + "additionalProperties": false, + "properties": { + "item_id": { + "type": "string" + }, + "checklist_id": { + "type": "string" + }, + "verdict": { + "type": "string", + "enum": ["passed", "failed", "open"] + } + } + }, + "ProgressSummary": { + "type": "object", + "required": ["total", "passed", "failed", "not_run", "complete"], + "additionalProperties": false, + "properties": { + "total": { + "type": "integer", + "minimum": 0 + }, + "passed": { + "type": "integer", + "minimum": 0 + }, + "failed": { + "type": "integer", + "minimum": 0 + }, + "not_run": { + "type": "integer", + "minimum": 0 + }, + "complete": { + "type": "boolean" + } + } + } + } +} \ No newline at end of file diff --git a/scripts_bazel/verification_evidence_schema.json b/scripts_bazel/verification_evidence_schema.json new file mode 100644 index 000000000..80ae68a04 --- /dev/null +++ b/scripts_bazel/verification_evidence_schema.json @@ -0,0 +1,275 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://eclipse-score.github.io/docs-as-code/verification-evidence-schema.json", + "title": "Verification Evidence Sections", + "description": "Evidence-heavy sections of an ISO26262-style verification report.", + "type": "object", + "required": [ + "schema_version", + "dfa_report", + "safety_analysis_report", + "unit_verification", + "software_component_qualification", + "test_results", + "test_logs" + ], + "additionalProperties": false, + "properties": { + "schema_version": { + "type": "string", + "const": "1", + "description": "Verification evidence schema version." + }, + "dfa_report": { + "$ref": "#/$defs/AnalysisReport" + }, + "safety_analysis_report": { + "$ref": "#/$defs/AnalysisReport" + }, + "unit_verification": { + "$ref": "#/$defs/UnitVerification" + }, + "software_component_qualification": { + "$ref": "#/$defs/QualificationReport" + }, + "test_results": { + "$ref": "#/$defs/TestCaseStatusList" + }, + "test_logs": { + "$ref": "#/$defs/TestCaseLogList" + } + }, + "$defs": { + "AnalysisReport": { + "type": "object", + "required": ["items"], + "additionalProperties": false, + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/AnalysisItem" + } + } + } + }, + "AnalysisItem": { + "type": "object", + "required": ["item_id", "status", "open_mitigations"], + "additionalProperties": false, + "properties": { + "item_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": ["passed", "failed"] + }, + "open_mitigations": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "UnitVerification": { + "type": "object", + "required": ["structural_coverage", "static_code_analysis", "manual_code_inspection"], + "additionalProperties": false, + "properties": { + "structural_coverage": { + "$ref": "#/$defs/UnitCoverageList" + }, + "static_code_analysis": { + "$ref": "#/$defs/StaticAnalysisList" + }, + "manual_code_inspection": { + "$ref": "#/$defs/ManualInspectionList" + } + } + }, + "UnitCoverageList": { + "type": "array", + "items": { + "$ref": "#/$defs/UnitCoverageEntry" + } + }, + "UnitCoverageEntry": { + "type": "object", + "required": [ + "unit_id", + "safety_rating", + "c0_lines_covered", + "c0_lines_total", + "c0_pct", + "c1_branches_covered", + "c1_branches_total", + "c1_pct" + ], + "additionalProperties": false, + "properties": { + "unit_id": { + "type": "string" + }, + "safety_rating": { + "type": "string", + "enum": ["QM", "ASIL_A", "ASIL_B", "ASIL_C", "ASIL_D"] + }, + "c0_lines_covered": { + "type": "integer", + "minimum": 0 + }, + "c0_lines_total": { + "type": "integer", + "minimum": 0 + }, + "c0_pct": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "c1_branches_covered": { + "type": "integer", + "minimum": 0 + }, + "c1_branches_total": { + "type": "integer", + "minimum": 0 + }, + "c1_pct": { + "type": "number", + "minimum": 0, + "maximum": 100 + } + } + }, + "StaticAnalysisList": { + "type": "array", + "items": { + "$ref": "#/$defs/StaticAnalysisEntry" + } + }, + "StaticAnalysisEntry": { + "type": "object", + "required": ["unit_id", "safety_rating", "compiler_warnings", "coding_rule_violations"], + "additionalProperties": false, + "properties": { + "unit_id": { + "type": "string" + }, + "safety_rating": { + "type": "string", + "enum": ["QM", "ASIL_A", "ASIL_B", "ASIL_C", "ASIL_D"] + }, + "compiler_warnings": { + "type": "integer", + "minimum": 0 + }, + "coding_rule_violations": { + "type": "integer", + "minimum": 0 + } + } + }, + "ManualInspectionList": { + "type": "array", + "items": { + "$ref": "#/$defs/ManualInspectionEntry" + } + }, + "ManualInspectionEntry": { + "type": "object", + "required": ["component_id", "safety_rating", "checklist_id", "verdict"], + "additionalProperties": false, + "properties": { + "component_id": { + "type": "string" + }, + "safety_rating": { + "type": "string", + "enum": ["ASIL_A", "ASIL_B", "ASIL_C", "ASIL_D"] + }, + "checklist_id": { + "type": "string" + }, + "verdict": { + "type": "string", + "enum": ["passed", "failed", "open"] + } + } + }, + "QualificationReport": { + "type": "object", + "required": ["components"], + "additionalProperties": false, + "properties": { + "components": { + "type": "array", + "items": { + "$ref": "#/$defs/QualificationEntry" + } + } + } + }, + "QualificationEntry": { + "type": "object", + "required": ["component_id", "verification_results"], + "additionalProperties": false, + "properties": { + "component_id": { + "type": "string" + }, + "verification_results": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "TestCaseStatusList": { + "type": "array", + "items": { + "$ref": "#/$defs/TestCaseStatus" + } + }, + "TestCaseStatus": { + "type": "object", + "required": ["testcase_id", "status"], + "additionalProperties": false, + "properties": { + "testcase_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": ["passed", "failed", "not_run"] + } + } + }, + "TestCaseLogList": { + "type": "array", + "items": { + "$ref": "#/$defs/TestCaseLog" + } + }, + "TestCaseLog": { + "type": "object", + "required": ["testcase_id", "status", "log_ref"], + "additionalProperties": false, + "properties": { + "testcase_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": ["passed", "failed", "not_run"] + }, + "log_ref": { + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/scripts_bazel/verification_report_schema.json b/scripts_bazel/verification_report_schema.json new file mode 100644 index 000000000..c1c312b3d --- /dev/null +++ b/scripts_bazel/verification_report_schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://eclipse-score.github.io/docs-as-code/verification-report-schema.json", + "title": "Verification Report", + "description": "Top-level schema for generated ISO26262-style module verification reports.", + "type": "object", + "required": [ + "schema_version", + "metadata", + "verification_coverage", + "dfa_report", + "safety_analysis_report", + "unit_verification", + "software_component_qualification", + "test_results", + "test_logs", + "traceability_metrics" + ], + "additionalProperties": false, + "properties": { + "schema_version": { + "type": "string", + "const": "1", + "description": "Verification report schema version." + }, + "metadata": { + "$ref": "#/$defs/ReportMetadata" + }, + "verification_coverage": { + "description": "Coverage on requirements, architecture, and detailed design including test and inspection results.", + "$ref": "./verification_coverage_schema.json" + }, + "dfa_report": { + "$ref": "./verification_evidence_schema.json#/$defs/AnalysisReport" + }, + "safety_analysis_report": { + "$ref": "./verification_evidence_schema.json#/$defs/AnalysisReport" + }, + "unit_verification": { + "$ref": "./verification_evidence_schema.json#/$defs/UnitVerification" + }, + "software_component_qualification": { + "$ref": "./verification_evidence_schema.json#/$defs/QualificationReport" + }, + "test_results": { + "$ref": "./verification_evidence_schema.json#/$defs/TestCaseStatusList" + }, + "test_logs": { + "$ref": "./verification_evidence_schema.json#/$defs/TestCaseLogList" + }, + "traceability_metrics": { + "$ref": "./traceability_metrics_schema.json" + } + }, + "$defs": { + "ReportMetadata": { + "type": "object", + "required": [ + "module_name", + "module_version_tag", + "report_id", + "generated_at", + "source_commit", + "asil_scope" + ], + "additionalProperties": false, + "properties": { + "module_name": { + "type": "string" + }, + "module_version_tag": { + "type": "string", + "description": "Module version tag covered by this report." + }, + "report_id": { + "type": "string" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "source_commit": { + "type": "string" + }, + "asil_scope": { + "type": "array", + "items": { + "type": "string", + "enum": ["QM", "ASIL_A", "ASIL_B", "ASIL_C", "ASIL_D"] + }, + "minItems": 1, + "uniqueItems": true + } + } + } + } +} \ No newline at end of file diff --git a/scripts_bazel/verification_schema_sync_check.py b/scripts_bazel/verification_schema_sync_check.py new file mode 100644 index 000000000..eab702ab8 --- /dev/null +++ b/scripts_bazel/verification_schema_sync_check.py @@ -0,0 +1,135 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Gate to detect drift between process template section contracts and schema text. + +Vertical slice scope (v1): validates the `verification_coverage` section only. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path + + +def _resolve_path(path_str: str) -> Path: + path = Path(path_str) + workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "").strip() + if not path.is_absolute() and workspace_dir: + return Path(workspace_dir) / path + return path + + +def _extract_section_description(template_text: str, section_key: str) -> str: + # list-table rows are represented as: + # * - + # - + # - + pattern = re.compile( + r"\*\s*-\s*\d+\s*\n\s*-\s*" + + re.escape(section_key) + + r"\s*\n\s*-\s*(.+)", + re.MULTILINE, + ) + match = pattern.search(template_text) + if not match: + raise ValueError( + f"Could not find section_key '{section_key}' in template section contract table" + ) + return match.group(1).strip() + + +def _load_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Detect drift between process template section contract fields and " + "verification schema descriptions." + ) + ) + parser.add_argument( + "--process-template", + required=True, + help="Path to process_description module verification template rst file.", + ) + parser.add_argument( + "--report-schema", + required=True, + help="Path to verification_report_schema.json.", + ) + parser.add_argument( + "--section-key", + default="verification_coverage", + help="Section key to validate (default: verification_coverage).", + ) + args = parser.parse_args() + + process_template = _resolve_path(args.process_template) + report_schema = _resolve_path(args.report_schema) + + if not process_template.exists(): + print(f"Error: process template not found: {process_template}", file=sys.stderr) + return 1 + if not report_schema.exists(): + print(f"Error: report schema not found: {report_schema}", file=sys.stderr) + return 1 + + template_text = process_template.read_text(encoding="utf-8") + schema = _load_json(report_schema) + + try: + expected_description = _extract_section_description( + template_text, args.section_key + ) + except ValueError as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + + actual_description = ( + schema.get("properties", {}) + .get(args.section_key, {}) + .get("description", "") + .strip() + ) + + if not actual_description: + print( + "Error: schema description missing for " + f"properties.{args.section_key}.description", + file=sys.stderr, + ) + return 2 + + if expected_description != actual_description: + print("Schema sync check failed:") + print(f" section_key: {args.section_key}") + print(f" template: {process_template}") + print(f" schema: {report_schema}") + print(f" expected: {expected_description}") + print(f" actual: {actual_description}") + return 2 + + print("Schema sync check passed.") + print(f" section_key: {args.section_key}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index 557e7b974..baa9163db 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -100,7 +100,7 @@ def graph_check(func: graph_check_function): def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: - """Write a schema-v1 metrics.json alongside needs.json in the build output. + """Write a schema-v2 metrics.json alongside needs.json in the build output. This is the single source of truth for traceability metrics. It runs inside the Sphinx build so it has access to all needs (local + external) @@ -133,10 +133,11 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: "include_external": type_summary["include_external"], "requirements": type_summary["requirements"], "tests": type_summary["tests"], + "process_requirements": type_summary["process_requirements"], } output: dict[str, Any] = { - "schema_version": "1", + "schema_version": "2", "generated_by": "sphinx_build", "metrics_by_type": metrics_by_type, } diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py index 764659874..163c3e50b 100644 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py +++ b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py @@ -38,6 +38,11 @@ def get_needs_view(self) -> dict[str, dict[str, object]]: "testlink": "", "is_external": False, }, + "LOCAL_PROC_REQ": { + "id": "LOCAL_PROC_REQ", + "type": "gd_req", + "is_external": False, + }, "EXT_REQ": { "id": "EXT_REQ", "type": "tool_req", @@ -73,10 +78,17 @@ def test_write_metrics_json_defaults_to_local_only( payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) metrics = payload["metrics_by_type"]["tool_req"] - assert payload["schema_version"] == "1" + assert payload["schema_version"] == "2" assert metrics["include_not_implemented"] is True assert metrics["include_external"] is False assert metrics["requirements"]["total"] == 1 + assert metrics["process_requirements"] == { + "total": 1, + "linked": 0, + "linked_by_tool_requirements": 0, + "linked_by_tool_requirements_pct": 0.0, + "unlinked_ids": ["LOCAL_PROC_REQ"], + } def test_write_metrics_json_can_include_external( @@ -94,3 +106,4 @@ def test_write_metrics_json_can_include_external( assert metrics["include_external"] is True assert metrics["requirements"]["total"] == 2 + assert metrics["process_requirements"]["total"] == 1 diff --git a/src/helper_lib/__init__.py b/src/helper_lib/__init__.py index a72fffb0b..9a011c3fc 100644 --- a/src/helper_lib/__init__.py +++ b/src/helper_lib/__init__.py @@ -17,7 +17,17 @@ from pathlib import Path from typing import Any -from python.runfiles import Runfiles +try: + from python.runfiles import Runfiles +except ModuleNotFoundError: + class Runfiles: # type: ignore[no-redef] + @staticmethod + def Create() -> "Runfiles": + return Runfiles() + + def EnvVars(self) -> dict[str, str]: + return {"RUNFILES_DIR": os.environ.get("RUNFILES_DIR", "")} + from sphinx.config import Config from sphinx_needs.logging import get_logger diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000..7518fc90b --- /dev/null +++ b/uv.lock @@ -0,0 +1,3 @@ +version = 1 +revision = 3 +requires-python = ">=3.12"