Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions docs/internals/requirements/requirements.rst
Original file line number Diff line number Diff line change
Expand Up @@ -944,6 +944,34 @@ Testing
* LOW
* HIGH

.. tool_req:: Verification report schema contract
:id: tool_req__docs_tvr_report_schema_contract
:tags: Tool Verification Reports
:implemented: PARTIAL
:version: 1
:parent_covered: NO
:satisfies: gd_req__verification_reporting, gd_req__verification_report_archiving
:source_code_link: scripts_bazel/verification_report_schema.json:1; scripts_bazel/verification_coverage_schema.json:1; scripts_bazel/verification_evidence_schema.json:1

Docs-as-Code shall provide machine-readable schema contracts for module verification
reporting and report archiving.

.. note:: Schema files are in place; generation and enforcement logic is not yet implemented.

.. tool_req:: Verification section schema checks
:id: tool_req__docs_tvr_section_schema_checks
:tags: Tool Verification Reports
:implemented: PARTIAL
:version: 1
:parent_covered: NO
:satisfies: gd_req__verification_checks_extended
:source_code_link: scripts_bazel/verification_coverage_schema.json:1; scripts_bazel/verification_evidence_schema.json:1

Docs-as-Code shall define machine-readable section-level schema contracts for
verification coverage and evidence to support extended verification checks.

.. note:: Schema files are in place; check execution and report generation are not yet implemented.

⚙️ Process / Other
###################

Expand Down
8 changes: 8 additions & 0 deletions scripts_bazel/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,11 @@ py_binary(
visibility = ["//visibility:public"],
deps = [],
)

py_binary(
name = "verification_schema_sync_check",
srcs = ["verification_schema_sync_check.py"],
main = "verification_schema_sync_check.py",
visibility = ["//visibility:public"],
deps = [],
)
9 changes: 9 additions & 0 deletions scripts_bazel/tests/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,12 @@ score_pytest(
] + all_requirements,
pytest_config = "//:pyproject.toml",
)

score_pytest(
name = "verification_schema_sync_check_test",
srcs = ["verification_schema_sync_check_test.py"],
deps = [
"//scripts_bazel:verification_schema_sync_check",
] + all_requirements,
pytest_config = "//:pyproject.toml",
)
122 changes: 122 additions & 0 deletions scripts_bazel/tests/verification_schema_sync_check_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
# *******************************************************************************
# Copyright (c) 2026 Contributors to the Eclipse Foundation
#
# See the NOTICE file(s) distributed with this work for additional
# information regarding copyright ownership.
#
# This program and the accompanying materials are made available under the
# terms of the Apache License Version 2.0 which is available at
# https://www.apache.org/licenses/LICENSE-2.0
#
# SPDX-License-Identifier: Apache-2.0
# *******************************************************************************

"""Tests for verification_schema_sync_check.py."""

from __future__ import annotations

import json
import subprocess
import sys
from pathlib import Path

_MY_PATH = Path(__file__).parent
_CHECK_SCRIPT = _MY_PATH.parent / "verification_schema_sync_check.py"


def _write_template(tmp_path: Path, description: str) -> Path:
path = tmp_path / "module_verification_report.rst"
path.write_text(
"""
Verification Report contains:

.. list-table:: Verification report section contract fields
:header-rows: 1
:widths: 1 2 5

* - section_index
- section_key
- section_description
* - 1
- verification_coverage
- DESCRIPTION_PLACEHOLDER
""".replace("DESCRIPTION_PLACEHOLDER", description),
encoding="utf-8",
)
return path


def _write_schema(tmp_path: Path, description: str | None) -> Path:
schema = {
"properties": {
"verification_coverage": {
"$ref": "./verification_coverage_schema.json",
}
}
}
if description is not None:
schema["properties"]["verification_coverage"]["description"] = description

path = tmp_path / "verification_report_schema.json"
path.write_text(json.dumps(schema), encoding="utf-8")
return path


def _run_check(template_path: Path, schema_path: Path) -> subprocess.CompletedProcess:
return subprocess.run(
[
sys.executable,
_CHECK_SCRIPT,
"--process-template",
str(template_path),
"--report-schema",
str(schema_path),
"--section-key",
"verification_coverage",
],
capture_output=True,
text=True,
)


def test_sync_check_passes_on_matching_description(tmp_path: Path) -> None:
description = (
"Coverage on requirements, architecture, and detailed design including "
"test and inspection results."
)
template_path = _write_template(tmp_path, description)
schema_path = _write_schema(tmp_path, description)

result = _run_check(template_path, schema_path)

assert result.returncode == 0
assert "Schema sync check passed." in result.stdout


def test_sync_check_fails_on_description_drift(tmp_path: Path) -> None:
template_path = _write_template(
tmp_path,
"Coverage on requirements, architecture, and detailed design including test and inspection results.",
)
schema_path = _write_schema(
tmp_path,
"Coverage-focused sections of the S-CORE process verification report.",
)

result = _run_check(template_path, schema_path)

assert result.returncode == 2
assert "Schema sync check failed:" in result.stdout


def test_sync_check_fails_on_missing_description(tmp_path: Path) -> None:
template_path = _write_template(
tmp_path,
"Coverage on requirements, architecture, and detailed design including test and inspection results.",
)
schema_path = _write_schema(tmp_path, None)

result = _run_check(template_path, schema_path)

assert result.returncode == 2
assert "schema description missing" in result.stderr
6 changes: 3 additions & 3 deletions scripts_bazel/traceability_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
CI gate → traceability_gate --metrics-json metrics.json [--min-* ...]

The gate never parses needs.json itself; it only reads the pre-computed
schema-v1 metrics file produced by the docs build.
schema-v2 metrics file produced by the docs build.
"""

from __future__ import annotations
Expand All @@ -34,7 +34,7 @@
from pathlib import Path
from typing import Any

_SUPPORTED_SCHEMA_VERSION = "1"
_SUPPORTED_SCHEMA_VERSION = "2"


def _print_type_summary(need_type: str, metrics: dict[str, Any]) -> None:
Expand Down Expand Up @@ -125,7 +125,7 @@ def _check_type_thresholds(
def main() -> int:
parser = argparse.ArgumentParser(
description=(
"Read a traceability metrics JSON (schema v1) and enforce coverage "
"Read a traceability metrics JSON (schema v2) and enforce coverage "
"thresholds. Exits 0 on pass, 2 on threshold failure, 1 on input error."
)
)
Expand Down
52 changes: 50 additions & 2 deletions scripts_bazel/traceability_metrics_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"properties": {
"schema_version": {
"type": "string",
"const": "1",
"const": "2",
"description": "Schema version. Bump when the shape changes incompatibly."
},
"generated_by": {
Expand All @@ -28,7 +28,13 @@
"$defs": {
"TypeMetrics": {
"type": "object",
"required": ["include_not_implemented", "include_external", "requirements", "tests"],
"required": [
"include_not_implemented",
"include_external",
"requirements",
"tests",
"process_requirements"
],
"additionalProperties": false,
"properties": {
"include_not_implemented": {
Expand All @@ -44,6 +50,9 @@
},
"tests": {
"$ref": "#/$defs/TestMetrics"
},
"process_requirements": {
"$ref": "#/$defs/ProcessRequirementMetrics"
}
}
},
Expand Down Expand Up @@ -157,6 +166,45 @@
}
}
},
"ProcessRequirementMetrics": {
"type": "object",
"required": [
"total",
"linked",
"linked_by_tool_requirements",
"linked_by_tool_requirements_pct",
"unlinked_ids"
],
"additionalProperties": false,
"properties": {
"total": {
"type": "integer",
"minimum": 0,
"description": "Total number of process requirements in scope."
},
"linked": {
"type": "integer",
"minimum": 0,
"description": "Process requirements linked by at least one tool requirement."
},
"linked_by_tool_requirements": {
"type": "integer",
"minimum": 0,
"description": "Process requirements linked by tool requirements."
},
"linked_by_tool_requirements_pct": {
"type": "number",
"minimum": 0,
"maximum": 100,
"description": "linked_by_tool_requirements / total * 100, or 100 when total == 0."
},
"unlinked_ids": {
"type": "array",
"items": { "type": "string" },
"description": "Sorted IDs of process requirements without a matching tool requirement."
}
}
},
"BrokenReference": {
"type": "object",
"required": ["testcase", "missing_need"],
Expand Down
Loading
Loading