[benchmarker] Load and validate configuration

monitoring/benchmarker/README.md

@@ -0,0 +1,15 @@
+# benchmarker
+
+`benchmarker` is a tool for evaluating the performance, throughput, and stability of UTM systems under load.
+
+## Running benchmarker
+
+To execute `benchmarker`, run the following command from the root of the `monitoring` repo:
+
+```bash
+PYTHONPATH=. uv run python monitoring/benchmarker/benchmark.py --config file://monitoring/benchmarker/configurations/interuss/isas_uncontended.jsonnet
+```
+
+## Security
+
+Warning: benchmarker has the capability of running shell commands specified in the configuration (see `actions`).  Treat the configuration as code and review at an appropriate level of scrutiny before executing.

monitoring/benchmarker/benchmark.py

@@ -0,0 +1,80 @@
+#!env/bin/python3
+
+import argparse
+import json
+import os
+import sys
+
+from implicitdict import ImplicitDict
+from loguru import logger
+
+from monitoring.benchmarker.configurations.configuration import BenchmarkConfiguration
+from monitoring.benchmarker.validation import validate_config
+from monitoring.uss_qualifier.fileio import (
+    load_dict_with_references,
+)  # TODO: factor fileio out of uss_qualifier into monitorlib
+
+
+def parseArgs() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Execute benchmarker")
+
+    parser.add_argument(
+        "--config",
+        help="Configuration string indicating file reference (e.g. file://path/to/config.jsonnet)",
+        required=True,
+    )
+
+    parser.add_argument(
+        "--skip-validation",
+        action="store_true",
+        help="If specified, do not validate the format of the provided configuration.",
+    )
+
+    return parser.parse_args()
+
+
+def run_config(
+    config_name: str,
+    skip_validation: bool,
+) -> int:
+    config_src = load_dict_with_references(config_name)
+
+    if not skip_validation:
+        logger.info("Validating configuration...")
+        validation_errors = validate_config(config_src)
+        if validation_errors:
+            for e in validation_errors:
+                logger.error("[{}]: {}", e.json_path, e.message)
+            raise ValueError(
+                f"{len(validation_errors)} benchmark configuration validation errors indicated above.  Hint: resolve the clearest error first and then rerun validation."
+            )
+
+    whole_config = ImplicitDict.parse(config_src, BenchmarkConfiguration)
+
+    # Print the resolved configuration to console as JSON
+    print(json.dumps(whole_config, indent=2, sort_keys=True))
+
+    return os.EX_OK
+
+
+def main() -> int:
+    args = parseArgs()
+
+    logger.info(
+        f"========== Running benchmarker for configuration {args.config} =========="
+    )
+    exit_code = run_config(
+        args.config,
+        args.skip_validation,
+    )
+    if exit_code != os.EX_OK:
+        return exit_code
+    logger.info(
+        f"========== Completed benchmarker for configuration {args.config} =========="
+    )
+
+    return os.EX_OK
+
+
+if __name__ == "__main__":
+    sys.exit(main())

monitoring/benchmarker/validation.py

@@ -0,0 +1,26 @@
+from monitoring.benchmarker.configurations.configuration import (
+    BenchmarkConfiguration,
+)
+from monitoring.monitorlib.schema_validation import (
+    ValidationError,
+    validate_implicitdict_object,
+)
+from monitoring.uss_qualifier.validation import validate_resource_declarations
+
+
+def validate_config(config: dict) -> list[ValidationError]:
+    """Validate raw data intended to be used to create a BenchmarkConfiguration.
+
+    Args:
+        config: Raw nested dict object intended to be parsed into a BenchmarkConfiguration.
+
+    Returns: List of validation errors discovered.
+    """
+    result = validate_implicitdict_object(config, BenchmarkConfiguration)
+
+    if not result:
+        result.extend(
+            validate_resource_declarations(config, "$.resources.resource_declarations")
+        )
+
+    return result

monitoring/uss_qualifier/validation.py

@@ -11,6 +11,52 @@
 from monitoring.uss_qualifier.resources.resource import get_resource_types
 
 
+def validate_resource_declarations(
+    config: dict, base_path: str
+) -> list[ValidationError]:
+    """Validate resource declarations located at base_path in config against their concrete specification schemas.
+
+    Args:
+        config: Raw nested dictionary containing resource declarations at base_path.
+        base_path: JSON path string indicating where resource_declarations is located (e.g. '$.resources.resource_declarations').
+
+    Returns: List of validation errors discovered across the resource declarations.
+    """
+    result: list[ValidationError] = []
+    resource_declarations = config
+    for child in base_path.split(".")[1:]:
+        resource_declarations = resource_declarations.get(child, {})
+    for resource_id, declaration in resource_declarations.items():
+        declaration = ImplicitDict.parse(declaration, ResourceDeclaration)
+        path = base_path + "." + resource_id
+        try:
+            _, specification_type = get_resource_types(declaration)
+        except ValueError as e:
+            result.append(ValidationError(message=str(e), json_path=path))
+            continue
+        if specification_type is not None:
+            for e in validate_implicitdict_object(
+                declaration.specification, specification_type
+            ):
+                subpath = e.json_path
+                if subpath.startswith("$."):
+                    subpath = subpath[2:]
+                result.append(
+                    ValidationError(
+                        message=e.message,
+                        json_path=path + ".specification." + subpath,
+                    )
+                )
+        elif declaration.specification:
+            result.append(
+                ValidationError(
+                    message=f"Resource type {declaration.resource_type} does not accept a specification, but one was provided",
+                    json_path=path + ".specification",
+                )
+            )
+    return result
+
+
 def validate_config(config: dict) -> list[ValidationError]:
     """Validate raw data intended to be used to create a USSQualifierConfiguration.
 
@@ -22,37 +68,10 @@ def validate_config(config: dict) -> list[ValidationError]:
     result = validate_implicitdict_object(config, USSQualifierConfiguration)
 
     if not result:
-        base_path = "$.v1.test_run.resources.resource_declarations"
-        resource_declarations = config
-        for child in base_path.split(".")[1:]:
-            resource_declarations = resource_declarations.get(child, {})
-        for resource_id, declaration in resource_declarations.items():
-            declaration = ImplicitDict.parse(declaration, ResourceDeclaration)
-            path = base_path + "." + resource_id
-            try:
-                _, specification_type = get_resource_types(declaration)
-            except ValueError as e:
-                result.append(ValidationError(message=str(e), json_path=path))
-                continue
-            if specification_type is not None:
-                for e in validate_implicitdict_object(
-                    declaration.specification, specification_type
-                ):
-                    subpath = e.json_path
-                    if subpath.startswith("$."):
-                        subpath = subpath[2:]
-                    result.append(
-                        ValidationError(
-                            message=e.message,
-                            json_path=path + ".specification." + subpath,
-                        )
-                    )
-            elif declaration.specification:
-                result.append(
-                    ValidationError(
-                        message=f"Resource type {declaration.resource_type} does not accept a specification, but one was provided",
-                        json_path=path + ".specification",
-                    )
-                )
+        result.extend(
+            validate_resource_declarations(
+                config, "$.v1.test_run.resources.resource_declarations"
+            )
+        )
 
     return result