From 1135617fda95774894f88ec9c1a3b38465824356 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Tue, 31 Mar 2026 00:13:55 +0200 Subject: [PATCH] feat(cli): expose structured violation data in policy develop eval output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automatically populate structured_violations in the eval output when the policy returns structured finding data. Uses protojson to serialize the proto violations directly — no custom types needed. The violations field (string messages) remains populated for backward compatibility. Closes chainloop-dev/chainloop#2968 Signed-off-by: Miguel Martinez Signed-off-by: Miguel Martinez Trivino --- app/cli/internal/policydevel/eval.go | 28 +++++++++-- app/cli/internal/policydevel/eval_test.go | 46 ++++++++++++++++++- .../testdata/sbom-structured-vuln-policy.yaml | 24 ++++++++++ app/cli/pkg/action/policy_develop_eval.go | 2 +- 4 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 app/cli/internal/policydevel/testdata/sbom-structured-vuln-policy.yaml diff --git a/app/cli/internal/policydevel/eval.go b/app/cli/internal/policydevel/eval.go index 998845897..9a36bb4bc 100644 --- a/app/cli/internal/policydevel/eval.go +++ b/app/cli/internal/policydevel/eval.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The Chainloop Authors. +// Copyright 2024-2026 The Chainloop Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ import ( "github.com/chainloop-dev/chainloop/pkg/policies" "github.com/rs/zerolog" "google.golang.org/grpc" + "google.golang.org/protobuf/encoding/protojson" v12 "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/api/attestation/v1" "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials" @@ -47,9 +48,10 @@ type EvalOptions struct { } type EvalResult struct { - Violations []string `json:"violations"` - SkipReasons []string `json:"skip_reasons"` - Skipped bool `json:"skipped"` + Violations []string `json:"violations"` + StructuredViolations []json.RawMessage `json:"structured_violations,omitempty"` + SkipReasons []string `json:"skip_reasons"` + Skipped bool `json:"skipped"` } type EvalSummary struct { @@ -136,9 +138,25 @@ func verifyMaterial(pol *v1.Policies, material *v12.Attestation_Material, materi }, } - // Collect violation messages + hasStructuredFindings := false for _, v := range policyEv.Violations { summary.Result.Violations = append(summary.Result.Violations, v.Message) + if v.GetFinding() != nil { + hasStructuredFindings = true + } + } + + // Include structured violations when any violation has finding data + if hasStructuredFindings { + marshaler := protojson.MarshalOptions{UseProtoNames: true} + summary.Result.StructuredViolations = make([]json.RawMessage, 0, len(policyEv.Violations)) + for _, v := range policyEv.Violations { + b, err := marshaler.Marshal(v) + if err != nil { + return nil, fmt.Errorf("marshaling structured violation: %w", err) + } + summary.Result.StructuredViolations = append(summary.Result.StructuredViolations, b) + } } // Include raw debug info if requested diff --git a/app/cli/internal/policydevel/eval_test.go b/app/cli/internal/policydevel/eval_test.go index cc45e2fa1..828ca24b8 100644 --- a/app/cli/internal/policydevel/eval_test.go +++ b/app/cli/internal/policydevel/eval_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 The Chainloop Authors. +// Copyright 2025-2026 The Chainloop Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ package policydevel import ( + "encoding/json" "os" "path/filepath" "testing" @@ -148,6 +149,49 @@ func TestEvaluateSimplifiedPolicies(t *testing.T) { assert.Contains(t, result.Result.Violations[0], "at least 2 components") }) + t.Run("structured violations populated for policies with finding_type", func(t *testing.T) { + opts := &EvalOptions{ + PolicyPath: "testdata/sbom-structured-vuln-policy.yaml", + MaterialPath: sbomPath, + } + + result, err := Evaluate(opts, logger) + require.NoError(t, err) + require.NotNil(t, result) + assert.False(t, result.Result.Skipped) + + // Both fields populated: violations (messages) and structured_violations (proto JSON) + require.Len(t, result.Result.Violations, 1) + assert.Contains(t, result.Result.Violations[0], "Vulnerability found in test-component@1.0.0") + + require.Len(t, result.Result.StructuredViolations, 1) + var sv map[string]any + require.NoError(t, json.Unmarshal(result.Result.StructuredViolations[0], &sv)) + assert.Contains(t, sv["message"], "Vulnerability found in test-component@1.0.0") + + vuln, ok := sv["vulnerability"].(map[string]any) + require.True(t, ok, "expected vulnerability finding in structured violation") + assert.Equal(t, "CVE-2024-1234", vuln["external_id"]) + assert.Equal(t, "pkg:generic/test-component@1.0.0", vuln["package_purl"]) + assert.Equal(t, "HIGH", vuln["severity"]) + assert.InDelta(t, 7.5, vuln["cvss_v3_score"], 0.001) + }) + + t.Run("no structured violations for plain string policies", func(t *testing.T) { + opts := &EvalOptions{ + PolicyPath: "testdata/sbom-min-components-policy.yaml", + MaterialPath: sbomPath, + } + + result, err := Evaluate(opts, logger) + require.NoError(t, err) + require.NotNil(t, result) + require.Len(t, result.Result.Violations, 1) + assert.Contains(t, result.Result.Violations[0], "at least 2 components") + // No structured_violations when policy returns plain strings + assert.Empty(t, result.Result.StructuredViolations) + }) + t.Run("sbom metadata component policy", func(t *testing.T) { opts := &EvalOptions{ PolicyPath: "testdata/sbom-metadata-component-policy.yaml", diff --git a/app/cli/internal/policydevel/testdata/sbom-structured-vuln-policy.yaml b/app/cli/internal/policydevel/testdata/sbom-structured-vuln-policy.yaml new file mode 100644 index 000000000..27f6529e7 --- /dev/null +++ b/app/cli/internal/policydevel/testdata/sbom-structured-vuln-policy.yaml @@ -0,0 +1,24 @@ +apiVersion: chainloop.dev/v1 +kind: Policy +metadata: + name: sbom-structured-vuln + description: Policy that returns structured vulnerability violations + finding_type: VULNERABILITY +spec: + policies: + - kind: SBOM_CYCLONEDX_JSON + embedded: | + package main + + import rego.v1 + + violations contains v if { + comp := input.components[_] + v := { + "message": sprintf("Vulnerability found in %s@%s", [comp.name, comp.version]), + "external_id": "CVE-2024-1234", + "package_purl": sprintf("pkg:generic/%s@%s", [comp.name, comp.version]), + "severity": "HIGH", + "cvss_v3_score": 7.5, + } + } diff --git a/app/cli/pkg/action/policy_develop_eval.go b/app/cli/pkg/action/policy_develop_eval.go index e159ceb2f..91d1f5fcd 100644 --- a/app/cli/pkg/action/policy_develop_eval.go +++ b/app/cli/pkg/action/policy_develop_eval.go @@ -1,5 +1,5 @@ // -// Copyright 2024-2025 The Chainloop Authors. +// Copyright 2024-2026 The Chainloop Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License.