Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/aw/safe-outputs-runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,9 @@ Fields that influence permission computation (`add-comment.discussions`, `create
- Increase this limit for long-running branches that touch many files
- `group-reports:` - Group workflow failure reports as sub-issues (boolean, default: `false`)
- When `true`, creates a parent `[aw] Failed runs` issue that tracks all workflow failures as sub-issues; useful for larger repositories
- `report-failure-as-issue:` - Control whether workflow failures are reported as GitHub issues (boolean, default: `true`)
- `report-failure-as-issue:` - Control whether workflow failures are reported as GitHub issues (boolean, expression, or category array; default: `true`)
- When `false`, suppresses automatic failure issue creation for this workflow
- Supports templatable boolean expressions, e.g. `report-failure-as-issue: ${{ inputs.report-failure-as-issue }}`
- Use to silence noisy failure reports for workflows where failures are expected or handled externally
- `failure-issue-repo:` - Repository to create failure tracking issues in (string, format: `"owner/repo"`)
- Defaults to the current repository when not specified
Expand Down
3 changes: 2 additions & 1 deletion actions/setup/js/handle_agent_failure.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const { AWF_INFRA_LINE_RE } = require("./log_parser_shared.cjs");
const { resolveFirewallAuditLogPath, resolveAICreditsFailureState, parseMaxAICreditsFromAuditLog, parseAICreditsErrorInfoFromAuditLog, parseUnknownModelAICreditsFromAuditLog } = require("./ai_credits_context.cjs");
const { formatAICCredits } = require("./daily_aic_workflow_helpers.cjs");
const { formatAIC } = require("./model_costs.cjs");
const { parseBoolTemplatable } = require("./templatable.cjs");
const { parseTokenUsageJsonl, generateTokenUsageSummary } = require("./parse_mcp_gateway_log.cjs");
const { readDedupedTokenUsage, TOKEN_USAGE_PATHS } = require("./parse_token_usage.cjs");
const { extractShellCommandFromToolData } = require("./tool_call_details.cjs");
Expand Down Expand Up @@ -2692,7 +2693,7 @@ async function main() {
const unknownModelAICreditsFromAudit = parseUnknownModelAICreditsFromAuditLog();
const unknownModelAICredits = unknownModelAICreditsFromAudit || (unknownModelAICreditsFromOutput && agentConclusion === "failure");
const pushRepoMemoryResult = process.env.GH_AW_PUSH_REPO_MEMORY_RESULT || "";
const reportFailureAsIssue = process.env.GH_AW_FAILURE_REPORT_AS_ISSUE !== "false"; // Default to true
const reportFailureAsIssue = parseBoolTemplatable(process.env.GH_AW_FAILURE_REPORT_AS_ISSUE, true);
// Parse included categories filter for report-failure-as-issue (optional JSON array of category strings)
const failureCategoriesFilterRaw = process.env.GH_AW_FAILURE_CATEGORIES_FILTER || "";
let failureCategoriesFilter = null;
Expand Down
28 changes: 28 additions & 0 deletions actions/setup/js/handle_agent_failure.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,34 @@ describe("handle_agent_failure", () => {
expect(createIssueMock).not.toHaveBeenCalled();
});

it("skips failure issue creation when the runtime report flag resolves to false", async () => {
const searchMock = vi.fn();
const createCommentMock = vi.fn();
const createIssueMock = vi.fn();
process.env.GH_AW_FAILURE_REPORT_AS_ISSUE = " False ";

global.github = {
rest: {
search: {
issuesAndPullRequests: searchMock,
},
issues: {
create: createIssueMock,
createComment: createCommentMock,
},
pulls: { get: vi.fn() },
},
graphql: vi.fn(),
};

await main();

expect(searchMock).not.toHaveBeenCalled();
expect(createCommentMock).not.toHaveBeenCalled();
expect(createIssueMock).not.toHaveBeenCalled();
expect(global.core.info).toHaveBeenCalledWith("Failure issue reporting is disabled (report-failure-as-issue: false), skipping failure issue creation");
});

it("adds a comment when existing issue metadata contains commas in free-form values", async () => {
const createCommentMock = vi.fn(async () => ({ data: { id: 1001 } }));
const createIssueMock = vi.fn();
Expand Down
4 changes: 3 additions & 1 deletion actions/setup/js/templatable.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
* - boolean `false` → `false`
* - string `"true"` → `true`
* - string `"false"` → `false`
* - string variants that normalize to `"false"` after trim/lowercase
* (for example `" False "`) → `false`
* - any other string (e.g. a resolved GitHub Actions expression value
* that was not "false") → `true`
*
Expand All @@ -32,7 +34,7 @@
*/
function parseBoolTemplatable(value, defaultValue = true) {
if (value === undefined || value === null) return defaultValue;
return String(value) !== "false";
return String(value).trim().toLowerCase() !== "false";
}

/**
Expand Down
5 changes: 5 additions & 0 deletions actions/setup/js/templatable.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ describe("templatable.cjs", () => {
expect(parseBoolTemplatable("false")).toBe(false);
});

it("handles normalized false string variants", () => {
expect(parseBoolTemplatable("False")).toBe(false);
expect(parseBoolTemplatable(" false ")).toBe(false);
});

it("treats a resolved expression value other than false as truthy", () => {
// GitHub Actions expressions that resolve to something other than "false"
// (e.g. "yes", "1", an empty object representation) should be truthy.
Expand Down
5 changes: 2 additions & 3 deletions pkg/parser/schemas/main_workflow_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -10303,8 +10303,7 @@
"report-failure-as-issue": {
"oneOf": [
{
"type": "boolean",
"description": "When false, disables creating failure tracking issues when workflows fail. When true, all failures trigger issues. Defaults to true."
"$ref": "#/$defs/templatable_boolean"
},
{
"type": "array",
Expand All @@ -10318,7 +10317,7 @@
}
],
"default": true,
"examples": [false, true, ["agent_failure", "missing_safe_outputs"], ["!inference_access_error", "!ai_credits_rate_limit_error"]]
"examples": [false, true, "${{ inputs.report-failure-as-issue }}", ["agent_failure", "missing_safe_outputs"], ["!inference_access_error", "!ai_credits_rate_limit_error"]]
},
"failure-issue-repo": {
"type": "string",
Expand Down
11 changes: 11 additions & 0 deletions pkg/workflow/compiler_safe_outputs_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3398,6 +3398,7 @@ func TestReportFailureAsIssueWithCategoriesFilter(t *testing.T) {
name string
reportValue any
expectBool *bool
expectString string
expectCategories []string
expectExcludedCategories []string
}{
Expand All @@ -3411,6 +3412,11 @@ func TestReportFailureAsIssueWithCategoriesFilter(t *testing.T) {
reportValue: false,
expectBool: boolPtr(false),
},
{
name: "templatable expression",
reportValue: "${{ inputs.report-failure-as-issue }}",
expectString: "${{ inputs.report-failure-as-issue }}",
},
{
name: "array of categories",
reportValue: []any{"agent_failure", "missing_safe_outputs"},
Expand Down Expand Up @@ -3454,6 +3460,11 @@ func TestReportFailureAsIssueWithCategoriesFilter(t *testing.T) {
require.True(t, ok, "ReportFailureAsIssue should be bool")
assert.Equal(t, *tt.expectBool, reportBool, "Boolean value should match")
}
if tt.expectString != "" {
reportString, ok := config.ReportFailureAsIssue.(string)
require.True(t, ok, "ReportFailureAsIssue should be string")
assert.Equal(t, tt.expectString, reportString, "String value should match")
}

if len(tt.expectCategories) > 0 {
assert.Equal(t, tt.expectCategories, config.ReportFailureAsIssueCategories, "Categories should match")
Expand Down
2 changes: 1 addition & 1 deletion pkg/workflow/compiler_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,7 @@ type SafeOutputsConfig struct {
Mentions *MentionsConfig `yaml:"mentions,omitempty"` // Configuration for @mention filtering in safe outputs
Footer *bool `yaml:"footer,omitempty"` // Global footer control - when false, omits visible footer from all safe outputs (XML markers still included)
GroupReports bool `yaml:"group-reports,omitempty"` // If true, create parent "Failed runs" issue for agent failures (default: false)
ReportFailureAsIssue any `yaml:"report-failure-as-issue,omitempty"` // Controls failure issue creation: bool (true/false) or []interface{} (parsed from YAML array, converted to []string in ReportFailureAsIssueCategories). Default: true
ReportFailureAsIssue any `yaml:"report-failure-as-issue,omitempty"` // Controls failure issue creation: bool, templatable expression string, or []interface{} categories (parsed to ReportFailureAsIssueCategories/ExcludedCategories). Default: true
ReportFailureAsIssueCategories []string `yaml:"-"` // Parsed failure categories for report-failure-as-issue (internal use only, included categories)
ReportFailureAsIssueExcludedCategories []string `yaml:"-"` // Parsed excluded failure categories for report-failure-as-issue (internal use only, categories starting with "!")
FailureIssueRepo string `yaml:"failure-issue-repo,omitempty"` // Repository to create failure issues in (format: "owner/repo"), defaults to current repo
Expand Down
5 changes: 5 additions & 0 deletions pkg/workflow/imports.go
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,11 @@ func mergeSafeOutputConfig(result *SafeOutputsConfig, config map[string]any, c *
if !result.GroupReports && importedConfig.GroupReports {
result.GroupReports = true
}
if result.ReportFailureAsIssue == nil && importedConfig.ReportFailureAsIssue != nil {
result.ReportFailureAsIssue = importedConfig.ReportFailureAsIssue
result.ReportFailureAsIssueCategories = importedConfig.ReportFailureAsIssueCategories
result.ReportFailureAsIssueExcludedCategories = importedConfig.ReportFailureAsIssueExcludedCategories
}
if result.FailureIssueRepo == "" && importedConfig.FailureIssueRepo != "" {
result.FailureIssueRepo = importedConfig.FailureIssueRepo
}
Expand Down
42 changes: 33 additions & 9 deletions pkg/workflow/notify_comment_conclusion_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,18 +311,42 @@ func buildAgentFailureReportingPolicyVars(data *WorkflowData) []string {
}
if data.SafeOutputs.ReportFailureAsIssue == nil {
envVars = append(envVars, " GH_AW_FAILURE_REPORT_AS_ISSUE: \"true\"\n")
} else if reportBool, ok := data.SafeOutputs.ReportFailureAsIssue.(bool); ok && !reportBool {
envVars = append(envVars, " GH_AW_FAILURE_REPORT_AS_ISSUE: \"false\"\n")
} else {
envVars = append(envVars, " GH_AW_FAILURE_REPORT_AS_ISSUE: \"true\"\n")
if len(data.SafeOutputs.ReportFailureAsIssueCategories) > 0 {
if categoriesJSON, err := json.Marshal(data.SafeOutputs.ReportFailureAsIssueCategories); err == nil {
envVars = append(envVars, fmt.Sprintf(" GH_AW_FAILURE_CATEGORIES_FILTER: %q\n", string(categoriesJSON)))
appendReportFailureEnvVar := func(enabled bool) {
envVars = append(envVars, fmt.Sprintf(" GH_AW_FAILURE_REPORT_AS_ISSUE: %q\n", strconv.FormatBool(enabled)))
}
shouldIncludeCategoryFilters := true
switch reportSetting := data.SafeOutputs.ReportFailureAsIssue.(type) {
case bool:
appendReportFailureEnvVar(reportSetting)
shouldIncludeCategoryFilters = reportSetting
case string:
reportExpression := reportSetting
switch reportExpression {
case "true":
appendReportFailureEnvVar(true)
case "false":
appendReportFailureEnvVar(false)
shouldIncludeCategoryFilters = false
default:
envVars = append(envVars, buildTemplatableBoolEnvVar("GH_AW_FAILURE_REPORT_AS_ISSUE", &reportExpression)...)
shouldIncludeCategoryFilters = false
}
case []any:
appendReportFailureEnvVar(true)
default:
appendReportFailureEnvVar(true)
}
if len(data.SafeOutputs.ReportFailureAsIssueExcludedCategories) > 0 {
if excludedJSON, err := json.Marshal(data.SafeOutputs.ReportFailureAsIssueExcludedCategories); err == nil {
envVars = append(envVars, fmt.Sprintf(" GH_AW_FAILURE_EXCLUDED_CATEGORIES_FILTER: %q\n", string(excludedJSON)))
if shouldIncludeCategoryFilters {
if len(data.SafeOutputs.ReportFailureAsIssueCategories) > 0 {
if categoriesJSON, err := json.Marshal(data.SafeOutputs.ReportFailureAsIssueCategories); err == nil {
envVars = append(envVars, fmt.Sprintf(" GH_AW_FAILURE_CATEGORIES_FILTER: %q\n", string(categoriesJSON)))
}
}
if len(data.SafeOutputs.ReportFailureAsIssueExcludedCategories) > 0 {
if excludedJSON, err := json.Marshal(data.SafeOutputs.ReportFailureAsIssueExcludedCategories); err == nil {
envVars = append(envVars, fmt.Sprintf(" GH_AW_FAILURE_EXCLUDED_CATEGORIES_FILTER: %q\n", string(excludedJSON)))
}
}
}
}
Expand Down
28 changes: 28 additions & 0 deletions pkg/workflow/notify_comment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1222,6 +1222,34 @@ func TestConclusionJobCategoriesFilterQuoting(t *testing.T) {
})
}

func TestConclusionJobReportFailureAsIssueTemplatableExpression(t *testing.T) {
compiler := NewCompiler()
workflowData := &WorkflowData{
Name: "Test Workflow",
SafeOutputs: &SafeOutputsConfig{
NoOp: &NoOpConfig{},
ReportFailureAsIssue: "${{ inputs.report-failure-as-issue }}",
ReportFailureAsIssueCategories: []string{"agent_failure"},
},
}

job, err := compiler.buildConclusionJob(workflowData, string(constants.AgentJobName), []string{})
if err != nil {
t.Fatalf("Failed to build conclusion job: %v", err)
}
if job == nil {
t.Fatal("Expected conclusion job to be created")
}

jobYAML := strings.Join(job.Steps, "")
if !strings.Contains(jobYAML, "GH_AW_FAILURE_REPORT_AS_ISSUE:") || !strings.Contains(jobYAML, "${{ inputs.report-failure-as-issue }}") {
t.Errorf("Expected templatable GH_AW_FAILURE_REPORT_AS_ISSUE env var in generated conclusion job YAML.\nGenerated YAML:\n%s", jobYAML)
}
if strings.Contains(jobYAML, "GH_AW_FAILURE_CATEGORIES_FILTER:") {
t.Errorf("Expected category filters to be omitted when report-failure-as-issue is templatable.\nGenerated YAML:\n%s", jobYAML)
}
}

func TestConclusionJobIncludesUsageArtifactSteps(t *testing.T) {
compiler := NewCompiler()
workflowData := &WorkflowData{
Expand Down
29 changes: 23 additions & 6 deletions pkg/workflow/safe_outputs_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -630,13 +630,10 @@ func (c *Compiler) extractSafeOutputsConfig(frontmatter map[string]any) *SafeOut
}
}

// Handle report-failure-as-issue flag or array of categories
// Handle report-failure-as-issue as templatable bool or array of categories.
if reportFailureAsIssue, exists := outputMap["report-failure-as-issue"]; exists {
// Support both bool (legacy) and []any (new with categories filter)
if reportFailureAsIssueBool, ok := reportFailureAsIssue.(bool); ok {
config.ReportFailureAsIssue = reportFailureAsIssueBool
safeOutputsConfigLog.Printf("Report failure as issue: %t", reportFailureAsIssueBool)
} else if categoriesList, ok := reportFailureAsIssue.([]any); ok {
// Support []any category filters.
if categoriesList, ok := reportFailureAsIssue.([]any); ok {
// Parse as array of category strings, separating included (no prefix) and excluded (! prefix)
includedCategories := make([]string, 0, len(categoriesList))
excludedCategories := make([]string, 0, len(categoriesList))
Expand All @@ -661,6 +658,26 @@ func (c *Compiler) extractSafeOutputsConfig(frontmatter map[string]any) *SafeOut
} else if len(excludedCategories) > 0 {
safeOutputsConfigLog.Printf("Report failure as issue with exclude filter: %v", excludedCategories)
}
} else {
// Support bool and templatable string values.
if err := preprocessBoolFieldAsString(outputMap, "report-failure-as-issue", safeOutputsConfigLog); err != nil {
safeOutputsConfigLog.Printf("Failed to preprocess report-failure-as-issue field: %v (ignoring invalid value and leaving field unset)", err)
} else {
if reportFailureAsIssueStr, ok := outputMap["report-failure-as-issue"].(string); ok {
switch reportFailureAsIssueStr {
case "true":
config.ReportFailureAsIssue = true
case "false":
config.ReportFailureAsIssue = false
default:
config.ReportFailureAsIssue = reportFailureAsIssueStr
}
safeOutputsConfigLog.Printf("Report failure as issue: %v", config.ReportFailureAsIssue)
} else if reportFailureAsIssueBool, ok := outputMap["report-failure-as-issue"].(bool); ok {
config.ReportFailureAsIssue = reportFailureAsIssueBool
safeOutputsConfigLog.Printf("Report failure as issue: %t", reportFailureAsIssueBool)
}
}
}
}

Expand Down
5 changes: 5 additions & 0 deletions pkg/workflow/safe_outputs_import_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,7 @@ safe-outputs:
allowed-domains:
- "example.com"
- "api.example.com"
report-failure-as-issue: ${{ inputs.report-failure-as-issue }}
staged: true
env:
TEST_VAR: "test_value"
Expand Down Expand Up @@ -785,6 +786,7 @@ This workflow uses the imported meta configuration.
assert.True(t, templatableBoolIsTrue(workflowData.SafeOutputs.Staged), "Staged should be imported and set to true")
assert.Equal(t, map[string]string{"TEST_VAR": "test_value"}, workflowData.SafeOutputs.Env, "Env should be imported")
assert.Equal(t, "${{ secrets.CUSTOM_TOKEN }}", workflowData.SafeOutputs.GitHubToken, "GitHubToken should be imported")
assert.Equal(t, "${{ inputs.report-failure-as-issue }}", workflowData.SafeOutputs.ReportFailureAsIssue, "ReportFailureAsIssue should be imported as templatable bool")
// Note: When main workflow has safe-outputs section, extractSafeOutputsConfig sets MaximumPatchSize default (4096)
// before merge happens, so imported value is not used. User should specify max-patch-size in main workflow.
assert.Equal(t, 4096, workflowData.SafeOutputs.MaximumPatchSize, "MaximumPatchSize defaults to 4096 when main has safe-outputs")
Expand All @@ -806,6 +808,7 @@ func TestSafeOutputsImportMetaFieldsMainTakesPrecedence(t *testing.T) {
safe-outputs:
allowed-domains:
- "shared.example.com"
report-failure-as-issue: false
github-token: "${{ secrets.SHARED_TOKEN }}"
max-patch-size: 1024
---
Expand All @@ -827,6 +830,7 @@ imports:
safe-outputs:
allowed-domains:
- "main.example.com"
report-failure-as-issue: ${{ inputs.report-failure-as-issue }}
github-token: "${{ secrets.MAIN_TOKEN }}"
max-patch-size: 2048
create-issue:
Expand Down Expand Up @@ -856,6 +860,7 @@ This workflow has its own meta configuration that should take precedence.

// Verify main workflow meta fields take precedence
assert.Equal(t, []string{"main.example.com"}, workflowData.SafeOutputs.AllowedDomains, "AllowedDomains from main should take precedence")
assert.Equal(t, "${{ inputs.report-failure-as-issue }}", workflowData.SafeOutputs.ReportFailureAsIssue, "ReportFailureAsIssue from main should take precedence")
assert.Equal(t, "${{ secrets.MAIN_TOKEN }}", workflowData.SafeOutputs.GitHubToken, "GitHubToken from main should take precedence")
assert.Equal(t, 2048, workflowData.SafeOutputs.MaximumPatchSize, "MaximumPatchSize from main should take precedence")
}
Expand Down
Loading