Skip to content

support background steps #4416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lokesh755 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

support background steps #4416

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b3cb470
support background steps
lokesh755 May 12, 2026
1002fd1
fix l0 tests
lokesh755 May 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/Runner.Worker/BackgroundStepContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
using System;
using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;

namespace GitHub.Runner.Worker
{
/// <summary>
/// Tracks a background step's execution state.
/// </summary>
internal sealed class BackgroundStepContext
{
public string StepId { get; }
public IStep Step { get; }
public Task ExecutionTask { get; set; }
public CancellationTokenSource Cts { get; set; }
public GitHub.DistributedTask.WebApi.TaskResult? Result { get; set; }
public bool IsCompleted => ExecutionTask?.IsCompleted ?? false;
public string ExternalId => Step.ExecutionContext == null || Step.ExecutionContext.Id == Guid.Empty ? null : Step.ExecutionContext.Id.ToString("N");
Copy link
Copy Markdown
Collaborator

@ericsciple ericsciple May 12, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering if Step.ExecutionContext.Id is ever null or empty?

iiuc external IDs are known during "Set up job"


public BackgroundStepContext(string stepId, IStep step)
{
StepId = stepId;
Step = step;
}
}
}
41 changes: 41 additions & 0 deletions src/Runner.Worker/CancelStepRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
using System;
using System.Threading.Tasks;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines.ContextData;

namespace GitHub.Runner.Worker
{
/// <summary>
/// A step that cancels a specific background step.
/// Execution is handled by StepsRunner, not by RunAsync.
/// </summary>
public sealed class CancelStepRunner : IStep
{
public string CancelStepId { get; set; }
public Guid StepId { get; set; }
public string StepName { get; set; }
public int RecordOrder { get; set; }
public string Condition { get; set; }
public TemplateToken ContinueOnError => null;
public string DisplayName { get; set; }
public IExecutionContext ExecutionContext { get; set; }
public TemplateToken Timeout => null;

public bool TryUpdateDisplayName(out bool updated)
{
updated = false;
return true;
}

public bool EvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context, out bool updated)
{
updated = false;
return true;
}

public Task RunAsync()
{
return Task.CompletedTask;
}
}
}
34 changes: 28 additions & 6 deletions src/Runner.Worker/ExecutionContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ public interface IExecutionContext : IRunnerService
void UpdateDetailTimelineRecord(TimelineRecord record);

void UpdateTimelineRecordDisplayName(string displayName);
void SetTimelineRecordVariable(string name, string value);

// matchers
void Add(OnMatcherChanged handler);
Expand Down Expand Up @@ -511,6 +512,24 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation =
Annotations = new List<Annotation>()
};

// Populate background step metadata from timeline record variables
if (_record.Variables.TryGetValue("is_background", out var bgVar) && bgVar.Value == "true")
{
stepResult.IsBackground = true;
}
if (_record.Variables.TryGetValue("step_type", out var stVar) && !string.IsNullOrEmpty(stVar.Value))
{
stepResult.StepType = stVar.Value;
}
if (_record.Variables.TryGetValue("wait_step_ids", out var wsVar) && !string.IsNullOrEmpty(wsVar.Value))
{
stepResult.WaitStepIds = wsVar.Value.Split(',');
}
if (_record.Variables.TryGetValue("cancel_step_id", out var csVar) && !string.IsNullOrEmpty(csVar.Value))
{
stepResult.CancelStepId = csVar.Value;
}

_record.Issues?.ForEach(issue =>
{
var annotation = issue.ToAnnotation();
Expand Down Expand Up @@ -807,6 +826,12 @@ public void UpdateTimelineRecordDisplayName(string displayName)
_jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
}

public void SetTimelineRecordVariable(string name, string value)
{
_record.Variables[name] = new VariableValue(value);
_jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
}

public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Expand Down Expand Up @@ -1332,8 +1357,9 @@ public void ApplyContinueOnError(TemplateToken continueOnErrorToken)
UpdateGlobalStepsContext();
}

internal IPipelineTemplateEvaluator ToPipelineTemplateEvaluatorInternal(bool allowServiceContainerCommand, ObjectTemplating.ITraceWriter traceWriter = null)
internal IPipelineTemplateEvaluator ToPipelineTemplateEvaluatorInternal(ObjectTemplating.ITraceWriter traceWriter = null)
{
var allowServiceContainerCommand = Global.Variables.GetBoolean(Constants.Runner.Features.ServiceContainerCommand) ?? false;
return new PipelineTemplateEvaluatorWrapper(HostContext, this, allowServiceContainerCommand, traceWriter);
}

Expand Down Expand Up @@ -1422,13 +1448,10 @@ public static IEnumerable<KeyValuePair<string, object>> ToExpressionState(this I

public static IPipelineTemplateEvaluator ToPipelineTemplateEvaluator(this IExecutionContext context, ObjectTemplating.ITraceWriter traceWriter = null)
{
var allowServiceContainerCommand = (context.Global.Variables.GetBoolean(Constants.Runner.Features.ServiceContainerCommand) ?? false)
|| StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("ACTIONS_SERVICE_CONTAINER_COMMAND"));

// Create wrapper?
if ((context.Global.Variables.GetBoolean(Constants.Runner.Features.CompareWorkflowParser) ?? false) || StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("ACTIONS_RUNNER_COMPARE_WORKFLOW_PARSER")))
{
return (context as ExecutionContext).ToPipelineTemplateEvaluatorInternal(allowServiceContainerCommand, traceWriter);
return (context as ExecutionContext).ToPipelineTemplateEvaluatorInternal(traceWriter);
}

// Legacy
Expand All @@ -1440,7 +1463,6 @@ public static IPipelineTemplateEvaluator ToPipelineTemplateEvaluator(this IExecu
return new PipelineTemplateEvaluator(traceWriter, schema, context.Global.FileTable)
{
MaxErrorMessageLength = int.MaxValue, // Don't truncate error messages otherwise we might not scrub secrets correctly
AllowServiceContainerCommand = allowServiceContainerCommand,
};
}

Expand Down
85 changes: 85 additions & 0 deletions src/Runner.Worker/JobExtension.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,8 +315,10 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
}

// Add action steps
var stepOrder = 0;
foreach (var step in message.Steps)
{
stepOrder++;
Comment on lines +318 to +321
Copy link
Copy Markdown
Collaborator

@ericsciple ericsciple May 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this used?

if (step.Type == Pipelines.StepType.Action)
{
var action = step as Pipelines.ActionStep;
Expand Down Expand Up @@ -345,6 +347,53 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
preJobSteps.Add(preStep);
}
}
else if (step.Type == Pipelines.StepType.Wait)
{
var waitStep = step as Pipelines.WaitStep;
Trace.Info($"Adding wait step for: {string.Join(", ", waitStep.WaitStepIds ?? System.Array.Empty<string>())}");
Trace.Info($"Wait step: DisplayNameToken={waitStep.DisplayNameToken?.GetType().Name ?? "null"}, DisplayName={step.DisplayName ?? "null"}, Name={step.Name ?? "null"}");
var waitStepName = (waitStep.DisplayNameToken as GitHub.DistributedTask.ObjectTemplating.Tokens.StringToken)?.Value
?? step.DisplayName ?? step.Name ?? "Wait for background steps";
Trace.Info($"Wait step resolved name: {waitStepName}");
var waitRunner = new WaitStepRunner
{
StepIds = waitStep.WaitStepIds,
DisplayName = waitStepName,
Condition = step.Condition,
StepId = step.Id,
StepName = step.Name,
};
// ExecutionContext created later in "Create execution context for job steps" loop
jobSteps.Add(waitRunner);
}
else if (step.Type == Pipelines.StepType.WaitAll)
{
Trace.Info("Adding wait-all step.");
var waitAllRunner = new WaitAllStepRunner
{
DisplayName = step.DisplayName ?? step.Name ?? "Wait for all background steps",
Condition = step.Condition,
StepId = step.Id,
StepName = step.Name,
};
// ExecutionContext created later in "Create execution context for job steps" loop
jobSteps.Add(waitAllRunner);
}
else if (step.Type == Pipelines.StepType.Cancel)
{
var cancelStep = step as Pipelines.CancelStep;
Trace.Info($"Adding cancel step for: {cancelStep.CancelStepId}");
var cancelRunner = new CancelStepRunner
{
CancelStepId = cancelStep.CancelStepId,
DisplayName = (cancelStep.DisplayNameToken as GitHub.DistributedTask.ObjectTemplating.Tokens.StringToken)?.Value ?? step.DisplayName ?? step.Name ?? "Cancel background step",
Condition = step.Condition,
StepId = step.Id,
StepName = step.Name,
};
// ExecutionContext created later in "Create execution context for job steps" loop
jobSteps.Add(cancelRunner);
}
}

if (message.Variables.TryGetValue("system.workflowFileFullPath", out VariableValue workflowFileFullPath))
Expand Down Expand Up @@ -407,6 +456,42 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
ArgUtil.NotNull(actionStep, step.DisplayName);
intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState);
actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, null, actionStep.Action.ContextName, ActionRunStage.Main, intraActionState);

// Store background step metadata on the timeline record for results service
if (actionStep.Action?.Background == true)
Copy link
Copy Markdown
Collaborator

@ericsciple ericsciple May 12, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering whether pre/post should not follow background

That is:

  • pre-stage would be sequential
  • main-stage could have background
  • implicit wait at the end of main-stage
  • post-stage is sequential

Otherwise might need implicit waits at the end of each stage, which might be fine too.

{
actionStep.ExecutionContext.SetTimelineRecordVariable("is_background", "true");
actionStep.ExecutionContext.SetTimelineRecordVariable("step_type", "action");
}
}
else if (step is WaitStepRunner waitRunner)
{
waitRunner.ExecutionContext = jobContext.CreateChild(
waitRunner.StepId, waitRunner.DisplayName, waitRunner.StepName,
null, waitRunner.StepName, ActionRunStage.Main);
waitRunner.ExecutionContext.SetTimelineRecordVariable("step_type", "wait");
if (waitRunner.StepIds != null && waitRunner.StepIds.Length > 0)
{
waitRunner.ExecutionContext.SetTimelineRecordVariable("wait_step_ids", string.Join(",", waitRunner.StepIds));
}
Comment on lines +472 to +476
}
else if (step is WaitAllStepRunner waitAllRunner)
{
waitAllRunner.ExecutionContext = jobContext.CreateChild(
waitAllRunner.StepId, waitAllRunner.DisplayName, waitAllRunner.StepName,
null, waitAllRunner.StepName, ActionRunStage.Main);
waitAllRunner.ExecutionContext.SetTimelineRecordVariable("step_type", "wait-all");
}
else if (step is CancelStepRunner cancelRunner)
{
cancelRunner.ExecutionContext = jobContext.CreateChild(
cancelRunner.StepId, cancelRunner.DisplayName, cancelRunner.StepName,
null, cancelRunner.StepName, ActionRunStage.Main);
cancelRunner.ExecutionContext.SetTimelineRecordVariable("step_type", "cancel");
if (!string.IsNullOrEmpty(cancelRunner.CancelStepId))
{
cancelRunner.ExecutionContext.SetTimelineRecordVariable("cancel_step_id", cancelRunner.CancelStepId);
}
}
}

Expand Down
36 changes: 23 additions & 13 deletions src/Runner.Worker/StepsContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public sealed class StepsContext
{
private static readonly Regex _propertyRegex = new("^[a-zA-Z_][a-zA-Z0-9_]*$", RegexOptions.Compiled);
private readonly DictionaryContextData _contextData = new();
private readonly object _lock = new();

/// <summary>
/// Clears memory for a composite action's isolated "steps" context, after the action
Expand Down Expand Up @@ -67,16 +68,19 @@ public void SetOutput(
string value,
out string reference)
{
var step = GetStep(scopeName, stepName);
var outputs = step["outputs"].AssertDictionary("outputs");
outputs[outputName] = new StringContextData(value);
if (_propertyRegex.IsMatch(outputName))
lock (_lock)
Copy link
Copy Markdown
Collaborator

@ericsciple ericsciple May 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm thinking background step outputs shouldn't be available to foreground steps before the wait synchronization point

{
reference = $"steps.{stepName}.outputs.{outputName}";
}
else
{
reference = $"steps['{stepName}']['outputs']['{outputName}']";
var step = GetStep(scopeName, stepName);
var outputs = step["outputs"].AssertDictionary("outputs");
outputs[outputName] = new StringContextData(value);
if (_propertyRegex.IsMatch(outputName))
{
reference = $"steps.{stepName}.outputs.{outputName}";
}
else
{
reference = $"steps['{stepName}']['outputs']['{outputName}']";
}
}
Comment on lines +71 to 84
}

Expand All @@ -85,17 +89,23 @@ public void SetConclusion(
string stepName,
ActionResult conclusion)
{
var step = GetStep(scopeName, stepName);
step["conclusion"] = new StringContextData(conclusion.ToString().ToLowerInvariant());
lock (_lock)
{
var step = GetStep(scopeName, stepName);
step["conclusion"] = new StringContextData(conclusion.ToString().ToLowerInvariant());
}
}

public void SetOutcome(
string scopeName,
string stepName,
ActionResult outcome)
{
var step = GetStep(scopeName, stepName);
step["outcome"] = new StringContextData(outcome.ToString().ToLowerInvariant());
lock (_lock)
{
var step = GetStep(scopeName, stepName);
step["outcome"] = new StringContextData(outcome.ToString().ToLowerInvariant());
}
}

private DictionaryContextData GetStep(string scopeName, string stepName)
Expand Down
Loading
Loading