From c91d07aae05fca43d86b435458bcae514c0f6650 Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Tue, 24 Mar 2026 16:13:59 +0100 Subject: [PATCH 01/10] feat(runners): support OnDemandOptions.AllocationStrategy in EC2 Fleet Previously, createInstances always set SpotOptions.AllocationStrategy even for on-demand fleets, making instance_allocation_strategy and instance_types ordering meaningless for on-demand users wanting `prioritized`. Now the fleet request conditionally sets SpotOptions or OnDemandOptions based on targetCapacityType, and the spot-to- on-demand failover path maps spot-only strategies to `lowest-price`. --- .../control-plane/src/aws/runners.d.ts | 3 +- .../control-plane/src/aws/runners.test.ts | 49 +++++++++++++++---- .../control-plane/src/aws/runners.ts | 32 ++++++++++-- modules/multi-runner/variables.tf | 2 +- modules/runners/variables.tf | 4 +- variables.tf | 4 +- 6 files changed, 74 insertions(+), 20 deletions(-) diff --git a/lambdas/functions/control-plane/src/aws/runners.d.ts b/lambdas/functions/control-plane/src/aws/runners.d.ts index 01cd4c1459..d7506b0b5d 100644 --- a/lambdas/functions/control-plane/src/aws/runners.d.ts +++ b/lambdas/functions/control-plane/src/aws/runners.d.ts @@ -1,5 +1,6 @@ import { DefaultTargetCapacityType, + FleetOnDemandAllocationStrategy, InstanceRequirementsRequest, SpotAllocationStrategy, _InstanceType, @@ -61,7 +62,7 @@ export interface RunnerInputParameters { instanceTypes: string[]; targetCapacityType: DefaultTargetCapacityType; maxSpotPrice?: string; - instanceAllocationStrategy: SpotAllocationStrategy; + instanceAllocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy; }; ec2OverrideConfig?: Ec2OverrideConfig; numberOfRunners: number; diff --git a/lambdas/functions/control-plane/src/aws/runners.test.ts b/lambdas/functions/control-plane/src/aws/runners.test.ts index e9f6c13969..8e35502998 100644 --- a/lambdas/functions/control-plane/src/aws/runners.test.ts +++ b/lambdas/functions/control-plane/src/aws/runners.test.ts @@ -10,6 +10,7 @@ import { DescribeInstancesCommand, type DescribeInstancesResult, EC2Client, + FleetOnDemandAllocationStrategy, RunInstancesCommand, SpotAllocationStrategy, TerminateInstancesCommand, @@ -390,11 +391,31 @@ describe('create runner', () => { }); it('calls create fleet of 1 instance with the on-demand capacity', async () => { - await createRunner(createRunnerConfig({ ...defaultRunnerConfig, capacityType: 'on-demand' })); + await createRunner( + createRunnerConfig({ ...defaultRunnerConfig, capacityType: 'on-demand', allocationStrategy: 'lowest-price' }), + ); expect(mockEC2Client).toHaveReceivedCommandWith(CreateFleetCommand, { ...expectedCreateFleetRequest({ ...defaultExpectedFleetRequestValues, capacityType: 'on-demand', + allocationStrategy: 'lowest-price', + }), + }); + }); + + it('calls create fleet with on-demand capacity and prioritized allocation strategy', async () => { + await createRunner( + createRunnerConfig({ + ...defaultRunnerConfig, + capacityType: 'on-demand', + allocationStrategy: FleetOnDemandAllocationStrategy.PRIORITIZED, + }), + ); + expect(mockEC2Client).toHaveReceivedCommandWith(CreateFleetCommand, { + ...expectedCreateFleetRequest({ + ...defaultExpectedFleetRequestValues, + capacityType: 'on-demand', + allocationStrategy: FleetOnDemandAllocationStrategy.PRIORITIZED, }), }); }); @@ -841,12 +862,13 @@ describe('create runner with errors fail over to OnDemand', () => { }), }); - // second call with with OnDemand fallback + // second call with with OnDemand fallback, allocation strategy defaults to lowest-price expect(mockEC2Client).toHaveReceivedNthCommandWith(2, CreateFleetCommand, { ...expectedCreateFleetRequest({ ...defaultExpectedFleetRequestValues, totalTargetCapacity: 1, capacityType: 'on-demand', + allocationStrategy: 'lowest-price', }), }); }); @@ -883,12 +905,13 @@ describe('create runner with errors fail over to OnDemand', () => { }), }); - // second call with with OnDemand failback, capacity is reduced by 1 + // second call with with OnDemand failback, capacity is reduced by 1, allocation strategy defaults to lowest-price expect(mockEC2Client).toHaveReceivedNthCommandWith(2, CreateFleetCommand, { ...expectedCreateFleetRequest({ ...defaultExpectedFleetRequestValues, totalTargetCapacity: 1, capacityType: 'on-demand', + allocationStrategy: 'lowest-price', }), }); }); @@ -958,7 +981,7 @@ function createFleetMockWithWithOnDemandFallback(errors: string[], instances?: s interface RunnerConfig { type: RunnerType; capacityType: DefaultTargetCapacityType; - allocationStrategy: SpotAllocationStrategy; + allocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy; maxSpotPrice?: string; amiIdSsmParameterName?: string; tracingEnabled?: boolean; @@ -994,7 +1017,7 @@ function createRunnerConfig(runnerConfig: RunnerConfig): RunnerInputParameters { interface ExpectedFleetRequestValues { type: 'Repo' | 'Org'; capacityType: DefaultTargetCapacityType; - allocationStrategy: SpotAllocationStrategy; + allocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy; maxSpotPrice?: string; totalTargetCapacity: number; imageId?: string; @@ -1043,10 +1066,18 @@ function expectedCreateFleetRequest(expectedValues: ExpectedFleetRequestValues): ], }, ], - SpotOptions: { - AllocationStrategy: expectedValues.allocationStrategy, - MaxTotalPrice: expectedValues.maxSpotPrice, - }, + ...(expectedValues.capacityType === 'spot' + ? { + SpotOptions: { + AllocationStrategy: expectedValues.allocationStrategy, + MaxTotalPrice: expectedValues.maxSpotPrice, + }, + } + : { + OnDemandOptions: { + AllocationStrategy: expectedValues.allocationStrategy, + }, + }), TagSpecifications: [ { ResourceType: 'instance', diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index 0240d86a77..08226dbb52 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -8,6 +8,8 @@ import { RunInstancesCommand, EC2Client, FleetLaunchTemplateOverridesRequest, + FleetOnDemandAllocationStrategy, + SpotAllocationStrategy, Tag, TerminateInstancesCommand, _InstanceType, @@ -205,11 +207,21 @@ async function processFleetResult( logger.warn(`Create fleet failed, initatiing fall back to on demand instances.`); logger.debug('Create fleet failed.', { data: fleet.Errors }); const numberOfInstances = runnerParameters.numberOfRunners - instances.length; + const onDemandValidStrategies = ['lowest-price', 'prioritized']; + const failoverAllocationStrategy = onDemandValidStrategies.includes( + runnerParameters.ec2instanceCriteria.instanceAllocationStrategy, + ) + ? runnerParameters.ec2instanceCriteria.instanceAllocationStrategy + : 'lowest-price'; const instancesOnDemand = await createRunner({ ...runnerParameters, numberOfRunners: numberOfInstances, onDemandFailoverOnError: ['InsufficientInstanceCapacity'], - ec2instanceCriteria: { ...runnerParameters.ec2instanceCriteria, targetCapacityType: 'on-demand' }, + ec2instanceCriteria: { + ...runnerParameters.ec2instanceCriteria, + targetCapacityType: 'on-demand', + instanceAllocationStrategy: failoverAllocationStrategy, + }, }); instances.push(...instancesOnDemand); return instances; @@ -287,10 +299,20 @@ async function createInstances( ), }, ], - SpotOptions: { - MaxTotalPrice: runnerParameters.ec2instanceCriteria.maxSpotPrice, - AllocationStrategy: runnerParameters.ec2instanceCriteria.instanceAllocationStrategy, - }, + ...(runnerParameters.ec2instanceCriteria.targetCapacityType === 'spot' + ? { + SpotOptions: { + MaxTotalPrice: runnerParameters.ec2instanceCriteria.maxSpotPrice, + AllocationStrategy: + runnerParameters.ec2instanceCriteria.instanceAllocationStrategy as SpotAllocationStrategy, + }, + } + : { + OnDemandOptions: { + AllocationStrategy: + runnerParameters.ec2instanceCriteria.instanceAllocationStrategy as FleetOnDemandAllocationStrategy, + }, + }), TargetCapacitySpecification: { TotalTargetCapacity: runnerParameters.numberOfRunners, DefaultTargetCapacityType: runnerParameters.ec2instanceCriteria.targetCapacityType, diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index 78b9ec2a25..279d9304ea 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -232,7 +232,7 @@ variable "multi_runner_config" { enable_runner_binaries_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI." enable_ssm_on_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances." enable_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI." - instance_allocation_strategy: "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`." + instance_allocation_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." instance_max_spot_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet." instance_target_capacity_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`." instance_types: "List of instance types for the action runner. Defaults are based on runner_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)." diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 52b2babb55..63c88f7767 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -129,12 +129,12 @@ variable "instance_target_capacity_type" { } variable "instance_allocation_strategy" { - description = "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`." + description = "The allocation strategy for creating instances. For spot, AWS recommends `capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." type = string default = "lowest-price" validation { - condition = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized"], var.instance_allocation_strategy) + condition = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized", "prioritized"], var.instance_allocation_strategy) error_message = "The instance allocation strategy does not match the allowed values." } } diff --git a/variables.tf b/variables.tf index 7aa90845de..e124f94725 100644 --- a/variables.tf +++ b/variables.tf @@ -581,11 +581,11 @@ variable "instance_target_capacity_type" { } variable "instance_allocation_strategy" { - description = "The allocation strategy for spot instances. AWS recommends using `price-capacity-optimized` however the AWS default is `lowest-price`." + description = "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." type = string default = "lowest-price" validation { - condition = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized"], var.instance_allocation_strategy) + condition = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized", "prioritized"], var.instance_allocation_strategy) error_message = "The instance allocation strategy does not match the allowed values." } } From 534c1e022ebdd73839710a8badcbe8af969b7590 Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Tue, 24 Mar 2026 16:33:00 +0100 Subject: [PATCH 02/10] feat(runners): add instance_type_priorities for prioritized allocation Add optional `instance_type_priorities` variable (map of instance type to priority number) to control EC2 Fleet override priorities. When not set, priorities default to the index position in `instance_types`, preserving the user's ordering. This makes the `prioritized` allocation strategy work correctly for both spot and on-demand fleets. --- .../control-plane/src/aws/runners.d.ts | 1 + .../control-plane/src/aws/runners.test.ts | 27 +++++++++++++++++++ .../control-plane/src/aws/runners.ts | 5 +++- .../functions/control-plane/src/pool/pool.ts | 4 +++ .../src/scale-runners/scale-up.ts | 4 +++ main.tf | 1 + modules/multi-runner/runners.tf | 1 + modules/multi-runner/variables.tf | 2 ++ modules/runners/pool.tf | 1 + modules/runners/pool/main.tf | 1 + modules/runners/pool/variables.tf | 1 + modules/runners/scale-up.tf | 1 + modules/runners/variables.tf | 6 +++++ variables.tf | 6 +++++ 14 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lambdas/functions/control-plane/src/aws/runners.d.ts b/lambdas/functions/control-plane/src/aws/runners.d.ts index d7506b0b5d..770106a98d 100644 --- a/lambdas/functions/control-plane/src/aws/runners.d.ts +++ b/lambdas/functions/control-plane/src/aws/runners.d.ts @@ -60,6 +60,7 @@ export interface RunnerInputParameters { launchTemplateName: string; ec2instanceCriteria: { instanceTypes: string[]; + instanceTypePriorities?: Record; targetCapacityType: DefaultTargetCapacityType; maxSpotPrice?: string; instanceAllocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy; diff --git a/lambdas/functions/control-plane/src/aws/runners.test.ts b/lambdas/functions/control-plane/src/aws/runners.test.ts index 8e35502998..a37d8fccd5 100644 --- a/lambdas/functions/control-plane/src/aws/runners.test.ts +++ b/lambdas/functions/control-plane/src/aws/runners.test.ts @@ -420,6 +420,26 @@ describe('create runner', () => { }); }); + it('calls create fleet with custom instance type priorities', async () => { + const priorities = { 'm5.large': 10, 'c5.large': 5 }; + await createRunner( + createRunnerConfig({ + ...defaultRunnerConfig, + capacityType: 'on-demand', + allocationStrategy: FleetOnDemandAllocationStrategy.PRIORITIZED, + instanceTypePriorities: priorities, + }), + ); + expect(mockEC2Client).toHaveReceivedCommandWith(CreateFleetCommand, { + ...expectedCreateFleetRequest({ + ...defaultExpectedFleetRequestValues, + capacityType: 'on-demand', + allocationStrategy: FleetOnDemandAllocationStrategy.PRIORITIZED, + instanceTypePriorities: priorities, + }), + }); + }); + it('calls run instances with the on-demand capacity', async () => { await createRunner(createRunnerConfig({ ...defaultRunnerConfig, maxSpotPrice: '0.1' })); expect(mockEC2Client).toHaveReceivedCommandWith(CreateFleetCommand, { @@ -982,6 +1002,7 @@ interface RunnerConfig { type: RunnerType; capacityType: DefaultTargetCapacityType; allocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy; + instanceTypePriorities?: Record; maxSpotPrice?: string; amiIdSsmParameterName?: string; tracingEnabled?: boolean; @@ -1000,6 +1021,7 @@ function createRunnerConfig(runnerConfig: RunnerConfig): RunnerInputParameters { launchTemplateName: LAUNCH_TEMPLATE, ec2instanceCriteria: { instanceTypes: ['m5.large', 'c5.large'], + instanceTypePriorities: runnerConfig.instanceTypePriorities, targetCapacityType: runnerConfig.capacityType, maxSpotPrice: runnerConfig.maxSpotPrice, instanceAllocationStrategy: runnerConfig.allocationStrategy, @@ -1018,6 +1040,7 @@ interface ExpectedFleetRequestValues { type: 'Repo' | 'Org'; capacityType: DefaultTargetCapacityType; allocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy; + instanceTypePriorities?: Record; maxSpotPrice?: string; totalTargetCapacity: number; imageId?: string; @@ -1050,18 +1073,22 @@ function expectedCreateFleetRequest(expectedValues: ExpectedFleetRequestValues): { InstanceType: 'm5.large', SubnetId: 'subnet-123', + Priority: expectedValues.instanceTypePriorities?.['m5.large'] ?? 0, }, { InstanceType: 'c5.large', SubnetId: 'subnet-123', + Priority: expectedValues.instanceTypePriorities?.['c5.large'] ?? 1, }, { InstanceType: 'm5.large', SubnetId: 'subnet-456', + Priority: expectedValues.instanceTypePriorities?.['m5.large'] ?? 0, }, { InstanceType: 'c5.large', SubnetId: 'subnet-456', + Priority: expectedValues.instanceTypePriorities?.['c5.large'] ?? 1, }, ], }, diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index 08226dbb52..a00248dfb8 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -129,6 +129,7 @@ function generateFleetOverrides( instancesTypes: string[], amiId?: string, ec2OverrideConfig?: Runners.Ec2OverrideConfig, + instanceTypePriorities?: Record, ): FleetLaunchTemplateOverridesRequest[] { const result: FleetLaunchTemplateOverridesRequest[] = []; @@ -138,12 +139,13 @@ function generateFleetOverrides( const amiIdToUse = ec2OverrideConfig?.ImageId ?? amiId; subnetsToUse.forEach((s) => { - instanceTypesToUse.forEach((i) => { + instanceTypesToUse.forEach((i, index) => { const item: FleetLaunchTemplateOverridesRequest = { SubnetId: s, InstanceType: i as _InstanceType, ImageId: amiIdToUse, ...ec2OverrideConfig, + Priority: instanceTypePriorities?.[i] ?? index, }; result.push(item); }); @@ -296,6 +298,7 @@ async function createInstances( runnerParameters.ec2instanceCriteria.instanceTypes, amiIdOverride, runnerParameters.ec2OverrideConfig, + runnerParameters.ec2instanceCriteria.instanceTypePriorities, ), }, ], diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index cece8d9951..c5cfcd1b7e 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -36,6 +36,9 @@ export async function adjust(event: PoolEvent): Promise { const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME; const instanceMaxSpotPrice = process.env.INSTANCE_MAX_SPOT_PRICE; const instanceAllocationStrategy = process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price'; // same as AWS default + const instanceTypePriorities = process.env.INSTANCE_TYPE_PRIORITIES + ? (JSON.parse(process.env.INSTANCE_TYPE_PRIORITIES) as Record) + : undefined; const runnerOwner = process.env.RUNNER_OWNER; const amiIdSsmParameterName = process.env.AMI_ID_SSM_PARAMETER_NAME; const tracingEnabled = yn(process.env.POWERTOOLS_TRACE_ENABLED, { default: false }); @@ -92,6 +95,7 @@ export async function adjust(event: PoolEvent): Promise { { ec2instanceCriteria: { instanceTypes, + instanceTypePriorities, targetCapacityType: instanceTargetCapacityType, maxSpotPrice: instanceMaxSpotPrice, instanceAllocationStrategy: instanceAllocationStrategy, diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index b742264842..1cf68b1e88 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -342,6 +342,9 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise) + : undefined; const enableJobQueuedCheck = yn(process.env.ENABLE_JOB_QUEUED_CHECK, { default: true }); const amiIdSsmParameterName = process.env.AMI_ID_SSM_PARAMETER_NAME; const runnerNamePrefix = process.env.RUNNER_NAME_PREFIX || ''; @@ -575,6 +578,7 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise Date: Tue, 24 Mar 2026 16:41:16 +0100 Subject: [PATCH 03/10] style(runners): fix prettier formatting --- lambdas/functions/control-plane/src/aws/runners.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index a00248dfb8..c0cd7796b3 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -306,14 +306,14 @@ async function createInstances( ? { SpotOptions: { MaxTotalPrice: runnerParameters.ec2instanceCriteria.maxSpotPrice, - AllocationStrategy: - runnerParameters.ec2instanceCriteria.instanceAllocationStrategy as SpotAllocationStrategy, + AllocationStrategy: runnerParameters.ec2instanceCriteria + .instanceAllocationStrategy as SpotAllocationStrategy, }, } : { OnDemandOptions: { - AllocationStrategy: - runnerParameters.ec2instanceCriteria.instanceAllocationStrategy as FleetOnDemandAllocationStrategy, + AllocationStrategy: runnerParameters.ec2instanceCriteria + .instanceAllocationStrategy as FleetOnDemandAllocationStrategy, }, }), TargetCapacitySpecification: { From 64b834e221a8303f704ed8ec3d8d20b701d6c2e6 Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Tue, 24 Mar 2026 17:00:14 +0100 Subject: [PATCH 04/10] refactor(runners): only set Priority on overrides for prioritized strategy --- .../control-plane/src/aws/runners.test.ts | 16 ++++++++++++---- .../functions/control-plane/src/aws/runners.ts | 8 +++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lambdas/functions/control-plane/src/aws/runners.test.ts b/lambdas/functions/control-plane/src/aws/runners.test.ts index a37d8fccd5..a1735c9c22 100644 --- a/lambdas/functions/control-plane/src/aws/runners.test.ts +++ b/lambdas/functions/control-plane/src/aws/runners.test.ts @@ -1073,22 +1073,30 @@ function expectedCreateFleetRequest(expectedValues: ExpectedFleetRequestValues): { InstanceType: 'm5.large', SubnetId: 'subnet-123', - Priority: expectedValues.instanceTypePriorities?.['m5.large'] ?? 0, + ...(expectedValues.allocationStrategy === 'prioritized' && { + Priority: expectedValues.instanceTypePriorities?.['m5.large'] ?? 0, + }), }, { InstanceType: 'c5.large', SubnetId: 'subnet-123', - Priority: expectedValues.instanceTypePriorities?.['c5.large'] ?? 1, + ...(expectedValues.allocationStrategy === 'prioritized' && { + Priority: expectedValues.instanceTypePriorities?.['c5.large'] ?? 1, + }), }, { InstanceType: 'm5.large', SubnetId: 'subnet-456', - Priority: expectedValues.instanceTypePriorities?.['m5.large'] ?? 0, + ...(expectedValues.allocationStrategy === 'prioritized' && { + Priority: expectedValues.instanceTypePriorities?.['m5.large'] ?? 0, + }), }, { InstanceType: 'c5.large', SubnetId: 'subnet-456', - Priority: expectedValues.instanceTypePriorities?.['c5.large'] ?? 1, + ...(expectedValues.allocationStrategy === 'prioritized' && { + Priority: expectedValues.instanceTypePriorities?.['c5.large'] ?? 1, + }), }, ], }, diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index c0cd7796b3..8045156774 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -145,7 +145,7 @@ function generateFleetOverrides( InstanceType: i as _InstanceType, ImageId: amiIdToUse, ...ec2OverrideConfig, - Priority: instanceTypePriorities?.[i] ?? index, + ...(instanceTypePriorities !== undefined && { Priority: instanceTypePriorities[i] ?? index }), }; result.push(item); }); @@ -297,8 +297,14 @@ async function createInstances( runnerParameters.subnets, runnerParameters.ec2instanceCriteria.instanceTypes, amiIdOverride, +<<<<<<< HEAD runnerParameters.ec2OverrideConfig, runnerParameters.ec2instanceCriteria.instanceTypePriorities, +======= + runnerParameters.ec2instanceCriteria.instanceAllocationStrategy === 'prioritized' + ? (runnerParameters.ec2instanceCriteria.instanceTypePriorities ?? {}) + : undefined, +>>>>>>> 68c399a9 (refactor(runners): only set Priority on overrides for prioritized strategy) ), }, ], From d5170512483954b418b665331d316bf8c2b6f09e Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Tue, 24 Mar 2026 17:11:20 +0100 Subject: [PATCH 05/10] refactor(runners): pass allocationStrategy to generateFleetOverrides Move the prioritized strategy check into generateFleetOverrides itself rather than having the caller decide what to pass, making the logic more explicit and self-contained. --- lambdas/functions/control-plane/src/aws/runners.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index 8045156774..a3ff7d3530 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -129,6 +129,7 @@ function generateFleetOverrides( instancesTypes: string[], amiId?: string, ec2OverrideConfig?: Runners.Ec2OverrideConfig, + allocationStrategy?: string, instanceTypePriorities?: Record, ): FleetLaunchTemplateOverridesRequest[] { const result: FleetLaunchTemplateOverridesRequest[] = []; @@ -145,7 +146,7 @@ function generateFleetOverrides( InstanceType: i as _InstanceType, ImageId: amiIdToUse, ...ec2OverrideConfig, - ...(instanceTypePriorities !== undefined && { Priority: instanceTypePriorities[i] ?? index }), + ...(allocationStrategy === 'prioritized' && { Priority: instanceTypePriorities?.[i] ?? index }), }; result.push(item); }); @@ -297,14 +298,9 @@ async function createInstances( runnerParameters.subnets, runnerParameters.ec2instanceCriteria.instanceTypes, amiIdOverride, -<<<<<<< HEAD runnerParameters.ec2OverrideConfig, + runnerParameters.ec2instanceCriteria.instanceAllocationStrategy, runnerParameters.ec2instanceCriteria.instanceTypePriorities, -======= - runnerParameters.ec2instanceCriteria.instanceAllocationStrategy === 'prioritized' - ? (runnerParameters.ec2instanceCriteria.instanceTypePriorities ?? {}) - : undefined, ->>>>>>> 68c399a9 (refactor(runners): only set Priority on overrides for prioritized strategy) ), }, ], From d81b270d6536b7fa7a87cd72e3442b97888d2c2e Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Thu, 11 Jun 2026 11:28:05 +0200 Subject: [PATCH 06/10] refactor(runners): let ec2OverrideConfig take precedence over computed priority Spread ec2OverrideConfig after the computed prioritized Priority so an explicitly provided override value (e.g. Priority from a ghr-ec2-* label) wins when both are present. Co-Authored-By: Claude Opus 4.8 (1M context) --- lambdas/functions/control-plane/src/aws/runners.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index a3ff7d3530..3508724603 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -145,8 +145,8 @@ function generateFleetOverrides( SubnetId: s, InstanceType: i as _InstanceType, ImageId: amiIdToUse, - ...ec2OverrideConfig, ...(allocationStrategy === 'prioritized' && { Priority: instanceTypePriorities?.[i] ?? index }), + ...ec2OverrideConfig, }; result.push(item); }); From 8534c95e099e65587852b01dcb966399cacf3cc5 Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Thu, 11 Jun 2026 11:42:08 +0200 Subject: [PATCH 07/10] fix(runners): sanitize allocation strategy per target capacity type The instance_allocation_strategy variable accepts the union of spot and on-demand strategies, so a value valid for one capacity type can be invalid for the other. Routing it unsanitized into SpotOptions/OnDemandOptions could send invalid values to AWS (e.g. capacity-optimized for on-demand, prioritized for spot) and fail CreateFleet, and broke backwards compatibility for on-demand users whose spot-only strategy was previously ignored. Add sanitizeAllocationStrategy() that falls back to lowest-price (the AWS default) when the configured strategy is invalid for the target capacity type, and reuse it in the on-demand failover path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../control-plane/src/aws/runners.ts | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index 3508724603..254496dd36 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -124,6 +124,30 @@ export async function untag(instanceId: string, tags: Tag[]): Promise { await ec2.send(new DeleteTagsCommand({ Resources: [instanceId], Tags: tags })); } +const SPOT_ALLOCATION_STRATEGIES = [ + 'lowest-price', + 'diversified', + 'capacity-optimized', + 'capacity-optimized-prioritized', + 'price-capacity-optimized', +]; +const ON_DEMAND_ALLOCATION_STRATEGIES = ['lowest-price', 'prioritized']; + +// The instance_allocation_strategy variable accepts the union of spot and on-demand strategies, +// so a value valid for one capacity type can be invalid for the other. AWS rejects CreateFleet +// when the strategy is not valid for the target capacity type, so fall back to 'lowest-price' +// (the AWS default) when the configured value is invalid for the given capacity type. +function sanitizeAllocationStrategy( + strategy: string, + targetCapacityType: string, +): SpotAllocationStrategy | FleetOnDemandAllocationStrategy { + const validStrategies = + targetCapacityType === 'spot' ? SPOT_ALLOCATION_STRATEGIES : ON_DEMAND_ALLOCATION_STRATEGIES; + return (validStrategies.includes(strategy) ? strategy : 'lowest-price') as + | SpotAllocationStrategy + | FleetOnDemandAllocationStrategy; +} + function generateFleetOverrides( subnetIds: string[], instancesTypes: string[], @@ -210,12 +234,10 @@ async function processFleetResult( logger.warn(`Create fleet failed, initatiing fall back to on demand instances.`); logger.debug('Create fleet failed.', { data: fleet.Errors }); const numberOfInstances = runnerParameters.numberOfRunners - instances.length; - const onDemandValidStrategies = ['lowest-price', 'prioritized']; - const failoverAllocationStrategy = onDemandValidStrategies.includes( + const failoverAllocationStrategy = sanitizeAllocationStrategy( runnerParameters.ec2instanceCriteria.instanceAllocationStrategy, - ) - ? runnerParameters.ec2instanceCriteria.instanceAllocationStrategy - : 'lowest-price'; + 'on-demand', + ); const instancesOnDemand = await createRunner({ ...runnerParameters, numberOfRunners: numberOfInstances, @@ -284,6 +306,12 @@ async function createInstances( tags.push({ Key: 'ghr:trace_id', Value: traceId! }); } + const targetCapacityType = runnerParameters.ec2instanceCriteria.targetCapacityType; + const allocationStrategy = sanitizeAllocationStrategy( + runnerParameters.ec2instanceCriteria.instanceAllocationStrategy, + targetCapacityType, + ); + let fleet: CreateFleetResult; try { // see for spec https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html @@ -299,28 +327,26 @@ async function createInstances( runnerParameters.ec2instanceCriteria.instanceTypes, amiIdOverride, runnerParameters.ec2OverrideConfig, - runnerParameters.ec2instanceCriteria.instanceAllocationStrategy, + allocationStrategy, runnerParameters.ec2instanceCriteria.instanceTypePriorities, ), }, ], - ...(runnerParameters.ec2instanceCriteria.targetCapacityType === 'spot' + ...(targetCapacityType === 'spot' ? { SpotOptions: { MaxTotalPrice: runnerParameters.ec2instanceCriteria.maxSpotPrice, - AllocationStrategy: runnerParameters.ec2instanceCriteria - .instanceAllocationStrategy as SpotAllocationStrategy, + AllocationStrategy: allocationStrategy as SpotAllocationStrategy, }, } : { OnDemandOptions: { - AllocationStrategy: runnerParameters.ec2instanceCriteria - .instanceAllocationStrategy as FleetOnDemandAllocationStrategy, + AllocationStrategy: allocationStrategy as FleetOnDemandAllocationStrategy, }, }), TargetCapacitySpecification: { TotalTargetCapacity: runnerParameters.numberOfRunners, - DefaultTargetCapacityType: runnerParameters.ec2instanceCriteria.targetCapacityType, + DefaultTargetCapacityType: targetCapacityType, }, TagSpecifications: [ { From 30ebc48e5177daab616d245fb7a15b0979b4d75c Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Thu, 11 Jun 2026 11:54:04 +0200 Subject: [PATCH 08/10] docs(runners): recommend price-capacity-optimized for spot allocation AWS now recommends price-capacity-optimized for spot fleets. Update the instance_allocation_strategy descriptions in the runners and multi-runner modules to match the root variable. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/multi-runner/variables.tf | 2 +- modules/runners/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index de7378e0b4..ab8f0f7dec 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -233,7 +233,7 @@ variable "multi_runner_config" { enable_runner_binaries_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI." enable_ssm_on_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances." enable_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI." - instance_allocation_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." + instance_allocation_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." instance_type_priorities: "A map of instance type to priority for the `prioritized` allocation strategy. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`." instance_max_spot_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet." instance_target_capacity_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`." diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index a233dd149f..9a7ffce665 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -129,7 +129,7 @@ variable "instance_target_capacity_type" { } variable "instance_allocation_strategy" { - description = "The allocation strategy for creating instances. For spot, AWS recommends `capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." + description = "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." type = string default = "lowest-price" From 28e67fc42560fc6785943fe98d86c42bf5d7a7c1 Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Thu, 11 Jun 2026 11:54:12 +0200 Subject: [PATCH 09/10] fix(runners): add prioritized to INSTANCE_ALLOCATION_STRATEGY env type The prioritized on-demand allocation strategy is now an allowed value for the instance_allocation_strategy variable and is wired into the lambda via the INSTANCE_ALLOCATION_STRATEGY env var. Add it to the declared ProcessEnv union so the type matches the values reachable at runtime. Co-Authored-By: Claude Opus 4.8 (1M context) --- lambdas/functions/control-plane/src/modules.d.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lambdas/functions/control-plane/src/modules.d.ts b/lambdas/functions/control-plane/src/modules.d.ts index ff447c0e51..0ec63317db 100644 --- a/lambdas/functions/control-plane/src/modules.d.ts +++ b/lambdas/functions/control-plane/src/modules.d.ts @@ -28,6 +28,7 @@ declare namespace NodeJS { | 'price-capacity-optimized' | 'diversified' | 'capacity-optimized' - | 'capacity-optimized-prioritized'; + | 'capacity-optimized-prioritized' + | 'prioritized'; } } From a1a2a4b3e6140f00496192760e276f75693b2840 Mon Sep 17 00:00:00 2001 From: Piotr Jakubowski Date: Thu, 11 Jun 2026 12:03:14 +0200 Subject: [PATCH 10/10] feat(runners): support priorities for capacity-optimized-prioritized spot fleets The spot capacity-optimized-prioritized strategy honors the Priority field of launch template overrides (best-effort), just like the on-demand prioritized strategy. Apply instance_type_priorities for both strategies instead of only prioritized, and update the instance_type_priorities variable descriptions accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../control-plane/src/aws/runners.test.ts | 31 ++++++++++++++++--- .../control-plane/src/aws/runners.ts | 7 ++++- modules/multi-runner/variables.tf | 2 +- modules/runners/variables.tf | 2 +- variables.tf | 2 +- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lambdas/functions/control-plane/src/aws/runners.test.ts b/lambdas/functions/control-plane/src/aws/runners.test.ts index a1735c9c22..6dbccd5d76 100644 --- a/lambdas/functions/control-plane/src/aws/runners.test.ts +++ b/lambdas/functions/control-plane/src/aws/runners.test.ts @@ -440,6 +440,26 @@ describe('create runner', () => { }); }); + it('calls create fleet with spot capacity-optimized-prioritized and instance type priorities', async () => { + const priorities = { 'm5.large': 10, 'c5.large': 5 }; + await createRunner( + createRunnerConfig({ + ...defaultRunnerConfig, + capacityType: 'spot', + allocationStrategy: SpotAllocationStrategy.CAPACITY_OPTIMIZED_PRIORITIZED, + instanceTypePriorities: priorities, + }), + ); + expect(mockEC2Client).toHaveReceivedCommandWith(CreateFleetCommand, { + ...expectedCreateFleetRequest({ + ...defaultExpectedFleetRequestValues, + capacityType: 'spot', + allocationStrategy: SpotAllocationStrategy.CAPACITY_OPTIMIZED_PRIORITIZED, + instanceTypePriorities: priorities, + }), + }); + }); + it('calls run instances with the on-demand capacity', async () => { await createRunner(createRunnerConfig({ ...defaultRunnerConfig, maxSpotPrice: '0.1' })); expect(mockEC2Client).toHaveReceivedCommandWith(CreateFleetCommand, { @@ -1062,6 +1082,9 @@ function expectedCreateFleetRequest(expectedValues: ExpectedFleetRequestValues): const traceId = tracer.getRootXrayTraceId(); tags.push({ Key: 'ghr:trace_id', Value: traceId! }); } + const usesPriority = + expectedValues.allocationStrategy === 'prioritized' || + expectedValues.allocationStrategy === 'capacity-optimized-prioritized'; const request: CreateFleetCommandInput = { LaunchTemplateConfigs: [ { @@ -1073,28 +1096,28 @@ function expectedCreateFleetRequest(expectedValues: ExpectedFleetRequestValues): { InstanceType: 'm5.large', SubnetId: 'subnet-123', - ...(expectedValues.allocationStrategy === 'prioritized' && { + ...(usesPriority && { Priority: expectedValues.instanceTypePriorities?.['m5.large'] ?? 0, }), }, { InstanceType: 'c5.large', SubnetId: 'subnet-123', - ...(expectedValues.allocationStrategy === 'prioritized' && { + ...(usesPriority && { Priority: expectedValues.instanceTypePriorities?.['c5.large'] ?? 1, }), }, { InstanceType: 'm5.large', SubnetId: 'subnet-456', - ...(expectedValues.allocationStrategy === 'prioritized' && { + ...(usesPriority && { Priority: expectedValues.instanceTypePriorities?.['m5.large'] ?? 0, }), }, { InstanceType: 'c5.large', SubnetId: 'subnet-456', - ...(expectedValues.allocationStrategy === 'prioritized' && { + ...(usesPriority && { Priority: expectedValues.instanceTypePriorities?.['c5.large'] ?? 1, }), }, diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index 254496dd36..b04286e06b 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -163,13 +163,18 @@ function generateFleetOverrides( const instanceTypesToUse = ec2OverrideConfig?.InstanceType ? [ec2OverrideConfig.InstanceType] : instancesTypes; const amiIdToUse = ec2OverrideConfig?.ImageId ?? amiId; + // Both the on-demand 'prioritized' and the spot 'capacity-optimized-prioritized' strategies + // honor the Priority field of the launch template overrides. + const usesPriority = + allocationStrategy === 'prioritized' || allocationStrategy === 'capacity-optimized-prioritized'; + subnetsToUse.forEach((s) => { instanceTypesToUse.forEach((i, index) => { const item: FleetLaunchTemplateOverridesRequest = { SubnetId: s, InstanceType: i as _InstanceType, ImageId: amiIdToUse, - ...(allocationStrategy === 'prioritized' && { Priority: instanceTypePriorities?.[i] ?? index }), + ...(usesPriority && { Priority: instanceTypePriorities?.[i] ?? index }), ...ec2OverrideConfig, }; result.push(item); diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index ab8f0f7dec..606271d32d 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -234,7 +234,7 @@ variable "multi_runner_config" { enable_ssm_on_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances." enable_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI." instance_allocation_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`." - instance_type_priorities: "A map of instance type to priority for the `prioritized` allocation strategy. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`." + instance_type_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`." instance_max_spot_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet." instance_target_capacity_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`." instance_types: "List of instance types for the action runner. Defaults are based on runner_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)." diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 9a7ffce665..4ac5687c04 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -140,7 +140,7 @@ variable "instance_allocation_strategy" { } variable "instance_type_priorities" { - description = "A map of instance type to priority for the `prioritized` allocation strategy. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`." + description = "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`." type = map(number) default = null } diff --git a/variables.tf b/variables.tf index c64735ab7e..bf3a874c04 100644 --- a/variables.tf +++ b/variables.tf @@ -591,7 +591,7 @@ variable "instance_allocation_strategy" { } variable "instance_type_priorities" { - description = "A map of instance type to priority for the `prioritized` allocation strategy. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`." + description = "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`." type = map(number) default = null }