support dual launch templates: spot for normal builds, on-demand for nested virtualization
Some checks failed
buildbot/nix-eval Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.package-nix-builder-autoscaler Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.package-default Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.app-autoscalerctl Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.app-default Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.app-nix-builder-autoscaler Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.nix-builder-autoscaler-pyright Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.nix-builder-autoscaler-integration-tests Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.nix-builder-autoscaler-ruff Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.nix-builder-autoscaler-unit-tests Build done.
buildbot/nix-build gitea:ops/nix-builder-autoscaler#checks.x86_64-linux.package-buildbot-autoscale-ext Build done.
buildbot/nix-build Build done.

AWS does not allow cpu_options.nested_virtualization with spot instances. Add a second
launch template (on-demand, cpu_options enabled) alongside the existing spot template.
The autoscaler selects the template per-system based on nested_virtualization config.

- RuntimeAdapter.launch_spot -> launch_instance(nested_virtualization=False)
- EC2Runtime: selects spot or on-demand LT; raises misconfiguration error if
  on_demand_launch_template_id is empty when nested_virtualization=True
- AwsConfig: add on_demand_launch_template_id field
- SystemConfig: add nested_virtualization field
- Scheduler: looks up system config to pass nested_virtualization flag
- NixOS module: new aws.onDemandLaunchTemplateIdFile + capacity.nestedVirtualization
  options; assertion prevents enabling nestedVirtualization without the LT ID file
This commit is contained in:
Abel Luck 2026-02-28 10:33:26 +01:00
parent 3f70094c0a
commit 02b1a063ab
9 changed files with 101 additions and 35 deletions

View file

@ -66,7 +66,13 @@ in
launchTemplateIdFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Runtime file containing the EC2 launch template ID.";
description = "Runtime file containing the EC2 spot launch template ID.";
};
onDemandLaunchTemplateIdFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Runtime file containing the EC2 on-demand launch template ID (required when capacity.nestedVirtualization is true).";
};
subnetIdsJsonFile = lib.mkOption {
@ -216,6 +222,12 @@ in
default = 1;
description = "Launch batch size for the default system entry.";
};
nestedVirtualization = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Whether slots use on-demand instances with nested virtualization. Requires aws.onDemandLaunchTemplateIdFile to be set.";
};
};
security = {
@ -256,6 +268,10 @@ in
assertion = cfg.aws.subnetIdsJsonFile != null;
message = "services.nix-builder-autoscaler.aws.subnetIdsJsonFile must be set.";
}
{
assertion = !cfg.capacity.nestedVirtualization || cfg.aws.onDemandLaunchTemplateIdFile != null;
message = "services.nix-builder-autoscaler.aws.onDemandLaunchTemplateIdFile must be set when capacity.nestedVirtualization is true.";
}
];
environment.systemPackages = [ cfg.package ];
@ -301,6 +317,9 @@ in
install -d -m 0750 -o ${cfg.user} -g ${cfg.group} /run/nix-builder-autoscaler
launch_template_id="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.launchTemplateIdFile})"
subnet_ids_json="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.subnetIdsJsonFile})"
${lib.optionalString (cfg.aws.onDemandLaunchTemplateIdFile != null) ''
on_demand_launch_template_id="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.onDemandLaunchTemplateIdFile})"
''}
cat > ${generatedConfigPath} <<EOF
[server]
@ -311,6 +330,9 @@ in
[aws]
region = "${cfg.aws.region}"
launch_template_id = "$launch_template_id"
${lib.optionalString (
cfg.aws.onDemandLaunchTemplateIdFile != null
) ''on_demand_launch_template_id = "$on_demand_launch_template_id"''}
subnet_ids = $subnet_ids_json
security_group_ids = ${tomlStringList cfg.aws.securityGroupIds}
instance_profile_arn = "${cfg.aws.instanceProfileArn}"
@ -351,6 +373,7 @@ in
launch_batch_size = ${toString cfg.capacity.launchBatchSize}
scale_down_idle_seconds = ${toString cfg.capacity.idleScaleDownSeconds}
termination_cooldown_seconds = ${toString cfg.capacity.terminationCooldownSeconds}
nested_virtualization = ${lib.boolToString cfg.capacity.nestedVirtualization}
EOF
chown ${cfg.user}:${cfg.group} ${generatedConfigPath}