340 lines
10 KiB
Nix
340 lines
10 KiB
Nix
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.nix-builder-autoscaler;
|
|
defaultAutoscalerPackage =
|
|
if builtins.hasAttr "nix-builder-autoscaler" pkgs then pkgs."nix-builder-autoscaler" else null;
|
|
generatedConfigPath = "/run/nix-builder-autoscaler/config.toml";
|
|
tomlStringList = values: "[${lib.concatMapStringsSep ", " (value: ''"${value}"'') values}]";
|
|
in
|
|
{
|
|
options.services.nix-builder-autoscaler = {
|
|
enable = lib.mkEnableOption "nix-builder-autoscaler daemon";
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.package;
|
|
default = defaultAutoscalerPackage;
|
|
description = "Package providing nix_builder_autoscaler.";
|
|
};
|
|
|
|
user = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "buildbot";
|
|
description = "User account for the autoscaler daemon.";
|
|
};
|
|
|
|
group = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "buildbot";
|
|
description = "Group for the autoscaler daemon.";
|
|
};
|
|
|
|
supplementaryGroups = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "haproxy" ];
|
|
description = "Supplementary groups for the autoscaler daemon.";
|
|
};
|
|
|
|
socketPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/run/nix-builder-autoscaler/daemon.sock";
|
|
description = "Unix socket path exposed by the autoscaler API server.";
|
|
};
|
|
|
|
logLevel = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "info";
|
|
description = "Daemon log level.";
|
|
};
|
|
|
|
dbPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/var/lib/nix-builder-autoscaler/state.db";
|
|
description = "SQLite database path.";
|
|
};
|
|
|
|
aws = {
|
|
region = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "AWS region for EC2 launches.";
|
|
};
|
|
|
|
launchTemplateIdFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "Runtime file containing the EC2 launch template ID.";
|
|
};
|
|
|
|
subnetIdsJsonFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "Runtime file containing JSON list of subnet IDs.";
|
|
};
|
|
|
|
securityGroupIds = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
description = "Static security group IDs used by the launch template.";
|
|
};
|
|
|
|
instanceProfileArn = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "";
|
|
description = "Optional instance profile ARN override.";
|
|
};
|
|
};
|
|
|
|
haproxy = {
|
|
generateConfig = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
description = "Whether this module manages HAProxy static slot configuration.";
|
|
};
|
|
|
|
runtimeSocket = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/run/haproxy/admin.sock";
|
|
description = "HAProxy admin runtime socket path.";
|
|
};
|
|
|
|
backend = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "all";
|
|
description = "HAProxy backend name used for static builder slots.";
|
|
};
|
|
|
|
slotPrefix = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "slot";
|
|
description = "Slot name prefix in HAProxy backend.";
|
|
};
|
|
|
|
slotCount = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 8;
|
|
description = "Number of static HAProxy slots.";
|
|
};
|
|
|
|
listenPort = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 2222;
|
|
description = "HAProxy frontend port for nix remote builders.";
|
|
};
|
|
|
|
checkReadyUpCount = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 2;
|
|
description = "Consecutive HAProxy UP checks required before slot becomes ready.";
|
|
};
|
|
};
|
|
|
|
capacity = {
|
|
defaultSystem = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "x86_64-linux";
|
|
description = "Default reservation system.";
|
|
};
|
|
|
|
minSlots = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 0;
|
|
description = "Minimum active slots.";
|
|
};
|
|
|
|
maxSlots = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 8;
|
|
description = "Maximum active slots.";
|
|
};
|
|
|
|
targetWarmSlots = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 0;
|
|
description = "Target number of warm slots.";
|
|
};
|
|
|
|
maxLeasesPerSlot = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 1;
|
|
description = "Maximum concurrent leases per slot.";
|
|
};
|
|
|
|
reservationTtlSeconds = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 1200;
|
|
description = "Reservation TTL in seconds.";
|
|
};
|
|
|
|
idleScaleDownSeconds = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 900;
|
|
description = "Idle seconds before draining a ready slot.";
|
|
};
|
|
|
|
drainTimeoutSeconds = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 120;
|
|
description = "Drain timeout before force termination.";
|
|
};
|
|
|
|
launchBatchSize = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 1;
|
|
description = "Launch batch size for the default system entry.";
|
|
};
|
|
};
|
|
|
|
security = {
|
|
socketMode = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "0660";
|
|
description = "API socket mode in daemon config.";
|
|
};
|
|
|
|
socketOwner = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "buildbot";
|
|
description = "API socket owner in daemon config.";
|
|
};
|
|
|
|
socketGroup = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "buildbot";
|
|
description = "API socket group in daemon config.";
|
|
};
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = cfg.package != null;
|
|
message = ''
|
|
services.nix-builder-autoscaler.package is not set and pkgs.nix-builder-autoscaler
|
|
was not found. Configure package explicitly.
|
|
'';
|
|
}
|
|
{
|
|
assertion = cfg.aws.launchTemplateIdFile != null;
|
|
message = "services.nix-builder-autoscaler.aws.launchTemplateIdFile must be set.";
|
|
}
|
|
{
|
|
assertion = cfg.aws.subnetIdsJsonFile != null;
|
|
message = "services.nix-builder-autoscaler.aws.subnetIdsJsonFile must be set.";
|
|
}
|
|
];
|
|
|
|
services.haproxy = lib.mkIf cfg.haproxy.generateConfig {
|
|
enable = true;
|
|
config = ''
|
|
global
|
|
stats socket ${cfg.haproxy.runtimeSocket} mode 660 level admin expose-fd listeners
|
|
stats timeout 2m
|
|
|
|
defaults
|
|
mode tcp
|
|
timeout connect 10s
|
|
timeout client 36h
|
|
timeout server 36h
|
|
timeout queue 30s
|
|
|
|
frontend nix-builders
|
|
bind *:${toString cfg.haproxy.listenPort}
|
|
default_backend ${cfg.haproxy.backend}
|
|
|
|
backend ${cfg.haproxy.backend}
|
|
balance leastconn
|
|
option tcp-check
|
|
tcp-check expect rstring SSH-2\\.0-OpenSSH.*
|
|
${lib.concatMapStrings (
|
|
i:
|
|
"server ${cfg.haproxy.slotPrefix}${lib.fixedWidthNumber 3 i} 127.0.0.2:22 disabled check inter 5s fall 2 rise 2 maxconn 2\n "
|
|
) (lib.range 1 cfg.haproxy.slotCount)}
|
|
'';
|
|
};
|
|
|
|
systemd.services.nix-builder-autoscaler = {
|
|
description = "Nix builder autoscaler daemon";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-online.target" ];
|
|
after = [
|
|
"network-online.target"
|
|
]
|
|
++ lib.optionals cfg.haproxy.generateConfig [ "haproxy.service" ];
|
|
preStart = ''
|
|
install -d -m 0750 -o ${cfg.user} -g ${cfg.group} /run/nix-builder-autoscaler
|
|
launch_template_id="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.launchTemplateIdFile})"
|
|
subnet_ids_json="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.subnetIdsJsonFile})"
|
|
|
|
cat > ${generatedConfigPath} <<EOF
|
|
[server]
|
|
socket_path = "${cfg.socketPath}"
|
|
log_level = "${cfg.logLevel}"
|
|
db_path = "${cfg.dbPath}"
|
|
|
|
[aws]
|
|
region = "${cfg.aws.region}"
|
|
launch_template_id = "$launch_template_id"
|
|
subnet_ids = $subnet_ids_json
|
|
security_group_ids = ${tomlStringList cfg.aws.securityGroupIds}
|
|
instance_profile_arn = "${cfg.aws.instanceProfileArn}"
|
|
|
|
[haproxy]
|
|
runtime_socket = "${cfg.haproxy.runtimeSocket}"
|
|
backend = "${cfg.haproxy.backend}"
|
|
slot_prefix = "${cfg.haproxy.slotPrefix}"
|
|
slot_count = ${toString cfg.haproxy.slotCount}
|
|
check_ready_up_count = ${toString cfg.haproxy.checkReadyUpCount}
|
|
|
|
[capacity]
|
|
default_system = "${cfg.capacity.defaultSystem}"
|
|
min_slots = ${toString cfg.capacity.minSlots}
|
|
max_slots = ${toString cfg.capacity.maxSlots}
|
|
target_warm_slots = ${toString cfg.capacity.targetWarmSlots}
|
|
max_leases_per_slot = ${toString cfg.capacity.maxLeasesPerSlot}
|
|
reservation_ttl_seconds = ${toString cfg.capacity.reservationTtlSeconds}
|
|
idle_scale_down_seconds = ${toString cfg.capacity.idleScaleDownSeconds}
|
|
drain_timeout_seconds = ${toString cfg.capacity.drainTimeoutSeconds}
|
|
|
|
[security]
|
|
socket_mode = "${cfg.security.socketMode}"
|
|
socket_owner = "${cfg.security.socketOwner}"
|
|
socket_group = "${cfg.security.socketGroup}"
|
|
|
|
[[systems]]
|
|
name = "${cfg.capacity.defaultSystem}"
|
|
min_slots = ${toString cfg.capacity.minSlots}
|
|
max_slots = ${toString cfg.capacity.maxSlots}
|
|
target_warm_slots = ${toString cfg.capacity.targetWarmSlots}
|
|
max_leases_per_slot = ${toString cfg.capacity.maxLeasesPerSlot}
|
|
launch_batch_size = ${toString cfg.capacity.launchBatchSize}
|
|
scale_down_idle_seconds = ${toString cfg.capacity.idleScaleDownSeconds}
|
|
EOF
|
|
|
|
chown ${cfg.user}:${cfg.group} ${generatedConfigPath}
|
|
chmod 0640 ${generatedConfigPath}
|
|
'';
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
User = cfg.user;
|
|
Group = cfg.group;
|
|
SupplementaryGroups = cfg.supplementaryGroups;
|
|
ExecStart = "${cfg.package}/bin/python -m nix_builder_autoscaler --config ${generatedConfigPath}";
|
|
Restart = "always";
|
|
RestartSec = 2;
|
|
RuntimeDirectory = "nix-builder-autoscaler";
|
|
RuntimeDirectoryMode = "0750";
|
|
StateDirectory = "nix-builder-autoscaler";
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
ReadWritePaths = [ (builtins.dirOf cfg.dbPath) ];
|
|
};
|
|
};
|
|
};
|
|
}
|