From 93749a66a9b8fd70d17fd31a9a01faa52f4a3aae Mon Sep 17 00:00:00 2001 From: Abel Luck Date: Fri, 27 Feb 2026 14:46:52 +0100 Subject: [PATCH] add nixos modules --- flake.nix | 6 + nix/modules/nixos/default.nix | 6 + .../services/buildbot-nix-autoscaler.nix | 187 ++++++++++ .../nixos/services/nix-builder-autoscaler.nix | 331 ++++++++++++++++++ 4 files changed, 530 insertions(+) create mode 100644 nix/modules/nixos/default.nix create mode 100644 nix/modules/nixos/services/buildbot-nix-autoscaler.nix create mode 100644 nix/modules/nixos/services/nix-builder-autoscaler.nix diff --git a/flake.nix b/flake.nix index fb3f1f4..a53811b 100644 --- a/flake.nix +++ b/flake.nix @@ -54,6 +54,12 @@ pkgs: (treefmt-nix.lib.evalModule pkgs ./treefmt.nix).config.build.wrapper ); + nixosModules = { + default = import ./nix/modules/nixos; + nix-builder-autoscaler = import ./nix/modules/nixos/services/nix-builder-autoscaler.nix; + buildbot-nix-autoscaler = import ./nix/modules/nixos/services/buildbot-nix-autoscaler.nix; + }; + packages = forAllSystems ( pkgs: let diff --git a/nix/modules/nixos/default.nix b/nix/modules/nixos/default.nix new file mode 100644 index 0000000..ec057fe --- /dev/null +++ b/nix/modules/nixos/default.nix @@ -0,0 +1,6 @@ +{ + imports = [ + ./services/nix-builder-autoscaler.nix + ./services/buildbot-nix-autoscaler.nix + ]; +} diff --git a/nix/modules/nixos/services/buildbot-nix-autoscaler.nix b/nix/modules/nixos/services/buildbot-nix-autoscaler.nix new file mode 100644 index 0000000..16a9ccd --- /dev/null +++ b/nix/modules/nixos/services/buildbot-nix-autoscaler.nix @@ -0,0 +1,187 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.buildbot-nix.nix-build-autoscaler; +in +{ + options.services.buildbot-nix.nix-build-autoscaler = { + enable = lib.mkEnableOption "buildbot-nix autoscaler gate integration"; + + extensionPackage = lib.mkOption { + type = lib.types.package; + default = pkgs.buildbot-autoscale-ext; + description = "Package providing buildbot_autoscale_ext."; + }; + + daemonSocket = lib.mkOption { + type = lib.types.str; + default = "/run/nix-builder-autoscaler/daemon.sock"; + description = "Autoscaler daemon Unix socket path for Buildbot gate/release steps."; + }; + + defaultSystem = lib.mkOption { + type = lib.types.str; + default = "x86_64-linux"; + description = "Default reservation system when build property is absent."; + }; + + reserveTimeoutSeconds = lib.mkOption { + type = lib.types.int; + default = 1200; + description = "Seconds CapacityGateStep waits for a ready reservation."; + }; + + pollIntervalSeconds = lib.mkOption { + type = lib.types.float; + default = 5.0; + description = "Reservation poll interval."; + }; + + retryMaxAttempts = lib.mkOption { + type = lib.types.int; + default = 5; + description = "Maximum daemon API retry attempts per request."; + }; + + retryBaseSeconds = lib.mkOption { + type = lib.types.float; + default = 0.5; + description = "Base retry backoff seconds."; + }; + + retryMaxSeconds = lib.mkOption { + type = lib.types.float; + default = 5.0; + description = "Max retry backoff seconds."; + }; + + releaseOnFinish = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Append CapacityReleaseStep to patched nix-build builders."; + }; + + clusterAlias = lib.mkOption { + type = lib.types.str; + default = "cluster"; + description = "SSH host alias used by nix buildMachines."; + }; + + builderClusterHost = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "SSH hostname for the HAProxy-backed builder cluster endpoint."; + }; + + clusterSshPort = lib.mkOption { + type = lib.types.int; + default = 2222; + description = "SSH port for the HAProxy-backed builder cluster endpoint."; + }; + + clusterSshUser = lib.mkOption { + type = lib.types.str; + default = "builder-ssh"; + description = "SSH user used by nix-daemon for remote builders."; + }; + + builderSshKeyFile = lib.mkOption { + type = lib.types.str; + default = "/var/lib/buildbot-worker/.ssh/id_ed25519"; + description = "SSH private key used by nix-daemon for cluster connections."; + }; + + systems = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "x86_64-linux" ]; + description = "Nix systems served by remote builder cluster."; + }; + + maxJobs = lib.mkOption { + type = lib.types.int; + default = 32; + description = "Max jobs for the buildMachines entry."; + }; + + speedFactor = lib.mkOption { + type = lib.types.int; + default = 1; + description = "Nix speedFactor for the cluster build machine."; + }; + + supportedFeatures = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "kvm" + "big-parallel" + ]; + description = "Nix supportedFeatures for cluster builders."; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.builderClusterHost != null; + message = "services.buildbot-nix.nix-build-autoscaler.builderClusterHost must be set."; + } + ]; + + services.buildbot-master.pythonPackages = _ps: [ cfg.extensionPackage ]; + + services.buildbot-master.extraImports = '' + from buildbot_autoscale_ext.configurator import AutoscaleConfigurator + from buildbot_autoscale_ext.settings import AutoscaleSettings + ''; + + services.buildbot-master.configurators = [ + '' + AutoscaleConfigurator( + AutoscaleSettings( + daemon_socket="${cfg.daemonSocket}", + default_system="${cfg.defaultSystem}", + reserve_timeout_seconds=${toString cfg.reserveTimeoutSeconds}, + poll_interval_seconds=${toString cfg.pollIntervalSeconds}, + retry_max_attempts=${toString cfg.retryMaxAttempts}, + retry_base_seconds=${toString cfg.retryBaseSeconds}, + retry_max_seconds=${toString cfg.retryMaxSeconds}, + release_on_finish=${if cfg.releaseOnFinish then "True" else "False"}, + ) + ) + '' + ]; + + nix = { + distributedBuilds = true; + settings.max-jobs = 0; + settings.builders-use-substitutes = true; + buildMachines = [ + { + hostName = cfg.clusterAlias; + protocol = "ssh-ng"; + sshUser = cfg.clusterSshUser; + sshKey = cfg.builderSshKeyFile; + systems = cfg.systems; + maxJobs = cfg.maxJobs; + speedFactor = cfg.speedFactor; + inherit (cfg) supportedFeatures; + } + ]; + }; + + programs.ssh.extraConfig = '' + Host ${cfg.clusterAlias} + HostName ${cfg.builderClusterHost} + Port ${toString cfg.clusterSshPort} + HostKeyAlias ${cfg.clusterAlias} + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + GlobalKnownHostsFile /dev/null + CheckHostIP no + ''; + }; +} diff --git a/nix/modules/nixos/services/nix-builder-autoscaler.nix b/nix/modules/nixos/services/nix-builder-autoscaler.nix new file mode 100644 index 0000000..5ebd217 --- /dev/null +++ b/nix/modules/nixos/services/nix-builder-autoscaler.nix @@ -0,0 +1,331 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.nix-builder-autoscaler; + generatedConfigPath = "/run/nix-builder-autoscaler/config.toml"; + tomlStringList = values: "[${lib.concatMapStringsSep ", " (value: ''"${value}"'') values}]"; +in +{ + options.services.nix-builder-autoscaler = { + enable = lib.mkEnableOption "nix-builder-autoscaler daemon"; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.nix-builder-autoscaler; + description = "Package providing nix_builder_autoscaler."; + }; + + user = lib.mkOption { + type = lib.types.str; + default = "buildbot"; + description = "User account for the autoscaler daemon."; + }; + + group = lib.mkOption { + type = lib.types.str; + default = "buildbot"; + description = "Group for the autoscaler daemon."; + }; + + supplementaryGroups = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "haproxy" ]; + description = "Supplementary groups for the autoscaler daemon."; + }; + + socketPath = lib.mkOption { + type = lib.types.str; + default = "/run/nix-builder-autoscaler/daemon.sock"; + description = "Unix socket path exposed by the autoscaler API server."; + }; + + logLevel = lib.mkOption { + type = lib.types.str; + default = "info"; + description = "Daemon log level."; + }; + + dbPath = lib.mkOption { + type = lib.types.str; + default = "/var/lib/nix-builder-autoscaler/state.db"; + description = "SQLite database path."; + }; + + aws = { + region = lib.mkOption { + type = lib.types.str; + description = "AWS region for EC2 launches."; + }; + + launchTemplateIdFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Runtime file containing the EC2 launch template ID."; + }; + + subnetIdsJsonFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Runtime file containing JSON list of subnet IDs."; + }; + + securityGroupIds = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Static security group IDs used by the launch template."; + }; + + instanceProfileArn = lib.mkOption { + type = lib.types.str; + default = ""; + description = "Optional instance profile ARN override."; + }; + }; + + haproxy = { + generateConfig = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether this module manages HAProxy static slot configuration."; + }; + + runtimeSocket = lib.mkOption { + type = lib.types.str; + default = "/run/haproxy/admin.sock"; + description = "HAProxy admin runtime socket path."; + }; + + backend = lib.mkOption { + type = lib.types.str; + default = "all"; + description = "HAProxy backend name used for static builder slots."; + }; + + slotPrefix = lib.mkOption { + type = lib.types.str; + default = "slot"; + description = "Slot name prefix in HAProxy backend."; + }; + + slotCount = lib.mkOption { + type = lib.types.int; + default = 8; + description = "Number of static HAProxy slots."; + }; + + listenPort = lib.mkOption { + type = lib.types.int; + default = 2222; + description = "HAProxy frontend port for nix remote builders."; + }; + + checkReadyUpCount = lib.mkOption { + type = lib.types.int; + default = 2; + description = "Consecutive HAProxy UP checks required before slot becomes ready."; + }; + }; + + capacity = { + defaultSystem = lib.mkOption { + type = lib.types.str; + default = "x86_64-linux"; + description = "Default reservation system."; + }; + + minSlots = lib.mkOption { + type = lib.types.int; + default = 0; + description = "Minimum active slots."; + }; + + maxSlots = lib.mkOption { + type = lib.types.int; + default = 8; + description = "Maximum active slots."; + }; + + targetWarmSlots = lib.mkOption { + type = lib.types.int; + default = 0; + description = "Target number of warm slots."; + }; + + maxLeasesPerSlot = lib.mkOption { + type = lib.types.int; + default = 1; + description = "Maximum concurrent leases per slot."; + }; + + reservationTtlSeconds = lib.mkOption { + type = lib.types.int; + default = 1200; + description = "Reservation TTL in seconds."; + }; + + idleScaleDownSeconds = lib.mkOption { + type = lib.types.int; + default = 900; + description = "Idle seconds before draining a ready slot."; + }; + + drainTimeoutSeconds = lib.mkOption { + type = lib.types.int; + default = 120; + description = "Drain timeout before force termination."; + }; + + launchBatchSize = lib.mkOption { + type = lib.types.int; + default = 1; + description = "Launch batch size for the default system entry."; + }; + }; + + security = { + socketMode = lib.mkOption { + type = lib.types.str; + default = "0660"; + description = "API socket mode in daemon config."; + }; + + socketOwner = lib.mkOption { + type = lib.types.str; + default = "buildbot"; + description = "API socket owner in daemon config."; + }; + + socketGroup = lib.mkOption { + type = lib.types.str; + default = "buildbot"; + description = "API socket group in daemon config."; + }; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.aws.launchTemplateIdFile != null; + message = "services.nix-builder-autoscaler.aws.launchTemplateIdFile must be set."; + } + { + assertion = cfg.aws.subnetIdsJsonFile != null; + message = "services.nix-builder-autoscaler.aws.subnetIdsJsonFile must be set."; + } + ]; + + services.haproxy = lib.mkIf cfg.haproxy.generateConfig { + enable = true; + config = '' + global + stats socket ${cfg.haproxy.runtimeSocket} mode 660 level admin expose-fd listeners + stats timeout 2m + + defaults + mode tcp + timeout connect 10s + timeout client 36h + timeout server 36h + timeout queue 30s + + frontend nix-builders + bind *:${toString cfg.haproxy.listenPort} + default_backend ${cfg.haproxy.backend} + + backend ${cfg.haproxy.backend} + balance leastconn + option tcp-check + tcp-check expect rstring SSH-2\\.0-OpenSSH.* + ${lib.concatMapStrings ( + i: + "server ${cfg.haproxy.slotPrefix}${lib.fixedWidthNumber 3 i} 127.0.0.2:22 disabled check inter 5s fall 2 rise 2 maxconn 2\n " + ) (lib.range 1 cfg.haproxy.slotCount)} + ''; + }; + + systemd.services.nix-builder-autoscaler = { + description = "Nix builder autoscaler daemon"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ + "network-online.target" + ] + ++ lib.optionals cfg.haproxy.generateConfig [ "haproxy.service" ]; + preStart = '' + install -d -m 0750 -o ${cfg.user} -g ${cfg.group} /run/nix-builder-autoscaler + launch_template_id="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.launchTemplateIdFile})" + subnet_ids_json="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.subnetIdsJsonFile})" + + cat > ${generatedConfigPath} <