add nixos modules

flake.nix

@@ -54,6 +54,12 @@
         pkgs: (treefmt-nix.lib.evalModule pkgs ./treefmt.nix).config.build.wrapper
       );
 
+      nixosModules = {
+        default = import ./nix/modules/nixos;
+        nix-builder-autoscaler = import ./nix/modules/nixos/services/nix-builder-autoscaler.nix;
+        buildbot-nix-autoscaler = import ./nix/modules/nixos/services/buildbot-nix-autoscaler.nix;
+      };
+
       packages = forAllSystems (
         pkgs:
         let

nix/modules/nixos/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./services/nix-builder-autoscaler.nix
+    ./services/buildbot-nix-autoscaler.nix
+  ];
+}

nix/modules/nixos/services/buildbot-nix-autoscaler.nix

@@ -0,0 +1,187 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.buildbot-nix.nix-build-autoscaler;
+in
+{
+  options.services.buildbot-nix.nix-build-autoscaler = {
+    enable = lib.mkEnableOption "buildbot-nix autoscaler gate integration";
+
+    extensionPackage = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.buildbot-autoscale-ext;
+      description = "Package providing buildbot_autoscale_ext.";
+    };
+
+    daemonSocket = lib.mkOption {
+      type = lib.types.str;
+      default = "/run/nix-builder-autoscaler/daemon.sock";
+      description = "Autoscaler daemon Unix socket path for Buildbot gate/release steps.";
+    };
+
+    defaultSystem = lib.mkOption {
+      type = lib.types.str;
+      default = "x86_64-linux";
+      description = "Default reservation system when build property is absent.";
+    };
+
+    reserveTimeoutSeconds = lib.mkOption {
+      type = lib.types.int;
+      default = 1200;
+      description = "Seconds CapacityGateStep waits for a ready reservation.";
+    };
+
+    pollIntervalSeconds = lib.mkOption {
+      type = lib.types.float;
+      default = 5.0;
+      description = "Reservation poll interval.";
+    };
+
+    retryMaxAttempts = lib.mkOption {
+      type = lib.types.int;
+      default = 5;
+      description = "Maximum daemon API retry attempts per request.";
+    };
+
+    retryBaseSeconds = lib.mkOption {
+      type = lib.types.float;
+      default = 0.5;
+      description = "Base retry backoff seconds.";
+    };
+
+    retryMaxSeconds = lib.mkOption {
+      type = lib.types.float;
+      default = 5.0;
+      description = "Max retry backoff seconds.";
+    };
+
+    releaseOnFinish = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Append CapacityReleaseStep to patched nix-build builders.";
+    };
+
+    clusterAlias = lib.mkOption {
+      type = lib.types.str;
+      default = "cluster";
+      description = "SSH host alias used by nix buildMachines.";
+    };
+
+    builderClusterHost = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "SSH hostname for the HAProxy-backed builder cluster endpoint.";
+    };
+
+    clusterSshPort = lib.mkOption {
+      type = lib.types.int;
+      default = 2222;
+      description = "SSH port for the HAProxy-backed builder cluster endpoint.";
+    };
+
+    clusterSshUser = lib.mkOption {
+      type = lib.types.str;
+      default = "builder-ssh";
+      description = "SSH user used by nix-daemon for remote builders.";
+    };
+
+    builderSshKeyFile = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/buildbot-worker/.ssh/id_ed25519";
+      description = "SSH private key used by nix-daemon for cluster connections.";
+    };
+
+    systems = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ "x86_64-linux" ];
+      description = "Nix systems served by remote builder cluster.";
+    };
+
+    maxJobs = lib.mkOption {
+      type = lib.types.int;
+      default = 32;
+      description = "Max jobs for the buildMachines entry.";
+    };
+
+    speedFactor = lib.mkOption {
+      type = lib.types.int;
+      default = 1;
+      description = "Nix speedFactor for the cluster build machine.";
+    };
+
+    supportedFeatures = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [
+        "kvm"
+        "big-parallel"
+      ];
+      description = "Nix supportedFeatures for cluster builders.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.builderClusterHost != null;
+        message = "services.buildbot-nix.nix-build-autoscaler.builderClusterHost must be set.";
+      }
+    ];
+
+    services.buildbot-master.pythonPackages = _ps: [ cfg.extensionPackage ];
+
+    services.buildbot-master.extraImports = ''
+      from buildbot_autoscale_ext.configurator import AutoscaleConfigurator
+      from buildbot_autoscale_ext.settings import AutoscaleSettings
+    '';
+
+    services.buildbot-master.configurators = [
+      ''
+        AutoscaleConfigurator(
+          AutoscaleSettings(
+            daemon_socket="${cfg.daemonSocket}",
+            default_system="${cfg.defaultSystem}",
+            reserve_timeout_seconds=${toString cfg.reserveTimeoutSeconds},
+            poll_interval_seconds=${toString cfg.pollIntervalSeconds},
+            retry_max_attempts=${toString cfg.retryMaxAttempts},
+            retry_base_seconds=${toString cfg.retryBaseSeconds},
+            retry_max_seconds=${toString cfg.retryMaxSeconds},
+            release_on_finish=${if cfg.releaseOnFinish then "True" else "False"},
+          )
+        )
+      ''
+    ];
+
+    nix = {
+      distributedBuilds = true;
+      settings.max-jobs = 0;
+      settings.builders-use-substitutes = true;
+      buildMachines = [
+        {
+          hostName = cfg.clusterAlias;
+          protocol = "ssh-ng";
+          sshUser = cfg.clusterSshUser;
+          sshKey = cfg.builderSshKeyFile;
+          systems = cfg.systems;
+          maxJobs = cfg.maxJobs;
+          speedFactor = cfg.speedFactor;
+          inherit (cfg) supportedFeatures;
+        }
+      ];
+    };
+
+    programs.ssh.extraConfig = ''
+      Host ${cfg.clusterAlias}
+        HostName ${cfg.builderClusterHost}
+        Port ${toString cfg.clusterSshPort}
+        HostKeyAlias ${cfg.clusterAlias}
+        StrictHostKeyChecking no
+        UserKnownHostsFile /dev/null
+        GlobalKnownHostsFile /dev/null
+        CheckHostIP no
+    '';
+  };
+}

nix/modules/nixos/services/nix-builder-autoscaler.nix

@@ -0,0 +1,331 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.nix-builder-autoscaler;
+  generatedConfigPath = "/run/nix-builder-autoscaler/config.toml";
+  tomlStringList = values: "[${lib.concatMapStringsSep ", " (value: ''"${value}"'') values}]";
+in
+{
+  options.services.nix-builder-autoscaler = {
+    enable = lib.mkEnableOption "nix-builder-autoscaler daemon";
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.nix-builder-autoscaler;
+      description = "Package providing nix_builder_autoscaler.";
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "buildbot";
+      description = "User account for the autoscaler daemon.";
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "buildbot";
+      description = "Group for the autoscaler daemon.";
+    };
+
+    supplementaryGroups = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ "haproxy" ];
+      description = "Supplementary groups for the autoscaler daemon.";
+    };
+
+    socketPath = lib.mkOption {
+      type = lib.types.str;
+      default = "/run/nix-builder-autoscaler/daemon.sock";
+      description = "Unix socket path exposed by the autoscaler API server.";
+    };
+
+    logLevel = lib.mkOption {
+      type = lib.types.str;
+      default = "info";
+      description = "Daemon log level.";
+    };
+
+    dbPath = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/nix-builder-autoscaler/state.db";
+      description = "SQLite database path.";
+    };
+
+    aws = {
+      region = lib.mkOption {
+        type = lib.types.str;
+        description = "AWS region for EC2 launches.";
+      };
+
+      launchTemplateIdFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        description = "Runtime file containing the EC2 launch template ID.";
+      };
+
+      subnetIdsJsonFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        description = "Runtime file containing JSON list of subnet IDs.";
+      };
+
+      securityGroupIds = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = "Static security group IDs used by the launch template.";
+      };
+
+      instanceProfileArn = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        description = "Optional instance profile ARN override.";
+      };
+    };
+
+    haproxy = {
+      generateConfig = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = "Whether this module manages HAProxy static slot configuration.";
+      };
+
+      runtimeSocket = lib.mkOption {
+        type = lib.types.str;
+        default = "/run/haproxy/admin.sock";
+        description = "HAProxy admin runtime socket path.";
+      };
+
+      backend = lib.mkOption {
+        type = lib.types.str;
+        default = "all";
+        description = "HAProxy backend name used for static builder slots.";
+      };
+
+      slotPrefix = lib.mkOption {
+        type = lib.types.str;
+        default = "slot";
+        description = "Slot name prefix in HAProxy backend.";
+      };
+
+      slotCount = lib.mkOption {
+        type = lib.types.int;
+        default = 8;
+        description = "Number of static HAProxy slots.";
+      };
+
+      listenPort = lib.mkOption {
+        type = lib.types.int;
+        default = 2222;
+        description = "HAProxy frontend port for nix remote builders.";
+      };
+
+      checkReadyUpCount = lib.mkOption {
+        type = lib.types.int;
+        default = 2;
+        description = "Consecutive HAProxy UP checks required before slot becomes ready.";
+      };
+    };
+
+    capacity = {
+      defaultSystem = lib.mkOption {
+        type = lib.types.str;
+        default = "x86_64-linux";
+        description = "Default reservation system.";
+      };
+
+      minSlots = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "Minimum active slots.";
+      };
+
+      maxSlots = lib.mkOption {
+        type = lib.types.int;
+        default = 8;
+        description = "Maximum active slots.";
+      };
+
+      targetWarmSlots = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "Target number of warm slots.";
+      };
+
+      maxLeasesPerSlot = lib.mkOption {
+        type = lib.types.int;
+        default = 1;
+        description = "Maximum concurrent leases per slot.";
+      };
+
+      reservationTtlSeconds = lib.mkOption {
+        type = lib.types.int;
+        default = 1200;
+        description = "Reservation TTL in seconds.";
+      };
+
+      idleScaleDownSeconds = lib.mkOption {
+        type = lib.types.int;
+        default = 900;
+        description = "Idle seconds before draining a ready slot.";
+      };
+
+      drainTimeoutSeconds = lib.mkOption {
+        type = lib.types.int;
+        default = 120;
+        description = "Drain timeout before force termination.";
+      };
+
+      launchBatchSize = lib.mkOption {
+        type = lib.types.int;
+        default = 1;
+        description = "Launch batch size for the default system entry.";
+      };
+    };
+
+    security = {
+      socketMode = lib.mkOption {
+        type = lib.types.str;
+        default = "0660";
+        description = "API socket mode in daemon config.";
+      };
+
+      socketOwner = lib.mkOption {
+        type = lib.types.str;
+        default = "buildbot";
+        description = "API socket owner in daemon config.";
+      };
+
+      socketGroup = lib.mkOption {
+        type = lib.types.str;
+        default = "buildbot";
+        description = "API socket group in daemon config.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.aws.launchTemplateIdFile != null;
+        message = "services.nix-builder-autoscaler.aws.launchTemplateIdFile must be set.";
+      }
+      {
+        assertion = cfg.aws.subnetIdsJsonFile != null;
+        message = "services.nix-builder-autoscaler.aws.subnetIdsJsonFile must be set.";
+      }
+    ];
+
+    services.haproxy = lib.mkIf cfg.haproxy.generateConfig {
+      enable = true;
+      config = ''
+        global
+          stats socket ${cfg.haproxy.runtimeSocket} mode 660 level admin expose-fd listeners
+          stats timeout 2m
+
+        defaults
+          mode tcp
+          timeout connect 10s
+          timeout client 36h
+          timeout server 36h
+          timeout queue 30s
+
+        frontend nix-builders
+          bind *:${toString cfg.haproxy.listenPort}
+          default_backend ${cfg.haproxy.backend}
+
+        backend ${cfg.haproxy.backend}
+          balance leastconn
+          option tcp-check
+          tcp-check expect rstring SSH-2\\.0-OpenSSH.*
+          ${lib.concatMapStrings (
+            i:
+            "server ${cfg.haproxy.slotPrefix}${lib.fixedWidthNumber 3 i} 127.0.0.2:22 disabled check inter 5s fall 2 rise 2 maxconn 2\n          "
+          ) (lib.range 1 cfg.haproxy.slotCount)}
+      '';
+    };
+
+    systemd.services.nix-builder-autoscaler = {
+      description = "Nix builder autoscaler daemon";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [
+        "network-online.target"
+      ]
+      ++ lib.optionals cfg.haproxy.generateConfig [ "haproxy.service" ];
+      preStart = ''
+                install -d -m 0750 -o ${cfg.user} -g ${cfg.group} /run/nix-builder-autoscaler
+                launch_template_id="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.launchTemplateIdFile})"
+                subnet_ids_json="$(tr -d '\n' < ${lib.escapeShellArg cfg.aws.subnetIdsJsonFile})"
+
+                cat > ${generatedConfigPath} <<EOF
+        [server]
+        socket_path = "${cfg.socketPath}"
+        log_level = "${cfg.logLevel}"
+        db_path = "${cfg.dbPath}"
+
+        [aws]
+        region = "${cfg.aws.region}"
+        launch_template_id = "$launch_template_id"
+        subnet_ids = $subnet_ids_json
+        security_group_ids = ${tomlStringList cfg.aws.securityGroupIds}
+        instance_profile_arn = "${cfg.aws.instanceProfileArn}"
+
+        [haproxy]
+        runtime_socket = "${cfg.haproxy.runtimeSocket}"
+        backend = "${cfg.haproxy.backend}"
+        slot_prefix = "${cfg.haproxy.slotPrefix}"
+        slot_count = ${toString cfg.haproxy.slotCount}
+        check_ready_up_count = ${toString cfg.haproxy.checkReadyUpCount}
+
+        [capacity]
+        default_system = "${cfg.capacity.defaultSystem}"
+        min_slots = ${toString cfg.capacity.minSlots}
+        max_slots = ${toString cfg.capacity.maxSlots}
+        target_warm_slots = ${toString cfg.capacity.targetWarmSlots}
+        max_leases_per_slot = ${toString cfg.capacity.maxLeasesPerSlot}
+        reservation_ttl_seconds = ${toString cfg.capacity.reservationTtlSeconds}
+        idle_scale_down_seconds = ${toString cfg.capacity.idleScaleDownSeconds}
+        drain_timeout_seconds = ${toString cfg.capacity.drainTimeoutSeconds}
+
+        [security]
+        socket_mode = "${cfg.security.socketMode}"
+        socket_owner = "${cfg.security.socketOwner}"
+        socket_group = "${cfg.security.socketGroup}"
+
+        [[systems]]
+        name = "${cfg.capacity.defaultSystem}"
+        min_slots = ${toString cfg.capacity.minSlots}
+        max_slots = ${toString cfg.capacity.maxSlots}
+        target_warm_slots = ${toString cfg.capacity.targetWarmSlots}
+        max_leases_per_slot = ${toString cfg.capacity.maxLeasesPerSlot}
+        launch_batch_size = ${toString cfg.capacity.launchBatchSize}
+        scale_down_idle_seconds = ${toString cfg.capacity.idleScaleDownSeconds}
+        EOF
+
+                chown ${cfg.user}:${cfg.group} ${generatedConfigPath}
+                chmod 0640 ${generatedConfigPath}
+      '';
+      serviceConfig = {
+        Type = "simple";
+        User = cfg.user;
+        Group = cfg.group;
+        SupplementaryGroups = cfg.supplementaryGroups;
+        ExecStart = "${cfg.package}/bin/python -m nix_builder_autoscaler --config ${generatedConfigPath}";
+        Restart = "always";
+        RestartSec = 2;
+        RuntimeDirectory = "nix-builder-autoscaler";
+        RuntimeDirectoryMode = "0750";
+        StateDirectory = "nix-builder-autoscaler";
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+        ReadWritePaths = [ (builtins.dirOf cfg.dbPath) ];
+      };
+    };
+  };
+}