add termination cooldown for slot scale-down

agent/nix_builder_autoscaler/tests/test_reconciler.py

@@ -71,6 +71,7 @@ def _make_env(
     boot_timeout=300,
     binding_timeout=180,
     terminating_timeout=300,
+    termination_cooldown=0,
 ):
     clock = FakeClock()
     db = StateDB(":memory:", clock=clock)
@@ -85,6 +86,7 @@ def _make_env(
             boot_timeout_seconds=boot_timeout,
             binding_timeout_seconds=binding_timeout,
             terminating_timeout_seconds=terminating_timeout,
+            termination_cooldown_seconds=termination_cooldown,
         ),
         aws=AwsConfig(region="us-east-1"),
     )
@@ -195,3 +197,34 @@ def test_terminating_timeout_reissues_terminate_with_pacing() -> None:
     # Immediate next tick should not retry yet because last_state_change was refreshed.
     reconciler.tick()
     assert runtime.terminate_calls == ["i-5"]
+
+
+def test_termination_cooldown_spaces_terminations() -> None:
+    db, runtime, reconciler, clock = _make_env(termination_cooldown=30)
+    runtime.instances["i-6"] = _Instance(state="running", slot_id="slot001")
+    db.update_slot_state("slot001", SlotState.DRAINING, instance_id="i-6", lease_count=0)
+
+    reconciler.tick()
+    slot = db.get_slot("slot001")
+    assert slot is not None
+    assert slot["state"] == SlotState.TERMINATING.value
+    assert runtime.terminate_calls == ["i-6"]
+
+    # New draining cycle before cooldown expires should not terminate yet.
+    runtime.instances["i-7"] = _Instance(state="running", slot_id="slot001")
+    db.update_slot_state("slot001", SlotState.DRAINING, instance_id="i-7", lease_count=0)
+    clock.advance(10)
+    reconciler.tick()
+
+    slot = db.get_slot("slot001")
+    assert slot is not None
+    assert slot["state"] == SlotState.DRAINING.value
+    assert runtime.terminate_calls == ["i-6"]
+
+    # After cooldown, termination proceeds.
+    clock.advance(21)
+    reconciler.tick()
+    slot = db.get_slot("slot001")
+    assert slot is not None
+    assert slot["state"] == SlotState.TERMINATING.value
+    assert runtime.terminate_calls == ["i-6", "i-7"]