feat(bridges): next generation bridge management

2023-01-26 15:42:25 +00:00 · 2023-01-26 15:42:25 +00:00 · 05285a4ae6
commit 05285a4ae6
parent 20fad30a06
12 changed files with 329 additions and 89 deletions
--- a/app/terraform/bridge/meta.py
+++ b/app/terraform/bridge/meta.py
@ -0,0 +1,122 @@
+import datetime
+import logging
+from typing import Tuple, List
+
+from app import db
+from app.models.bridges import BridgeConf, Bridge
+from app.terraform import BaseAutomation
+from app.terraform.bridge.gandi import BridgeGandiAutomation
+from app.terraform.bridge.hcloud import BridgeHcloudAutomation
+from app.terraform.bridge.ovh import BridgeOvhAutomation
+
+BRIDGE_PROVIDERS = {p.provider: p for p in [
+    # In order of cost
+    BridgeHcloudAutomation,
+    BridgeGandiAutomation,
+    BridgeOvhAutomation,
+    # BridgeAWSAutomation,  TODO: This module is broken right now
+] if p.enabled}
+
+
+def create_bridges(bridgeconf: BridgeConf, count: int) -> int:
+    """
+    Creates a bridge resource for the given bridge configuration.
+    """
+    logging.debug("Creating %s bridges for configuration %s", count, bridgeconf.id)
+    created = 0
+    # TODO: deal with the fact that I created a dictionary and then forgot it wasn't ordered
+    while created < count:
+        for provider in BRIDGE_PROVIDERS:
+            if BRIDGE_PROVIDERS[provider].max_bridges > BRIDGE_PROVIDERS[provider].active_bridges_count():
+                logging.debug("Creating bridge for configuration %s with provider %s", bridgeconf.id, provider)
+                bridge = Bridge()
+                bridge.pool_id = bridgeconf.pool.id
+                bridge.conf_id = bridgeconf.id
+                bridge.provider = provider
+                bridge.added = datetime.datetime.utcnow()
+                bridge.updated = datetime.datetime.utcnow()
+                logging.debug("Creating bridge %s", bridge)
+                db.session.add(bridge)
+                created += 1
+                break
+            else:
+                logging.debug("No provider has available quota to create missing bridge for configuration %s",
+                              bridgeconf.id)
+    logging.debug("Created %s bridges", created)
+    return created
+
+
+def deprecate_bridges(bridgeconf: BridgeConf, count: int, reason: str = "redundant") -> int:
+    logging.debug("Deprecating %s bridges (%s) for configuration %s", count, reason, bridgeconf.id)
+    deprecated = 0
+    active_conf_bridges = iter(Bridge.query.filter(
+        Bridge.conf_id == bridgeconf.id,
+        Bridge.deprecated.is_(None),
+        Bridge.destroyed.is_(None),
+    ).all())
+    while deprecated < count:
+        logging.debug("Deprecating bridge %s for configuration %s", deprecated + 1, bridgeconf.id)
+        bridge = next(active_conf_bridges)
+        logging.debug("Bridge %r", bridge)
+        bridge.deprecate(reason=reason)
+        deprecated += 1
+    return deprecated
+
+
+class BridgeMetaAutomation(BaseAutomation):
+    short_name = "bridge_meta"
+    description = "Housekeeping for bridges"
+    frequency = 1
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
+        # Destroy expired bridges
+        deprecated_bridges: List[Bridge] = Bridge.query.filter(
+            Bridge.destroyed.is_(None),
+            Bridge.deprecated.is_not(None),
+        ).all()
+        logging.debug("Found %s deprecated bridges", len(deprecated_bridges))
+        for bridge in deprecated_bridges:
+            cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=bridge.conf.expiry_hours)
+            if bridge.deprecated < cutoff:
+                logging.debug("Destroying expired bridge")
+                bridge.destroy()
+        # Deprecate orphaned bridges
+        active_bridges = Bridge.query.filter(
+            Bridge.deprecated.is_(None),
+            Bridge.destroyed.is_(None),
+        ).all()
+        logging.debug("Found %s active bridges", len(active_bridges))
+        for bridge in active_bridges:
+            if bridge.conf.destroyed is not None:
+                bridge.deprecate(reason="conf_destroyed")
+        # Create new bridges
+        activate_bridgeconfs = BridgeConf.query.filter(
+            BridgeConf.destroyed.is_(None),
+        ).all()
+        logging.debug("Found %s active bridge configurations", len(activate_bridgeconfs))
+        for bridgeconf in activate_bridgeconfs:
+            active_conf_bridges = Bridge.query.filter(
+                Bridge.conf_id == bridgeconf.id,
+                Bridge.deprecated.is_(None),
+                Bridge.destroyed.is_(None),
+            ).all()
+            total_conf_bridges = Bridge.query.filter(
+                Bridge.conf_id == bridgeconf.id,
+                Bridge.destroyed.is_(None),
+            ).all()
+            logging.debug("Generating new bridges for %s (active: %s, total: %s, target: %s, max: %s)",
+                          bridgeconf.id,
+                          len(active_conf_bridges),
+                          len(total_conf_bridges),
+                          bridgeconf.target_number,
+                          bridgeconf.max_number
+                          )
+            missing = min(
+                bridgeconf.target_number - len(active_conf_bridges),
+                bridgeconf.max_number - len(total_conf_bridges))
+            if missing > 0:
+                create_bridges(bridgeconf, missing)
+            elif missing < 0:
+                deprecate_bridges(bridgeconf, 0 - missing)
+        db.session.commit()
+        return True, ""