majuna/app/terraform/bridge/meta.py

import datetime
import logging
import random
from typing import Tuple, List

from app import db
from app.models.bridges import BridgeConf, Bridge, ProviderAllocation
from app.models.cloud import CloudProvider, CloudAccount
from app.terraform import BaseAutomation

BRIDGE_PROVIDERS = [
    # In order of cost
    CloudProvider.HCLOUD,
    CloudProvider.GANDI,
    CloudProvider.OVH,
    CloudProvider.AWS,
]


def active_bridges_in_account(account: CloudAccount) -> List[Bridge]:
    bridges: List[Bridge] = Bridge.query.filter(
        Bridge.cloud_account_id == account.id,
        Bridge.destroyed.is_(None),
    ).all()
    return bridges


def create_bridges_in_account(bridgeconf: BridgeConf, account: CloudAccount, count: int) -> int:
    created = 0
    while created < count and len(active_bridges_in_account(account)) < account.max_instances:
        logging.debug("Creating bridge for configuration %s in account %s", bridgeconf.id, account)
        bridge = Bridge()
        bridge.pool_id = bridgeconf.pool.id
        bridge.conf_id = bridgeconf.id
        bridge.cloud_account = account
        bridge.added = datetime.datetime.utcnow()
        bridge.updated = datetime.datetime.utcnow()
        logging.debug("Creating bridge %s", bridge)
        db.session.add(bridge)
        created += 1
    return created


def create_bridges_by_cost(bridgeconf: BridgeConf, count: int) -> int:
    """
    Creates bridge resources for the given bridge configuration using the cheapest available provider.
    """
    logging.debug("Creating %s bridges by cost for configuration %s", count, bridgeconf.id)
    created = 0
    for provider in BRIDGE_PROVIDERS:
        if created >= count:
            break
        logging.info("Creating bridges in %s accounts", provider.description)
        for account in CloudAccount.query.filter(
                CloudAccount.destroyed.is_(None),
                CloudAccount.enabled.is_(True),
                CloudAccount.provider == provider,
        ).all():
            logging.info("Creating bridges in %s", account)
            created += create_bridges_in_account(bridgeconf, account, count - created)
    logging.debug("Created %s bridges", created)
    return created


def _accounts_with_room() -> List[CloudAccount]:
    accounts = CloudAccount.query.filter(
        CloudAccount.destroyed.is_(None),
        CloudAccount.enabled.is_(True),
    ).all()
    accounts_with_room: List[CloudAccount] = []
    for account in accounts:
        if len(active_bridges_in_account(account)) < account.max_instances:
            accounts_with_room.append(account)
    return accounts_with_room


def create_bridges_by_random(bridgeconf: BridgeConf, count: int) -> int:
    """
    Creates bridge resources for the given bridge configuration using random providers.
    """
    logging.debug("Creating %s bridges by random for configuration %s", count, bridgeconf.id)
    created = 0
    while candidate_accounts := _accounts_with_room():
        #  Not security-critical random number generation
        account = random.choice(candidate_accounts)  # nosec: B311
        create_bridges_in_account(bridgeconf, account, 1)
        created += 1
        if created == count:
            return count
    return created  # No account with room


def create_bridges(bridgeconf: BridgeConf, count: int) -> int:
    if bridgeconf.provider_allocation == ProviderAllocation.COST:
        return create_bridges_by_cost(bridgeconf, count)
    else:
        return create_bridges_by_random(bridgeconf, count)


def deprecate_bridges(bridgeconf: BridgeConf, count: int, reason: str = "redundant") -> int:
    logging.debug("Deprecating %s bridges (%s) for configuration %s", count, reason, bridgeconf.id)
    deprecated = 0
    active_conf_bridges = iter(Bridge.query.filter(
        Bridge.conf_id == bridgeconf.id,
        Bridge.deprecated.is_(None),
        Bridge.destroyed.is_(None),
    ).all())
    while deprecated < count:
        logging.debug("Deprecating bridge %s for configuration %s", deprecated + 1, bridgeconf.id)
        bridge = next(active_conf_bridges)
        logging.debug("Bridge %r", bridge)
        bridge.deprecate(reason=reason)
        deprecated += 1
    return deprecated


class BridgeMetaAutomation(BaseAutomation):
    short_name = "bridge_meta"
    description = "Housekeeping for bridges"
    frequency = 1

    def automate(self, full: bool = False) -> Tuple[bool, str]:
        # Destroy expired bridges
        deprecated_bridges: List[Bridge] = Bridge.query.filter(
            Bridge.destroyed.is_(None),
            Bridge.deprecated.is_not(None),
        ).all()
        logging.debug("Found %s deprecated bridges", len(deprecated_bridges))
        for bridge in deprecated_bridges:
            cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=bridge.conf.expiry_hours)
            if bridge.deprecated < cutoff:
                logging.debug("Destroying expired bridge")
                bridge.destroy()
        # Deprecate orphaned bridges
        active_bridges = Bridge.query.filter(
            Bridge.deprecated.is_(None),
            Bridge.destroyed.is_(None),
        ).all()
        logging.debug("Found %s active bridges", len(active_bridges))
        for bridge in active_bridges:
            if bridge.conf.destroyed is not None:
                bridge.deprecate(reason="conf_destroyed")
        # Create new bridges
        activate_bridgeconfs = BridgeConf.query.filter(
            BridgeConf.destroyed.is_(None),
        ).all()
        logging.debug("Found %s active bridge configurations", len(activate_bridgeconfs))
        for bridgeconf in activate_bridgeconfs:
            active_conf_bridges = Bridge.query.filter(
                Bridge.conf_id == bridgeconf.id,
                Bridge.deprecated.is_(None),
                Bridge.destroyed.is_(None),
            ).all()
            total_conf_bridges = Bridge.query.filter(
                Bridge.conf_id == bridgeconf.id,
                Bridge.destroyed.is_(None),
            ).all()
            logging.debug("Generating new bridges for %s (active: %s, total: %s, target: %s, max: %s)",
                          bridgeconf.id,
                          len(active_conf_bridges),
                          len(total_conf_bridges),
                          bridgeconf.target_number,
                          bridgeconf.max_number
                          )
            missing = min(
                bridgeconf.target_number - len(active_conf_bridges),
                bridgeconf.max_number - len(total_conf_bridges))
            if missing > 0:
                create_bridges(bridgeconf, missing)
            elif missing < 0:
                deprecate_bridges(bridgeconf, 0 - missing)
        db.session.commit()
        return True, ""