import datetime import logging import random from typing import Tuple, List from app import db from app.models.bridges import BridgeConf, Bridge, ProviderAllocation from app.models.cloud import CloudProvider, CloudAccount from app.terraform import BaseAutomation BRIDGE_PROVIDERS = [ # In order of cost CloudProvider.HCLOUD, CloudProvider.GANDI, CloudProvider.OVH, CloudProvider.AWS, ] def active_bridges_in_account(account: CloudAccount) -> List[Bridge]: bridges: List[Bridge] = Bridge.query.filter( Bridge.cloud_account_id == account.id, Bridge.destroyed.is_(None), ).all() return bridges def create_bridges_in_account(bridgeconf: BridgeConf, account: CloudAccount, count: int) -> int: created = 0 while created < count and len(active_bridges_in_account(account)) < account.max_instances: logging.debug("Creating bridge for configuration %s in account %s", bridgeconf.id, account) bridge = Bridge() bridge.pool_id = bridgeconf.pool.id bridge.conf_id = bridgeconf.id bridge.cloud_account = account bridge.added = datetime.datetime.utcnow() bridge.updated = datetime.datetime.utcnow() logging.debug("Creating bridge %s", bridge) db.session.add(bridge) created += 1 return created def create_bridges_by_cost(bridgeconf: BridgeConf, count: int) -> int: """ Creates bridge resources for the given bridge configuration using the cheapest available provider. """ logging.debug("Creating %s bridges by cost for configuration %s", count, bridgeconf.id) created = 0 for provider in BRIDGE_PROVIDERS: if created >= count: break logging.info("Creating bridges in %s accounts", provider.description) for account in CloudAccount.query.filter( CloudAccount.destroyed.is_(None), CloudAccount.enabled.is_(True), CloudAccount.provider == provider, ).all(): logging.info("Creating bridges in %s", account) created += create_bridges_in_account(bridgeconf, account, count - created) logging.debug("Created %s bridges", created) return created def _accounts_with_room() -> List[CloudAccount]: accounts = CloudAccount.query.filter( CloudAccount.destroyed.is_(None), CloudAccount.enabled.is_(True), ).all() accounts_with_room: List[CloudAccount] = [] for account in accounts: if len(active_bridges_in_account(account)) < account.max_instances: accounts_with_room.append(account) return accounts_with_room def create_bridges_by_random(bridgeconf: BridgeConf, count: int) -> int: """ Creates bridge resources for the given bridge configuration using random providers. """ logging.debug("Creating %s bridges by random for configuration %s", count, bridgeconf.id) created = 0 while candidate_accounts := _accounts_with_room(): # Not security-critical random number generation account = random.choice(candidate_accounts) # nosec: B311 create_bridges_in_account(bridgeconf, account, 1) created += 1 if created == count: return count return created # No account with room def create_bridges(bridgeconf: BridgeConf, count: int) -> int: if bridgeconf.provider_allocation == ProviderAllocation.COST: return create_bridges_by_cost(bridgeconf, count) else: return create_bridges_by_random(bridgeconf, count) def deprecate_bridges(bridgeconf: BridgeConf, count: int, reason: str = "redundant") -> int: logging.debug("Deprecating %s bridges (%s) for configuration %s", count, reason, bridgeconf.id) deprecated = 0 active_conf_bridges = iter(Bridge.query.filter( Bridge.conf_id == bridgeconf.id, Bridge.deprecated.is_(None), Bridge.destroyed.is_(None), ).all()) while deprecated < count: logging.debug("Deprecating bridge %s for configuration %s", deprecated + 1, bridgeconf.id) bridge = next(active_conf_bridges) logging.debug("Bridge %r", bridge) bridge.deprecate(reason=reason) deprecated += 1 return deprecated class BridgeMetaAutomation(BaseAutomation): short_name = "bridge_meta" description = "Housekeeping for bridges" frequency = 1 def automate(self, full: bool = False) -> Tuple[bool, str]: # Destroy expired bridges deprecated_bridges: List[Bridge] = Bridge.query.filter( Bridge.destroyed.is_(None), Bridge.deprecated.is_not(None), ).all() logging.debug("Found %s deprecated bridges", len(deprecated_bridges)) for bridge in deprecated_bridges: cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=bridge.conf.expiry_hours) if bridge.deprecated < cutoff: logging.debug("Destroying expired bridge") bridge.destroy() # Deprecate orphaned bridges active_bridges = Bridge.query.filter( Bridge.deprecated.is_(None), Bridge.destroyed.is_(None), ).all() logging.debug("Found %s active bridges", len(active_bridges)) for bridge in active_bridges: if bridge.conf.destroyed is not None: bridge.deprecate(reason="conf_destroyed") # Create new bridges activate_bridgeconfs = BridgeConf.query.filter( BridgeConf.destroyed.is_(None), ).all() logging.debug("Found %s active bridge configurations", len(activate_bridgeconfs)) for bridgeconf in activate_bridgeconfs: active_conf_bridges = Bridge.query.filter( Bridge.conf_id == bridgeconf.id, Bridge.deprecated.is_(None), Bridge.destroyed.is_(None), ).all() total_conf_bridges = Bridge.query.filter( Bridge.conf_id == bridgeconf.id, Bridge.destroyed.is_(None), ).all() logging.debug("Generating new bridges for %s (active: %s, total: %s, target: %s, max: %s)", bridgeconf.id, len(active_conf_bridges), len(total_conf_bridges), bridgeconf.target_number, bridgeconf.max_number ) missing = min( bridgeconf.target_number - len(active_conf_bridges), bridgeconf.max_number - len(total_conf_bridges)) if missing > 0: create_bridges(bridgeconf, missing) elif missing < 0: deprecate_bridges(bridgeconf, 0 - missing) db.session.commit() return True, ""