import logging import random from datetime import datetime, timedelta, timezone from typing import List, Tuple from app import db from app.models.bridges import Bridge, BridgeConf, ProviderAllocation from app.models.cloud import CloudAccount, CloudProvider from app.terraform import BaseAutomation BRIDGE_PROVIDERS = [ # In order of cost CloudProvider.HCLOUD, CloudProvider.GANDI, CloudProvider.OVH, CloudProvider.AWS, ] def active_bridges_in_account(account: CloudAccount) -> List[Bridge]: bridges: List[Bridge] = Bridge.query.filter( Bridge.cloud_account_id == account.id, Bridge.destroyed.is_(None), ).all() return bridges def create_bridges_in_account(bridgeconf: BridgeConf, account: CloudAccount, count: int) -> int: created = 0 while created < count and len(active_bridges_in_account(account)) < account.max_instances: logging.debug("Creating bridge for configuration %s in account %s", bridgeconf.id, account) bridge = Bridge() bridge.pool_id = bridgeconf.pool.id bridge.conf_id = bridgeconf.id bridge.cloud_account = account bridge.added = datetime.now(tz=timezone.utc) bridge.updated = datetime.now(tz=timezone.utc) logging.debug("Creating bridge %s", bridge) db.session.add(bridge) created += 1 return created def create_bridges_by_cost(bridgeconf: BridgeConf, count: int) -> int: """ Creates bridge resources for the given bridge configuration using the cheapest available provider. """ logging.debug("Creating %s bridges by cost for configuration %s", count, bridgeconf.id) created = 0 for provider in BRIDGE_PROVIDERS: if created >= count: break logging.info("Creating bridges in %s accounts", provider.description) for account in CloudAccount.query.filter( CloudAccount.destroyed.is_(None), CloudAccount.enabled.is_(True), CloudAccount.provider == provider, ).all(): logging.info("Creating bridges in %s", account) created += create_bridges_in_account(bridgeconf, account, count - created) logging.debug("Created %s bridges", created) return created def _accounts_with_room() -> List[CloudAccount]: accounts = CloudAccount.query.filter( CloudAccount.destroyed.is_(None), CloudAccount.enabled.is_(True), ).all() accounts_with_room: List[CloudAccount] = [] for account in accounts: if len(active_bridges_in_account(account)) < account.max_instances: accounts_with_room.append(account) return accounts_with_room def create_bridges_by_random(bridgeconf: BridgeConf, count: int) -> int: """ Creates bridge resources for the given bridge configuration using random providers. """ logging.debug("Creating %s bridges by random for configuration %s", count, bridgeconf.id) created = 0 while candidate_accounts := _accounts_with_room(): # Not security-critical random number generation account = random.choice(candidate_accounts) # nosec: B311 create_bridges_in_account(bridgeconf, account, 1) created += 1 if created == count: return count return created # No account with room def create_bridges(bridgeconf: BridgeConf, count: int) -> int: if bridgeconf.provider_allocation == ProviderAllocation.COST: return create_bridges_by_cost(bridgeconf, count) else: return create_bridges_by_random(bridgeconf, count) def deprecate_bridges(bridgeconf: BridgeConf, count: int, reason: str = "redundant") -> int: logging.debug("Deprecating %s bridges (%s) for configuration %s", count, reason, bridgeconf.id) deprecated = 0 active_conf_bridges = iter(Bridge.query.filter( Bridge.conf_id == bridgeconf.id, Bridge.deprecated.is_(None), Bridge.destroyed.is_(None), ).all()) while deprecated < count: logging.debug("Deprecating bridge %s for configuration %s", deprecated + 1, bridgeconf.id) bridge = next(active_conf_bridges) logging.debug("Bridge %r", bridge) bridge.deprecate(reason=reason) deprecated += 1 return deprecated class BridgeMetaAutomation(BaseAutomation): short_name = "bridge_meta" description = "Housekeeping for bridges" frequency = 1 def automate(self, full: bool = False) -> Tuple[bool, str]: # Destroy expired bridges deprecated_bridges: List[Bridge] = Bridge.query.filter( Bridge.destroyed.is_(None), Bridge.deprecated.is_not(None), ).all() logging.debug("Found %s deprecated bridges", len(deprecated_bridges)) for bridge in deprecated_bridges: if bridge.deprecated is None: continue # Possible due to SQLAlchemy lazy loading cutoff = datetime.now(tz=timezone.utc) - timedelta(hours=bridge.conf.expiry_hours) if bridge.deprecated < cutoff: logging.debug("Destroying expired bridge") bridge.destroy() # Deprecate orphaned bridges active_bridges = Bridge.query.filter( Bridge.deprecated.is_(None), Bridge.destroyed.is_(None), ).all() logging.debug("Found %s active bridges", len(active_bridges)) for bridge in active_bridges: if bridge.conf.destroyed is not None: bridge.deprecate(reason="conf_destroyed") # Create new bridges activate_bridgeconfs = BridgeConf.query.filter( BridgeConf.destroyed.is_(None), ).all() logging.debug("Found %s active bridge configurations", len(activate_bridgeconfs)) for bridgeconf in activate_bridgeconfs: active_conf_bridges = Bridge.query.filter( Bridge.conf_id == bridgeconf.id, Bridge.deprecated.is_(None), Bridge.destroyed.is_(None), ).all() total_conf_bridges = Bridge.query.filter( Bridge.conf_id == bridgeconf.id, Bridge.destroyed.is_(None), ).all() logging.debug("Generating new bridges for %s (active: %s, total: %s, target: %s, max: %s)", bridgeconf.id, len(active_conf_bridges), len(total_conf_bridges), bridgeconf.target_number, bridgeconf.max_number ) missing = min( bridgeconf.target_number - len(active_conf_bridges), bridgeconf.max_number - len(total_conf_bridges)) if missing > 0: create_bridges(bridgeconf, missing) elif missing < 0: deprecate_bridges(bridgeconf, 0 - missing) db.session.commit() return True, ""