import datetime import logging import random import string from typing import Tuple, List from tldextract import tldextract from app import db from app.models.base import Pool from app.models.mirrors import Proxy, Origin from app.terraform import BaseAutomation from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation from app.terraform.proxy.fastly import ProxyFastlyAutomation PROXY_PROVIDERS = {p.provider: p for p in [ # In order of preference ProxyCloudfrontAutomation, ProxyFastlyAutomation, ProxyAzureCdnAutomation ] if p.enabled} # type: ignore[truthy-function] def create_proxy(pool: Pool, origin: Origin) -> bool: for desperate in [False, True]: for provider in PROXY_PROVIDERS.values(): if origin.smart and not provider.smart_proxies: continue # This origin cannot be supported on this provider if provider.smart_proxies and not (desperate or origin.smart): continue next_subgroup = provider.next_subgroup(origin.group_id) if next_subgroup is None: continue proxy = Proxy() proxy.pool_id = pool.id proxy.origin_id = origin.id proxy.provider = provider.provider proxy.psg = provider.next_subgroup(origin.group_id) # The random usage below is good enough for its purpose: to create a slug that # hasn't been used recently. proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join( random.choices(string.ascii_lowercase, k=12)) # nosec proxy.added = datetime.datetime.utcnow() proxy.updated = datetime.datetime.utcnow() logging.debug("Creating proxy %s", proxy) db.session.add(proxy) return True return False class ProxyMetaAutomation(BaseAutomation): short_name = "proxy_meta" description = "Housekeeping for proxies" frequency = 1 def automate(self, full: bool = False) -> Tuple[bool, str]: # Destroy expired proxies cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3) proxies: List[Proxy] = Proxy.query.filter( Proxy.destroyed.is_(None), Proxy.deprecated < cutoff ).all() for proxy in proxies: logging.debug("Destroying expired proxy") proxy.destroy() # Deprecate orphaned proxies and mismatched proxies proxies = Proxy.query.filter( Proxy.deprecated.is_(None), Proxy.destroyed.is_(None), ).all() for proxy in proxies: if proxy.origin.destroyed is not None: proxy.deprecate(reason="origin_destroyed") if proxy.origin.smart and not PROXY_PROVIDERS[proxy.provider].smart_proxies: proxy.deprecate(reason="not_smart_enough") # Create new proxies pools = Pool.query.all() for pool in pools: for group in pool.groups: for origin in group.origins: if origin.destroyed is not None: continue proxies = [ x for x in origin.proxies if x.pool_id == pool.id and x.deprecated is None and x.destroyed is None ] if not proxies: logging.debug("Creating new proxy for %s in pool %s", origin, pool) create_proxy(pool, origin) db.session.commit() return True, ""