import datetime import logging import random import string from typing import Tuple, List from flask import current_app from tldextract import tldextract from app import db from app.models.base import Pool from app.models.mirrors import Proxy, Origin from app.terraform import BaseAutomation from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation from app.terraform.proxy.fastly import ProxyFastlyAutomation PROXY_PROVIDERS = {p.provider: p for p in [ # type: ignore[attr-defined] # In order of preference ProxyCloudfrontAutomation, ProxyFastlyAutomation, ProxyAzureCdnAutomation ] if p.enabled} # type: ignore[attr-defined] def create_proxy(pool: Pool, origin: Origin) -> bool: """ Creates a web proxy resource for the given origin and pool combination. Initially it will attempt to create smart proxies on providers that support smart proxies, and "simple" proxies on other providers. If other providers have exhausted their quota already then a "simple" proxy may be created on a platform that supports smart proxies. A boolean is returned to indicate whether a proxy resource was created. :param pool: pool to create the resource for :param origin: origin to create the resource for :return: whether a proxy resource was created """ for desperate in [False, True]: for provider in PROXY_PROVIDERS.values(): if origin.smart and not provider.smart_proxies: # type: ignore[attr-defined] continue # This origin cannot be supported on this provider if provider.smart_proxies and not (desperate or origin.smart): # type: ignore[attr-defined] continue next_subgroup = provider.next_subgroup(origin.group_id) # type: ignore[attr-defined] if next_subgroup is None: continue # Exceeded maximum number of subgroups and last subgroup is full proxy = Proxy() proxy.pool_id = pool.id proxy.origin_id = origin.id proxy.provider = provider.provider # type: ignore[attr-defined] proxy.psg = next_subgroup # The random usage below is good enough for its purpose: to create a slug that # hasn't been used recently. proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join( random.choices(string.ascii_lowercase, k=12)) # nosec proxy.added = datetime.datetime.utcnow() proxy.updated = datetime.datetime.utcnow() logging.debug("Creating proxy %s", proxy) db.session.add(proxy) return True return False class ProxyMetaAutomation(BaseAutomation): short_name = "proxy_meta" description = "Housekeeping for proxies" frequency = 1 def automate(self, full: bool = False) -> Tuple[bool, str]: # Deprecate orphaned proxies, old proxies and mismatched proxies proxies: List[Proxy] = Proxy.query.filter( Proxy.deprecated.is_(None), Proxy.destroyed.is_(None), ).all() for proxy in proxies: if proxy.origin.destroyed is not None: proxy.deprecate(reason="origin_destroyed") if proxy.origin_id in current_app.config.get("DAILY_REPLACEMENT_ORIGINS", []): max_age_cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=1, seconds=86400 * random.random()) # nosec: B311 else: max_age_cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=5, seconds=86400 * random.random()) # nosec: B311 if proxy.added < max_age_cutoff: proxy.deprecate(reason="max_age_reached") if proxy.origin.smart and not PROXY_PROVIDERS[proxy.provider].smart_proxies: # type: ignore[attr-defined] proxy.deprecate(reason="not_smart_enough") # Create new proxies pools = Pool.query.all() for pool in pools: for group in pool.groups: for origin in group.origins: if origin.destroyed is not None: continue proxies = [ x for x in origin.proxies if x.pool_id == pool.id and x.deprecated is None and x.destroyed is None ] if not proxies: logging.debug("Creating new proxy for %s in pool %s", origin, pool) create_proxy(pool, origin) # Destroy expired proxies expiry_cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=4) proxies = Proxy.query.filter( Proxy.destroyed.is_(None), Proxy.deprecated < expiry_cutoff ).all() for proxy in proxies: logging.debug("Destroying expired proxy") proxy.destroy() db.session.commit() return True, ""