majuna/app/terraform/proxy/meta.py

import datetime
import logging
import random
import string
from typing import Tuple, List

from tldextract import tldextract

from app import db
from app.models.base import Pool
from app.models.mirrors import Proxy, Origin
from app.terraform import BaseAutomation
from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
from app.terraform.proxy.fastly import ProxyFastlyAutomation

PROXY_PROVIDERS = {p.provider: p for p in [  # type: ignore[attr-defined]
    # In order of preference
    ProxyCloudfrontAutomation,
    ProxyFastlyAutomation,
    ProxyAzureCdnAutomation
] if p.enabled}  # type: ignore[attr-defined]


def create_proxy(pool: Pool, origin: Origin) -> bool:
    """
    Creates a web proxy resource for the given origin and pool combination.

    Initially it will attempt to create smart proxies on providers that support smart proxies,
    and "simple" proxies on other providers. If other providers have exhausted their quota
    already then a "simple" proxy may be created on a platform that supports smart proxies.

    A boolean is returned to indicate whether a proxy resource was created.

    :param pool: pool to create the resource for
    :param origin: origin to create the resource for
    :return: whether a proxy resource was created
    """
    for desperate in [False, True]:
        for provider in PROXY_PROVIDERS.values():
            if origin.smart and not provider.smart_proxies:  # type: ignore[attr-defined]
                continue  # This origin cannot be supported on this provider
            if provider.smart_proxies and not (desperate or origin.smart):  # type: ignore[attr-defined]
                continue
            next_subgroup = provider.next_subgroup(origin.group_id)  # type: ignore[attr-defined]
            if next_subgroup is None:
                continue  # Exceeded maximum number of subgroups and last subgroup is full
            proxy = Proxy()
            proxy.pool_id = pool.id
            proxy.origin_id = origin.id
            proxy.provider = provider.provider  # type: ignore[attr-defined]
            proxy.psg = next_subgroup
            # The random usage below is good enough for its purpose: to create a slug that
            # hasn't been used recently.
            proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
                random.choices(string.ascii_lowercase, k=12))  # nosec
            proxy.added = datetime.datetime.utcnow()
            proxy.updated = datetime.datetime.utcnow()
            logging.debug("Creating proxy %s", proxy)
            db.session.add(proxy)
            return True
    return False


class ProxyMetaAutomation(BaseAutomation):
    short_name = "proxy_meta"
    description = "Housekeeping for proxies"
    frequency = 1

    def automate(self, full: bool = False) -> Tuple[bool, str]:
        # Destroy expired proxies
        cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=4)
        proxies: List[Proxy] = Proxy.query.filter(
            Proxy.destroyed.is_(None),
            Proxy.deprecated < cutoff
        ).all()
        for proxy in proxies:
            logging.debug("Destroying expired proxy")
            proxy.destroy()
        # Deprecate orphaned proxies and mismatched proxies
        proxies = Proxy.query.filter(
            Proxy.deprecated.is_(None),
            Proxy.destroyed.is_(None),
        ).all()
        for proxy in proxies:
            if proxy.origin.destroyed is not None:
                proxy.deprecate(reason="origin_destroyed")
            if proxy.origin.smart and not PROXY_PROVIDERS[proxy.provider].smart_proxies:  # type: ignore[attr-defined]
                proxy.deprecate(reason="not_smart_enough")
        # Create new proxies
        pools = Pool.query.all()
        for pool in pools:
            for group in pool.groups:
                for origin in group.origins:
                    if origin.destroyed is not None:
                        continue
                    proxies = [
                        x for x in origin.proxies
                        if x.pool_id == pool.id and x.deprecated is None and x.destroyed is None
                    ]
                    if not proxies:
                        logging.debug("Creating new proxy for %s in pool %s", origin, pool)
                        create_proxy(pool, origin)
        db.session.commit()
        return True, ""