resource pool system

app/terraform/proxy/meta.py

@@ -0,0 +1,90 @@
+import datetime
+import logging
+import random
+import string
+from typing import Tuple, List
+
+from tldextract import tldextract
+
+from app import db
+from app.models.base import Pool
+from app.models.mirrors import Proxy, Origin
+from app.terraform import BaseAutomation
+from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
+from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
+from app.terraform.proxy.fastly import ProxyFastlyAutomation
+
+PROXY_PROVIDERS = {p.short_name: p for p in [  # In order of preference
+    ProxyCloudfrontAutomation,
+    ProxyFastlyAutomation,
+    ProxyAzureCdnAutomation
+]}
+
+
+def create_proxy(pool: Pool, origin: Origin) -> bool:
+    for desperate in [False, True]:
+        for provider in PROXY_PROVIDERS.values():
+            if origin.smart and not provider.smart_proxies:
+                continue  # This origin cannot be supported on this provider
+            if provider.smart_proxies and not (desperate or origin.smart):
+                continue
+            next_subgroup = provider.next_subgroup(origin.group_id)
+            if next_subgroup is None:
+                continue
+            proxy = Proxy()
+            proxy.pool_id = pool.id
+            proxy.origin_id = origin.id
+            proxy.provider = provider.provider
+            proxy.psg = provider.next_subgroup(origin.group_id)
+            # The random usage below is good enough for its purpose: to create a slug that
+            # hasn't been used recently.
+            proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
+                random.choices(string.ascii_lowercase, k=12))  # nosec
+            proxy.added = datetime.datetime.utcnow()
+            proxy.updated = datetime.datetime.utcnow()
+            logging.debug("Creating proxy %s", proxy)
+            db.session.add(proxy)
+            return True
+    return False
+
+
+class ProxyMetaAutomation(BaseAutomation):
+    short_name = "proxy_meta"
+    description = "Housekeeping for proxies"
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
+        # Destroy expired proxies
+        cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
+        proxies: List[Proxy] = Proxy.query.filter(
+            Proxy.destroyed.is_(None),
+            Proxy.deprecated < cutoff
+        ).all()
+        for proxy in proxies:
+            logging.debug("Destroying expired proxy")
+            proxy.destroy()
+        # Deprecate orphaned proxies and mismatched proxies
+        proxies = Proxy.query.filter(
+            Proxy.deprecated.is_(None),
+            Proxy.destroyed.is_(None),
+        ).all()
+        for proxy in proxies:
+            if proxy.origin.destroyed is not None:
+                proxy.deprecate(reason="origin_destroyed")
+            if proxy.origin.smart and not PROXY_PROVIDERS[proxy.provider].smart_proxies:
+                proxy.deprecate(reason="not_smart_enough")
+        # Create new proxies
+        pools = Pool.query.all()
+        for pool in pools:
+            for group in pool.groups:
+                for origin in group.origins:
+                    if origin.destroyed is not None:
+                        continue
+                    proxies = [
+                        x for x in origin.proxies
+                        if x.pool_id == pool.id and x.deprecated is None and x.destroyed is None
+                    ]
+                    if not proxies:
+                        logging.debug("Creating new proxy for %s in pool %s", origin, pool)
+                        create_proxy(pool, origin)
+        db.session.commit()
+        return True, ""