92 lines
3.8 KiB
Python
92 lines
3.8 KiB
Python
import datetime
|
|
import logging
|
|
import random
|
|
import string
|
|
from typing import Tuple, List
|
|
|
|
from tldextract import tldextract
|
|
|
|
from app import db
|
|
from app.models.base import Pool
|
|
from app.models.mirrors import Proxy, Origin
|
|
from app.terraform import BaseAutomation
|
|
from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
|
|
from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
|
|
from app.terraform.proxy.fastly import ProxyFastlyAutomation
|
|
|
|
PROXY_PROVIDERS = {p.provider: p for p in [ # type: ignore[attr-defined]
|
|
# In order of preference
|
|
ProxyCloudfrontAutomation,
|
|
ProxyFastlyAutomation,
|
|
ProxyAzureCdnAutomation
|
|
] if p.enabled} # type: ignore[attr-defined]
|
|
|
|
|
|
def create_proxy(pool: Pool, origin: Origin) -> bool:
|
|
for desperate in [False, True]:
|
|
for provider in PROXY_PROVIDERS.values():
|
|
if origin.smart and not provider.smart_proxies: # type: ignore[attr-defined]
|
|
continue # This origin cannot be supported on this provider
|
|
if provider.smart_proxies and not (desperate or origin.smart): # type: ignore[attr-defined]
|
|
continue
|
|
next_subgroup = provider.next_subgroup(origin.group_id) # type: ignore[attr-defined]
|
|
if next_subgroup is None:
|
|
continue
|
|
proxy = Proxy()
|
|
proxy.pool_id = pool.id
|
|
proxy.origin_id = origin.id
|
|
proxy.provider = provider.provider # type: ignore[attr-defined]
|
|
proxy.psg = provider.next_subgroup(origin.group_id) # type: ignore[attr-defined]
|
|
# The random usage below is good enough for its purpose: to create a slug that
|
|
# hasn't been used recently.
|
|
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
|
|
random.choices(string.ascii_lowercase, k=12)) # nosec
|
|
proxy.added = datetime.datetime.utcnow()
|
|
proxy.updated = datetime.datetime.utcnow()
|
|
logging.debug("Creating proxy %s", proxy)
|
|
db.session.add(proxy)
|
|
return True
|
|
return False
|
|
|
|
|
|
class ProxyMetaAutomation(BaseAutomation):
|
|
short_name = "proxy_meta"
|
|
description = "Housekeeping for proxies"
|
|
frequency = 1
|
|
|
|
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
|
# Destroy expired proxies
|
|
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
|
|
proxies: List[Proxy] = Proxy.query.filter(
|
|
Proxy.destroyed.is_(None),
|
|
Proxy.deprecated < cutoff
|
|
).all()
|
|
for proxy in proxies:
|
|
logging.debug("Destroying expired proxy")
|
|
proxy.destroy()
|
|
# Deprecate orphaned proxies and mismatched proxies
|
|
proxies = Proxy.query.filter(
|
|
Proxy.deprecated.is_(None),
|
|
Proxy.destroyed.is_(None),
|
|
).all()
|
|
for proxy in proxies:
|
|
if proxy.origin.destroyed is not None:
|
|
proxy.deprecate(reason="origin_destroyed")
|
|
if proxy.origin.smart and not PROXY_PROVIDERS[proxy.provider].smart_proxies: # type: ignore[attr-defined]
|
|
proxy.deprecate(reason="not_smart_enough")
|
|
# Create new proxies
|
|
pools = Pool.query.all()
|
|
for pool in pools:
|
|
for group in pool.groups:
|
|
for origin in group.origins:
|
|
if origin.destroyed is not None:
|
|
continue
|
|
proxies = [
|
|
x for x in origin.proxies
|
|
if x.pool_id == pool.id and x.deprecated is None and x.destroyed is None
|
|
]
|
|
if not proxies:
|
|
logging.debug("Creating new proxy for %s in pool %s", origin, pool)
|
|
create_proxy(pool, origin)
|
|
db.session.commit()
|
|
return True, ""
|