jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

resource pool system

Browse source
This commit is contained in:
Iain Learmonth 2022-09-26 13:40:59 +01:00
parent dc989dd7cb
commit 16f7e2199d
19 changed files with 382 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

123
app/terraform/proxy/__init__.py
View file

@ -1,14 +1,11 @@
import os.path
import sys
from abc import abstractmethod
from collections import defaultdict
import datetime
import math
import string
import random
from typing import Dict, Optional, Any, List
from collections import defaultdict
from typing import Optional, Any, List, Dict
from sqlalchemy import text
from tldextract import tldextract
from app import app
from app.extensions import db
@ -44,13 +41,19 @@ def sp_trusted_prefixes() -> str:
class ProxyAutomation(TerraformAutomation):
subgroup_max = math.inf
subgroup_members_max = sys.maxsize
"""
Maximum number of proxies to deploy per sub-group. This is required for some providers
where the number origins per group may exceed the number of proxies that can be configured
in a single "configuration block", e.g. Azure CDN's profiles.
"""
subgroup_count_max = sys.maxsize
"""
Maximum number of subgroups that can be deployed. This is required for some providers where
the total number of subgroups is limited by a quota, e.g. Azure CDN's profiles.
"""
template: str
"""
Terraform configuration template using Jinja 2.
@ -67,83 +70,11 @@ class ProxyAutomation(TerraformAutomation):
Whether this provider supports "smart" proxies.
"""
def get_subgroups(self) -> Dict[int, Dict[int, int]]:
conn = db.engine.connect()
result = conn.execute(text("""
SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin
WHERE proxy.origin_id = origin.id
AND proxy.destroyed IS NULL
AND proxy.provider = :provider
GROUP BY origin.group_id, proxy.psg;
"""), provider=self.provider)
subgroups: Dict[int, Dict[int, int]] = defaultdict(lambda: defaultdict(lambda: 0))
for row in result:
subgroups[row[0]][row[1]] = row[2]
return subgroups
def create_missing_proxies(self) -> None:
groups = Group.query.all()
subgroups = self.get_subgroups()
for group in groups:
subgroup = 0
for origin in group.origins:
if origin.destroyed is not None:
continue
while True:
if subgroups[group.id][subgroup] >= self.subgroup_max:
subgroup += 1
else:
break
proxies = [
x for x in origin.proxies
if x.provider == self.provider and x.deprecated is None and x.destroyed is None
]
if not proxies:
subgroups[group.id][subgroup] += 1
proxy = Proxy()
proxy.origin_id = origin.id
proxy.provider = self.provider
proxy.psg = subgroup
# The random usage below is good enough for its purpose: to create a slug that
# hasn't been used before.
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
random.choices(string.ascii_lowercase, k=12)) # nosec
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
db.session.add(proxy)
db.session.commit()
def deprecate_orphaned_proxies(self) -> None:
proxies = Proxy.query.filter(
Proxy.deprecated.is_(None),
Proxy.destroyed.is_(None),
Proxy.provider == self.provider
).all()
for proxy in proxies:
if proxy.origin.destroyed is not None:
proxy.deprecate(reason="origin_destroyed")
db.session.commit()
def destroy_expired_proxies(self) -> None:
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
proxies = Proxy.query.filter(
Proxy.destroyed.is_(None),
Proxy.provider == self.provider,
Proxy.deprecated < cutoff
).all()
for proxy in proxies:
proxy.destroyed = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
db.session.commit()
@abstractmethod
def import_state(self, state: Any) -> None:
raise NotImplementedError()
def tf_prehook(self) -> Optional[Any]: # pylint: disable=useless-return
self.create_missing_proxies()
self.deprecate_orphaned_proxies()
self.destroy_expired_proxies()
return None
def tf_posthook(self, *, prehook_result: Any = None) -> None:
@ -223,3 +154,37 @@ class ProxyAutomation(TerraformAutomation):
provider=self.provider,
origins=group_origins,
smart_zone=app.config['SMART_ZONE'])
@classmethod
def get_subgroups(cls) -> Dict[int, Dict[int, int]]:
conn = db.engine.connect()
result = conn.execute(text("""
SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin
WHERE proxy.origin_id = origin.id
AND proxy.destroyed IS NULL
AND proxy.provider = :provider
GROUP BY origin.group_id, proxy.psg;
"""), provider=cls.provider)
subgroups: Dict[int, Dict[int, int]] = defaultdict(lambda: defaultdict(lambda: 0))
for row in result:
subgroups[row[0]][row[1]] = row[2]
return subgroups
@classmethod
def next_subgroup(cls, group_id: int) -> Optional[int]:
conn = db.engine.connect()
result = conn.execute(text("""
SELECT proxy.psg, COUNT(proxy.id) FROM proxy, origin
WHERE proxy.origin_id = origin.id
AND proxy.destroyed IS NULL
AND origin.group_id = :group_id
AND proxy.provider = :provider
GROUP BY proxy.psg ORDER BY proxy.psg;
"""), provider=cls.short_name, group_id=group_id)
subgroups = {
row[0]: row[1] for row in result
}
for subgroup in range(0, cls.subgroup_count_max):
if subgroups.get(subgroup, 0) < cls.subgroup_members_max:
return subgroup
return None

11
app/terraform/proxy/azure_cdn.py
View file

@ -9,7 +9,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
short_name = "proxy_azure_cdn"
description = "Deploy proxies to Azure CDN"
provider = "azure_cdn"
subgroup_max = 25
subgroup_members_max = 25
parallelism = 1
template_parameters = [
@ -125,21 +125,12 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
location = "{{ azure_location }}"
resource_group_name = data.azurerm_resource_group.this.name
{% if proxy.origin.smart %}
origin_host_header = "origin-{{ proxy.origin.id }}.cloudfront.smart.{{ smart_zone[:-1] }}"
origin {
name = "upstream"
host_name = "origin-{{ proxy.origin.id }}.cloudfront.smart.{{ smart_zone[:-1] }}"
}
{% else %}
origin_host_header = "{{ proxy.origin.domain_name }}"
origin {
name = "upstream"
host_name = "{{ proxy.origin.domain_name }}"
}
{% endif %}
global_delivery_rule {
modify_request_header_action {

1
app/terraform/proxy/fastly.py
View file

@ -11,6 +11,7 @@ class ProxyFastlyAutomation(ProxyAutomation):
short_name = "proxy_fastly"
description = "Deploy proxies to Fastly"
provider = "fastly"
subgroup_members_max = 20
template_parameters = [
"aws_access_key",

12
app/terraform/proxy/lib.py
View file

@ -64,10 +64,10 @@ def all_cdn_prefixes() -> Iterable[str]:
aws = AWS()
prefixes.update(aws.ipv4_ranges)
prefixes.update(aws.ipv6_ranges)
azure = AzureFrontDoorBackend()
prefixes.update(azure.ipv4_ranges)
prefixes.update(azure.ipv6_ranges)
fastly = Fastly()
prefixes.update(fastly.ipv4_ranges)
prefixes.update(fastly.ipv6_ranges)
# azure = AzureFrontDoorBackend()
# prefixes.update(azure.ipv4_ranges)
# prefixes.update(azure.ipv6_ranges)
# fastly = Fastly()
# prefixes.update(fastly.ipv4_ranges)
# prefixes.update(fastly.ipv6_ranges)
return [str(p) for p in prefixes]

90
app/terraform/proxy/meta.py Normal file
View file

@ -0,0 +1,90 @@
import datetime
import logging
import random
import string
from typing import Tuple, List
from tldextract import tldextract
from app import db
from app.models.base import Pool
from app.models.mirrors import Proxy, Origin
from app.terraform import BaseAutomation
from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
from app.terraform.proxy.fastly import ProxyFastlyAutomation
PROXY_PROVIDERS = {p.short_name: p for p in [ # In order of preference
ProxyCloudfrontAutomation,
ProxyFastlyAutomation,
ProxyAzureCdnAutomation
]}
def create_proxy(pool: Pool, origin: Origin) -> bool:
for desperate in [False, True]:
for provider in PROXY_PROVIDERS.values():
if origin.smart and not provider.smart_proxies:
continue # This origin cannot be supported on this provider
if provider.smart_proxies and not (desperate or origin.smart):
continue
next_subgroup = provider.next_subgroup(origin.group_id)
if next_subgroup is None:
continue
proxy = Proxy()
proxy.pool_id = pool.id
proxy.origin_id = origin.id
proxy.provider = provider.provider
proxy.psg = provider.next_subgroup(origin.group_id)
# The random usage below is good enough for its purpose: to create a slug that
# hasn't been used recently.
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
random.choices(string.ascii_lowercase, k=12)) # nosec
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
logging.debug("Creating proxy %s", proxy)
db.session.add(proxy)
return True
return False
class ProxyMetaAutomation(BaseAutomation):
short_name = "proxy_meta"
description = "Housekeeping for proxies"
def automate(self, full: bool = False) -> Tuple[bool, str]:
# Destroy expired proxies
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
proxies: List[Proxy] = Proxy.query.filter(
Proxy.destroyed.is_(None),
Proxy.deprecated < cutoff
).all()
for proxy in proxies:
logging.debug("Destroying expired proxy")
proxy.destroy()
# Deprecate orphaned proxies and mismatched proxies
proxies = Proxy.query.filter(
Proxy.deprecated.is_(None),
Proxy.destroyed.is_(None),
).all()
for proxy in proxies:
if proxy.origin.destroyed is not None:
proxy.deprecate(reason="origin_destroyed")
if proxy.origin.smart and not PROXY_PROVIDERS[proxy.provider].smart_proxies:
proxy.deprecate(reason="not_smart_enough")
# Create new proxies
pools = Pool.query.all()
for pool in pools:
for group in pool.groups:
for origin in group.origins:
if origin.destroyed is not None:
continue
proxies = [
x for x in origin.proxies
if x.pool_id == pool.id and x.deprecated is None and x.destroyed is None
]
if not proxies:
logging.debug("Creating new proxy for %s in pool %s", origin, pool)
create_proxy(pool, origin)
db.session.commit()
return True, ""