import os.path import sys from abc import abstractmethod import datetime from collections import defaultdict from typing import Optional, Any, List, Dict from flask import current_app from sqlalchemy import text from app import app from app.extensions import db from app.models.base import Group from app.models.mirrors import Proxy, Origin, SmartProxy from app.terraform.terraform import TerraformAutomation def update_smart_proxy_instance(group_id: int, provider: str, region: str, instance_id: str) -> None: instance = SmartProxy.query.filter( SmartProxy.group_id == group_id, SmartProxy.region == region, SmartProxy.provider == provider, SmartProxy.destroyed.is_(None) ).first() if instance is None: instance = SmartProxy() instance.added = datetime.datetime.utcnow() instance.group_id = group_id instance.provider = provider instance.region = region db.session.add(instance) instance.updated = datetime.datetime.utcnow() instance.instance_id = instance_id class ProxyAutomation(TerraformAutomation): subgroup_members_max = sys.maxsize """ Maximum number of proxies to deploy per sub-group. This is required for some providers where the number origins per group may exceed the number of proxies that can be configured in a single "configuration block", e.g. Azure CDN's profiles. """ subgroup_count_max = sys.maxsize """ Maximum number of subgroups that can be deployed. This is required for some providers where the total number of subgroups is limited by a quota, e.g. Azure CDN's profiles. """ template: str """ Terraform configuration template using Jinja 2. """ template_parameters: List[str] """ List of parameters to be read from the application configuration for use in the templating of the Terraform configuration. """ smart_proxies = False """ Whether this provider supports "smart" proxies. """ cloud_name: str """ The name of the cloud provider used by this proxy provider. Used to determine if this provider can be enabled. """ @abstractmethod def import_state(self, state: Any) -> None: raise NotImplementedError() def enabled(self) -> bool: return current_app.config.get(self.cloud_name.upper() + "_ENABLED", False) def tf_prehook(self) -> Optional[Any]: # pylint: disable=useless-return return None def tf_posthook(self, *, prehook_result: Any = None) -> None: self.import_state(self.tf_show()) def tf_generate(self) -> None: groups = Group.query.all() self.tf_write( self.template, groups=groups, proxies=Proxy.query.filter( Proxy.provider == self.provider, Proxy.destroyed.is_(None)).all(), subgroups=self.get_subgroups(), global_namespace=app.config['GLOBAL_NAMESPACE'], bypass_token=app.config['BYPASS_TOKEN'], terraform_modules_path=os.path.join(*list(os.path.split(app.root_path))[:-1], 'terraform-modules'), backend_config=f"""backend "http" {{ lock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" unlock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" }}""", **{k: app.config[k.upper()] for k in self.template_parameters}) if self.smart_proxies: for group in groups: self.sp_config(group) def sp_config(self, group: Group) -> None: group_origins: List[Origin] = Origin.query.filter( Origin.group_id == group.id, Origin.destroyed.is_(None), Origin.smart.is_(True) ).all() self.tmpl_write(f"smart_proxy.{group.id}.conf", """ {% for origin in origins %} server { listen 443 ssl; server_name origin-{{ origin.id }}.{{ provider }}.smart.{{ smart_zone[:-1] }}; location / { proxy_set_header Accept-Encoding ""; proxy_ssl_server_name on; proxy_pass https://{{ origin.domain_name }}/; subs_filter_types text/html text/css text/xml; subs_filter https://{{ origin.domain_name }}/ /; subs_filter "([^:]|)\\\"https://{{ origin.domain_name }}\\\"" \\1\\\"/\\\"; {%- for asset_origin in origin.group.origins | selectattr("assets") -%} {%- for asset_proxy in asset_origin.proxies | selectattr("provider", "equalto", provider) | selectattr("deprecated", "none") | selectattr("destroyed", "none") -%} {%- if loop.first %} subs_filter https://{{ asset_origin.domain_name }}/ {{ asset_proxy.url }}/; {%- endif -%} {%- endfor -%} {%- endfor %} } ssl_certificate /etc/ssl/smart_proxy.crt; ssl_certificate_key /etc/ssl/private/smart_proxy.key; } {% endfor %} """, provider=self.provider, origins=group_origins, smart_zone=app.config['SMART_ZONE']) @classmethod def get_subgroups(cls) -> Dict[int, Dict[int, int]]: conn = db.engine.connect() result = conn.execute(text(""" SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin WHERE proxy.origin_id = origin.id AND proxy.destroyed IS NULL AND proxy.provider = :provider GROUP BY origin.group_id, proxy.psg; """), provider=cls.provider) subgroups: Dict[int, Dict[int, int]] = defaultdict(lambda: defaultdict(lambda: 0)) for row in result: subgroups[row[0]][row[1]] = row[2] return subgroups @classmethod def next_subgroup(cls, group_id: int) -> Optional[int]: conn = db.engine.connect() result = conn.execute(text(""" SELECT proxy.psg, COUNT(proxy.id) FROM proxy, origin WHERE proxy.origin_id = origin.id AND proxy.destroyed IS NULL AND origin.group_id = :group_id AND proxy.provider = :provider GROUP BY proxy.psg ORDER BY proxy.psg; """), provider=cls.short_name, group_id=group_id) subgroups = { row[0]: row[1] for row in result } for subgroup in range(0, cls.subgroup_count_max): if subgroups.get(subgroup, 0) < cls.subgroup_members_max: return subgroup return None