From 16f7e2199dcadc89da7a7ef34d0ae346a3abbf37 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Mon, 26 Sep 2022 13:40:59 +0100 Subject: [PATCH] resource pool system --- app/cli/automate.py | 4 +- app/models/base.py | 23 +++++ app/models/mirrors.py | 2 + app/portal/__init__.py | 2 + app/portal/automation.py | 1 + app/portal/forms.py | 5 -- app/portal/list.py | 13 ++- app/portal/pool.py | 78 +++++++++++++++++ app/portal/templates/base.html.j2 | 6 ++ app/portal/templates/icons.html.j2 | 5 ++ app/portal/templates/list.html.j2 | 6 +- app/portal/templates/pool.html.j2 | 21 +++++ app/portal/templates/tables.html.j2 | 23 +++++ app/terraform/proxy/__init__.py | 123 ++++++++++----------------- app/terraform/proxy/azure_cdn.py | 11 +-- app/terraform/proxy/fastly.py | 1 + app/terraform/proxy/lib.py | 12 +-- app/terraform/proxy/meta.py | 90 ++++++++++++++++++++ migrations/versions/45fedef32318_.py | 61 +++++++++++++ 19 files changed, 382 insertions(+), 105 deletions(-) create mode 100644 app/portal/pool.py create mode 100644 app/portal/templates/pool.html.j2 create mode 100644 app/terraform/proxy/meta.py create mode 100644 migrations/versions/45fedef32318_.py diff --git a/app/cli/automate.py b/app/cli/automate.py index d0038dc..3bf4ee0 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -28,6 +28,7 @@ from app.terraform.bridge.ovh import BridgeOvhAutomation from app.terraform.list.github import ListGithubAutomation from app.terraform.list.gitlab import ListGitlabAutomation from app.terraform.list.s3 import ListS3Automation +from app.terraform.proxy.meta import ProxyMetaAutomation from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation from app.terraform.proxy.fastly import ProxyFastlyAutomation @@ -56,7 +57,8 @@ jobs = { ListS3Automation, ProxyAzureCdnAutomation, ProxyCloudfrontAutomation, - ProxyFastlyAutomation + ProxyFastlyAutomation, + ProxyMetaAutomation ] } diff --git a/app/models/base.py b/app/models/base.py index d7fb5b6..72b0b6a 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -14,6 +14,7 @@ class Group(AbstractConfiguration): eotks = db.relationship("Eotk", back_populates="group") onions = db.relationship("Onion", back_populates="group") smart_proxies = db.relationship("SmartProxy", back_populates="group") + pools = db.relationship("Pool", secondary="pool_group", back_populates="groups") @classmethod def csv_header(cls) -> List[str]: @@ -22,7 +23,27 @@ class Group(AbstractConfiguration): ] +class Pool(AbstractConfiguration): + pool_name = db.Column(db.String(80), unique=True, nullable=False) + + @classmethod + def csv_header(cls) -> List[str]: + return super().csv_header() + [ + "pool_name" + ] + + proxies = db.relationship("Proxy", back_populates="pool") + lists = db.relationship("MirrorList", back_populates="pool") + groups = db.relationship("Group", secondary="pool_group", back_populates="pools") + + +class PoolGroup(db.Model): # type: ignore[misc] + pool_id = db.Column(db.Integer, db.ForeignKey("pool.id"), primary_key=True) + group_id = db.Column(db.Integer, db.ForeignKey("group.id"), primary_key=True) + + class MirrorList(AbstractConfiguration): + pool_id = db.Column(db.Integer, db.ForeignKey("pool.id")) provider = db.Column(db.String(255), nullable=False) format = db.Column(db.String(20), nullable=False) encoding = db.Column(db.String(20), nullable=False) @@ -31,6 +52,8 @@ class MirrorList(AbstractConfiguration): role = db.Column(db.String(255), nullable=True) filename = db.Column(db.String(255), nullable=False) + pool = db.relationship("Pool", back_populates="lists") + providers_supported = { "github": "GitHub", "gitlab": "GitLab", diff --git a/app/models/mirrors.py b/app/models/mirrors.py index 022d80f..669c224 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -50,6 +50,7 @@ class Origin(AbstractConfiguration): class Proxy(AbstractResource): origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False) + pool_id = db.Column(db.Integer, db.ForeignKey("pool.id")) provider = db.Column(db.String(20), nullable=False) psg = db.Column(db.Integer, nullable=True) slug = db.Column(db.String(20), nullable=True) @@ -57,6 +58,7 @@ class Proxy(AbstractResource): url = db.Column(db.String(255), nullable=True) origin = db.relationship("Origin", back_populates="proxies") + pool = db.relationship("Pool", back_populates="proxies") @property def brn(self) -> BRN: diff --git a/app/portal/__init__.py b/app/portal/__init__.py index 719e36b..aff3ff1 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -21,6 +21,7 @@ from app.portal.group import bp as group from app.portal.list import bp as list_ from app.portal.origin import bp as origin from app.portal.onion import bp as onion +from app.portal.pool import bp as pool from app.portal.proxy import bp as proxy from app.portal.smart_proxy import bp as smart_proxy from app.portal.webhook import bp as webhook @@ -34,6 +35,7 @@ portal.register_blueprint(group, url_prefix="/group") portal.register_blueprint(list_, url_prefix="/list") portal.register_blueprint(origin, url_prefix="/origin") portal.register_blueprint(onion, url_prefix="/onion") +portal.register_blueprint(pool, url_prefix="/pool") portal.register_blueprint(proxy, url_prefix="/proxy") portal.register_blueprint(smart_proxy, url_prefix="/smart") portal.register_blueprint(webhook, url_prefix="/webhook") diff --git a/app/portal/automation.py b/app/portal/automation.py index abb7d01..4f60a2c 100644 --- a/app/portal/automation.py +++ b/app/portal/automation.py @@ -74,6 +74,7 @@ def automation_kick(automation_id: int) -> ResponseReturnValue: return view_lifecycle( header="Kick automation timer?", message=automation.description, + section="automation", success_view="portal.automation.automation_list", success_message="This automation job will next run within 1 minute.", resource=automation, diff --git a/app/portal/forms.py b/app/portal/forms.py index e2a6d6f..0f053d7 100644 --- a/app/portal/forms.py +++ b/app/portal/forms.py @@ -6,8 +6,3 @@ class EditMirrorForm(FlaskForm): # type: ignore origin = SelectField('Origin') url = StringField('URL') submit = SubmitField('Save Changes') - - -class EditProxyForm(FlaskForm): # type: ignore - origin = SelectField('Origin') - submit = SubmitField('Save Changes') diff --git a/app/portal/list.py b/app/portal/list.py index 77e7cb7..4475323 100644 --- a/app/portal/list.py +++ b/app/portal/list.py @@ -1,6 +1,6 @@ import json from datetime import datetime -from typing import Optional +from typing import Optional, Any from flask import render_template, url_for, flash, redirect, Blueprint, Response from flask.typing import ResponseReturnValue @@ -13,7 +13,7 @@ from app.extensions import db from app.lists.bc2 import mirror_sites from app.lists.bridgelines import bridgelines from app.lists.mirror_mapping import mirror_mapping -from app.models.base import MirrorList +from app.models.base import MirrorList, Pool from app.portal.util import response_404, view_lifecycle bp = Blueprint("list", __name__) @@ -96,6 +96,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue: form.encoding.choices = list(MirrorList.encodings_supported.items()) if form.validate_on_submit(): list_ = MirrorList() + list_.pool_id = form.pool.data list_.provider = form.provider.data list_.format = form.format.data list_.encoding = form.encoding.data @@ -122,6 +123,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue: class NewMirrorListForm(FlaskForm): # type: ignore + pool = SelectField('Resource Pool', validators=[DataRequired()]) provider = SelectField('Provider', validators=[DataRequired()]) format = SelectField('Distribution Method', validators=[DataRequired()]) encoding = SelectField('Encoding', validators=[DataRequired()]) @@ -136,6 +138,12 @@ class NewMirrorListForm(FlaskForm): # type: ignore filename = StringField('Filename', validators=[DataRequired()]) submit = SubmitField('Save Changes') + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.pool.choices = [ + (pool.id, pool.pool_name) for pool in Pool.query.all() + ] + @bp.route('/edit/', methods=['GET', 'POST']) def list_edit(list_id: int) -> ResponseReturnValue: @@ -160,6 +168,7 @@ def list_edit(list_id: int) -> ResponseReturnValue: form.format.choices = list(MirrorList.formats_supported.items()) form.encoding.choices = list(MirrorList.encodings_supported.items()) if form.validate_on_submit(): + list_.pool_id = form.pool.data list_.provider = form.provider.data list_.format = form.format.data list_.encoding = form.encoding.data diff --git a/app/portal/pool.py b/app/portal/pool.py new file mode 100644 index 0000000..b2c9422 --- /dev/null +++ b/app/portal/pool.py @@ -0,0 +1,78 @@ +from datetime import datetime + +from flask import render_template, url_for, flash, redirect, Response, Blueprint +from flask.typing import ResponseReturnValue +from flask_wtf import FlaskForm +import sqlalchemy +from wtforms import StringField, SubmitField +from wtforms.validators import DataRequired + +from app.extensions import db +from app.models.base import Pool + +bp = Blueprint("pool", __name__) + + +class NewPoolForm(FlaskForm): # type: ignore + group_name = StringField("Short Name", validators=[DataRequired()]) + description = StringField("Description", validators=[DataRequired()]) + submit = SubmitField('Save Changes', render_kw={"class": "btn btn-success"}) + + +class EditPoolForm(FlaskForm): # type: ignore + description = StringField("Description", validators=[DataRequired()]) + submit = SubmitField('Save Changes', render_kw={"class": "btn btn-success"}) + + +@bp.route("/list") +def pool_list() -> ResponseReturnValue: + pools = Pool.query.order_by(Pool.pool_name).all() + return render_template("list.html.j2", + section="pool", + title="Resource Pools", + item="pool", + items=pools, + new_link=url_for("portal.pool.pool_new")) + + +@bp.route("/new", methods=['GET', 'POST']) +def pool_new() -> ResponseReturnValue: + form = NewPoolForm() + if form.validate_on_submit(): + pool = Pool() + pool.pool_name = form.group_name.data + pool.description = form.description.data + pool.created = datetime.utcnow() + pool.updated = datetime.utcnow() + try: + db.session.add(pool) + db.session.commit() + flash(f"Created new pool {pool.pool_name}.", "success") + return redirect(url_for("portal.pool.pool_edit", pool_id=pool.id)) + except sqlalchemy.exc.SQLAlchemyError: + flash("Failed to create new pool.", "danger") + return redirect(url_for("portal.pool.pool_list")) + return render_template("new.html.j2", section="pool", form=form) + + +@bp.route('/edit/', methods=['GET', 'POST']) +def pool_edit(pool_id: int) -> ResponseReturnValue: + pool = Pool.query.filter(Pool.id == pool_id).first() + if pool is None: + return Response(render_template("error.html.j2", + section="pool", + header="404 Pool Not Found", + message="The requested pool could not be found."), + status=404) + form = EditPoolForm(description=pool.description) + if form.validate_on_submit(): + pool.description = form.description.data + pool.updated = datetime.utcnow() + try: + db.session.commit() + flash("Saved changes to pool.", "success") + except sqlalchemy.exc.SQLAlchemyError: + flash("An error occurred saving the changes to the pool.", "danger") + return render_template("pool.html.j2", + section="pool", + pool=pool, form=form) diff --git a/app/portal/templates/base.html.j2 b/app/portal/templates/base.html.j2 index 91688c2..6215318 100644 --- a/app/portal/templates/base.html.j2 +++ b/app/portal/templates/base.html.j2 @@ -87,6 +87,12 @@ {{ icon("collection") }} Groups +
  • + + {{ icon("stack") }} Resource Pools + +
  • diff --git a/app/portal/templates/icons.html.j2 b/app/portal/templates/icons.html.j2 index f9bc666..139caba 100644 --- a/app/portal/templates/icons.html.j2 +++ b/app/portal/templates/icons.html.j2 @@ -98,5 +98,10 @@ + {% elif i == "stack" %} + + + + {% endif %} {% endmacro %} \ No newline at end of file diff --git a/app/portal/templates/list.html.j2 b/app/portal/templates/list.html.j2 index 3b62dcd..961d493 100644 --- a/app/portal/templates/list.html.j2 +++ b/app/portal/templates/list.html.j2 @@ -1,7 +1,7 @@ {% extends "base.html.j2" %} {% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, eotk_table, - groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, proxies_table, - webhook_table %} + groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, pools_table, + proxies_table, webhook_table %} {% block content %}

    {{ title }}

    @@ -33,6 +33,8 @@ {% endif %} {% elif item == "origin" %} {{ origins_table(items) }} + {% elif item == "pool" %} + {{ pools_table(items) }} {% elif item == "proxy" %} {{ proxies_table(items) }} {% elif item == "smart proxy" %} diff --git a/app/portal/templates/pool.html.j2 b/app/portal/templates/pool.html.j2 new file mode 100644 index 0000000..c6c0104 --- /dev/null +++ b/app/portal/templates/pool.html.j2 @@ -0,0 +1,21 @@ +{% extends "base.html.j2" %} +{% from 'bootstrap5/form.html' import render_form %} +{% from "tables.html.j2" import groups_table, mirrorlists_table, proxies_table %} + +{% block content %} +

    Resource Pool

    +

    {{ pool.pool_name }}

    + +
    + {{ render_form(form) }} +
    + +

    Groups

    + {{ groups_table(pool.groups) }} + +

    Distribution Lists

    + {{ mirrorlists_table(pool.lists) }} + +

    Simple Proxies

    + {{ proxies_table(pool.proxies) }} +{% endblock %} diff --git a/app/portal/templates/tables.html.j2 b/app/portal/templates/tables.html.j2 index 8f00bb7..baa81ae 100644 --- a/app/portal/templates/tables.html.j2 +++ b/app/portal/templates/tables.html.j2 @@ -334,6 +334,29 @@ {% endmacro %} +{% macro pools_table(pools) %} +
    + + + + + + + + + + {% for pool in pools %} + + + + + + {% endfor %} + +
    NameDescriptionActions
    {{ pool.pool_name }}{{ pool.description }}View/Edit
    +
    +{% endmacro %} + {% macro proxies_table(proxies) %}
    diff --git a/app/terraform/proxy/__init__.py b/app/terraform/proxy/__init__.py index 76d44ba..615e392 100644 --- a/app/terraform/proxy/__init__.py +++ b/app/terraform/proxy/__init__.py @@ -1,14 +1,11 @@ import os.path +import sys from abc import abstractmethod -from collections import defaultdict import datetime -import math -import string -import random -from typing import Dict, Optional, Any, List +from collections import defaultdict +from typing import Optional, Any, List, Dict from sqlalchemy import text -from tldextract import tldextract from app import app from app.extensions import db @@ -44,13 +41,19 @@ def sp_trusted_prefixes() -> str: class ProxyAutomation(TerraformAutomation): - subgroup_max = math.inf + subgroup_members_max = sys.maxsize """ Maximum number of proxies to deploy per sub-group. This is required for some providers where the number origins per group may exceed the number of proxies that can be configured in a single "configuration block", e.g. Azure CDN's profiles. """ + subgroup_count_max = sys.maxsize + """ + Maximum number of subgroups that can be deployed. This is required for some providers where + the total number of subgroups is limited by a quota, e.g. Azure CDN's profiles. + """ + template: str """ Terraform configuration template using Jinja 2. @@ -67,83 +70,11 @@ class ProxyAutomation(TerraformAutomation): Whether this provider supports "smart" proxies. """ - def get_subgroups(self) -> Dict[int, Dict[int, int]]: - conn = db.engine.connect() - result = conn.execute(text(""" - SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin - WHERE proxy.origin_id = origin.id - AND proxy.destroyed IS NULL - AND proxy.provider = :provider - GROUP BY origin.group_id, proxy.psg; - """), provider=self.provider) - subgroups: Dict[int, Dict[int, int]] = defaultdict(lambda: defaultdict(lambda: 0)) - for row in result: - subgroups[row[0]][row[1]] = row[2] - return subgroups - - def create_missing_proxies(self) -> None: - groups = Group.query.all() - subgroups = self.get_subgroups() - for group in groups: - subgroup = 0 - for origin in group.origins: - if origin.destroyed is not None: - continue - while True: - if subgroups[group.id][subgroup] >= self.subgroup_max: - subgroup += 1 - else: - break - proxies = [ - x for x in origin.proxies - if x.provider == self.provider and x.deprecated is None and x.destroyed is None - ] - if not proxies: - subgroups[group.id][subgroup] += 1 - proxy = Proxy() - proxy.origin_id = origin.id - proxy.provider = self.provider - proxy.psg = subgroup - # The random usage below is good enough for its purpose: to create a slug that - # hasn't been used before. - proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join( - random.choices(string.ascii_lowercase, k=12)) # nosec - proxy.added = datetime.datetime.utcnow() - proxy.updated = datetime.datetime.utcnow() - db.session.add(proxy) - db.session.commit() - - def deprecate_orphaned_proxies(self) -> None: - proxies = Proxy.query.filter( - Proxy.deprecated.is_(None), - Proxy.destroyed.is_(None), - Proxy.provider == self.provider - ).all() - for proxy in proxies: - if proxy.origin.destroyed is not None: - proxy.deprecate(reason="origin_destroyed") - db.session.commit() - - def destroy_expired_proxies(self) -> None: - cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3) - proxies = Proxy.query.filter( - Proxy.destroyed.is_(None), - Proxy.provider == self.provider, - Proxy.deprecated < cutoff - ).all() - for proxy in proxies: - proxy.destroyed = datetime.datetime.utcnow() - proxy.updated = datetime.datetime.utcnow() - db.session.commit() - @abstractmethod def import_state(self, state: Any) -> None: raise NotImplementedError() def tf_prehook(self) -> Optional[Any]: # pylint: disable=useless-return - self.create_missing_proxies() - self.deprecate_orphaned_proxies() - self.destroy_expired_proxies() return None def tf_posthook(self, *, prehook_result: Any = None) -> None: @@ -223,3 +154,37 @@ class ProxyAutomation(TerraformAutomation): provider=self.provider, origins=group_origins, smart_zone=app.config['SMART_ZONE']) + + @classmethod + def get_subgroups(cls) -> Dict[int, Dict[int, int]]: + conn = db.engine.connect() + result = conn.execute(text(""" + SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin + WHERE proxy.origin_id = origin.id + AND proxy.destroyed IS NULL + AND proxy.provider = :provider + GROUP BY origin.group_id, proxy.psg; + """), provider=cls.provider) + subgroups: Dict[int, Dict[int, int]] = defaultdict(lambda: defaultdict(lambda: 0)) + for row in result: + subgroups[row[0]][row[1]] = row[2] + return subgroups + + @classmethod + def next_subgroup(cls, group_id: int) -> Optional[int]: + conn = db.engine.connect() + result = conn.execute(text(""" + SELECT proxy.psg, COUNT(proxy.id) FROM proxy, origin + WHERE proxy.origin_id = origin.id + AND proxy.destroyed IS NULL + AND origin.group_id = :group_id + AND proxy.provider = :provider + GROUP BY proxy.psg ORDER BY proxy.psg; + """), provider=cls.short_name, group_id=group_id) + subgroups = { + row[0]: row[1] for row in result + } + for subgroup in range(0, cls.subgroup_count_max): + if subgroups.get(subgroup, 0) < cls.subgroup_members_max: + return subgroup + return None diff --git a/app/terraform/proxy/azure_cdn.py b/app/terraform/proxy/azure_cdn.py index fcd8849..3e151e0 100644 --- a/app/terraform/proxy/azure_cdn.py +++ b/app/terraform/proxy/azure_cdn.py @@ -9,7 +9,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation): short_name = "proxy_azure_cdn" description = "Deploy proxies to Azure CDN" provider = "azure_cdn" - subgroup_max = 25 + subgroup_members_max = 25 parallelism = 1 template_parameters = [ @@ -125,21 +125,12 @@ class ProxyAzureCdnAutomation(ProxyAutomation): location = "{{ azure_location }}" resource_group_name = data.azurerm_resource_group.this.name - {% if proxy.origin.smart %} - origin_host_header = "origin-{{ proxy.origin.id }}.cloudfront.smart.{{ smart_zone[:-1] }}" - - origin { - name = "upstream" - host_name = "origin-{{ proxy.origin.id }}.cloudfront.smart.{{ smart_zone[:-1] }}" - } - {% else %} origin_host_header = "{{ proxy.origin.domain_name }}" origin { name = "upstream" host_name = "{{ proxy.origin.domain_name }}" } - {% endif %} global_delivery_rule { modify_request_header_action { diff --git a/app/terraform/proxy/fastly.py b/app/terraform/proxy/fastly.py index 18d59d6..4a0da57 100644 --- a/app/terraform/proxy/fastly.py +++ b/app/terraform/proxy/fastly.py @@ -11,6 +11,7 @@ class ProxyFastlyAutomation(ProxyAutomation): short_name = "proxy_fastly" description = "Deploy proxies to Fastly" provider = "fastly" + subgroup_members_max = 20 template_parameters = [ "aws_access_key", diff --git a/app/terraform/proxy/lib.py b/app/terraform/proxy/lib.py index b155f65..49f5e47 100644 --- a/app/terraform/proxy/lib.py +++ b/app/terraform/proxy/lib.py @@ -64,10 +64,10 @@ def all_cdn_prefixes() -> Iterable[str]: aws = AWS() prefixes.update(aws.ipv4_ranges) prefixes.update(aws.ipv6_ranges) - azure = AzureFrontDoorBackend() - prefixes.update(azure.ipv4_ranges) - prefixes.update(azure.ipv6_ranges) - fastly = Fastly() - prefixes.update(fastly.ipv4_ranges) - prefixes.update(fastly.ipv6_ranges) + # azure = AzureFrontDoorBackend() + # prefixes.update(azure.ipv4_ranges) + # prefixes.update(azure.ipv6_ranges) + # fastly = Fastly() + # prefixes.update(fastly.ipv4_ranges) + # prefixes.update(fastly.ipv6_ranges) return [str(p) for p in prefixes] diff --git a/app/terraform/proxy/meta.py b/app/terraform/proxy/meta.py new file mode 100644 index 0000000..b715693 --- /dev/null +++ b/app/terraform/proxy/meta.py @@ -0,0 +1,90 @@ +import datetime +import logging +import random +import string +from typing import Tuple, List + +from tldextract import tldextract + +from app import db +from app.models.base import Pool +from app.models.mirrors import Proxy, Origin +from app.terraform import BaseAutomation +from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation +from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation +from app.terraform.proxy.fastly import ProxyFastlyAutomation + +PROXY_PROVIDERS = {p.short_name: p for p in [ # In order of preference + ProxyCloudfrontAutomation, + ProxyFastlyAutomation, + ProxyAzureCdnAutomation +]} + + +def create_proxy(pool: Pool, origin: Origin) -> bool: + for desperate in [False, True]: + for provider in PROXY_PROVIDERS.values(): + if origin.smart and not provider.smart_proxies: + continue # This origin cannot be supported on this provider + if provider.smart_proxies and not (desperate or origin.smart): + continue + next_subgroup = provider.next_subgroup(origin.group_id) + if next_subgroup is None: + continue + proxy = Proxy() + proxy.pool_id = pool.id + proxy.origin_id = origin.id + proxy.provider = provider.provider + proxy.psg = provider.next_subgroup(origin.group_id) + # The random usage below is good enough for its purpose: to create a slug that + # hasn't been used recently. + proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join( + random.choices(string.ascii_lowercase, k=12)) # nosec + proxy.added = datetime.datetime.utcnow() + proxy.updated = datetime.datetime.utcnow() + logging.debug("Creating proxy %s", proxy) + db.session.add(proxy) + return True + return False + + +class ProxyMetaAutomation(BaseAutomation): + short_name = "proxy_meta" + description = "Housekeeping for proxies" + + def automate(self, full: bool = False) -> Tuple[bool, str]: + # Destroy expired proxies + cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3) + proxies: List[Proxy] = Proxy.query.filter( + Proxy.destroyed.is_(None), + Proxy.deprecated < cutoff + ).all() + for proxy in proxies: + logging.debug("Destroying expired proxy") + proxy.destroy() + # Deprecate orphaned proxies and mismatched proxies + proxies = Proxy.query.filter( + Proxy.deprecated.is_(None), + Proxy.destroyed.is_(None), + ).all() + for proxy in proxies: + if proxy.origin.destroyed is not None: + proxy.deprecate(reason="origin_destroyed") + if proxy.origin.smart and not PROXY_PROVIDERS[proxy.provider].smart_proxies: + proxy.deprecate(reason="not_smart_enough") + # Create new proxies + pools = Pool.query.all() + for pool in pools: + for group in pool.groups: + for origin in group.origins: + if origin.destroyed is not None: + continue + proxies = [ + x for x in origin.proxies + if x.pool_id == pool.id and x.deprecated is None and x.destroyed is None + ] + if not proxies: + logging.debug("Creating new proxy for %s in pool %s", origin, pool) + create_proxy(pool, origin) + db.session.commit() + return True, "" diff --git a/migrations/versions/45fedef32318_.py b/migrations/versions/45fedef32318_.py new file mode 100644 index 0000000..df5c945 --- /dev/null +++ b/migrations/versions/45fedef32318_.py @@ -0,0 +1,61 @@ +"""empty message + +Revision ID: 45fedef32318 +Revises: 665e340dbe09 +Create Date: 2022-09-07 16:20:04.603554 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '45fedef32318' +down_revision = '665e340dbe09' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('pool', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.String(length=255), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('destroyed', sa.DateTime(), nullable=True), + sa.Column('pool_name', sa.String(length=80), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_pool')), + sa.UniqueConstraint('pool_name', name=op.f('uq_pool_pool_name')) + ) + op.create_table('pool_group', + sa.Column('pool_id', sa.Integer(), nullable=False), + sa.Column('group_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_pool_group_group_id_group')), + sa.ForeignKeyConstraint(['pool_id'], ['pool.id'], name=op.f('fk_pool_group_pool_id_pool')), + sa.PrimaryKeyConstraint('pool_id', 'group_id', name=op.f('pk_pool_group')) + ) + with op.batch_alter_table('mirror_list', schema=None) as batch_op: + batch_op.add_column(sa.Column('pool_id', sa.Integer(), nullable=True)) + batch_op.create_foreign_key(batch_op.f('fk_mirror_list_pool_id_pool'), 'pool', ['pool_id'], ['id']) + + with op.batch_alter_table('proxy', schema=None) as batch_op: + batch_op.add_column(sa.Column('pool_id', sa.Integer(), nullable=True)) + batch_op.create_foreign_key(batch_op.f('fk_proxy_pool_id_pool'), 'pool', ['pool_id'], ['id']) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('proxy', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('fk_proxy_pool_id_pool'), type_='foreignkey') + batch_op.drop_column('pool_id') + + with op.batch_alter_table('mirror_list', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('fk_mirror_list_pool_id_pool'), type_='foreignkey') + batch_op.drop_column('pool_id') + + op.drop_table('pool_group') + op.drop_table('pool') + # ### end Alembic commands ###