diff --git a/app/models/mirrors.py b/app/models/mirrors.py index 62b7f34..e993b6c 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -26,6 +26,7 @@ class Origin(AbstractConfiguration): class Proxy(AbstractResource): origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False) provider = db.Column(db.String(20), nullable=False) + psg = db.Column(db.Integer, nullable=True) slug = db.Column(db.String(20), nullable=True) terraform_updated = db.Column(db.DateTime(), nullable=True) url = db.Column(db.String(255), nullable=True) @@ -36,7 +37,7 @@ class Proxy(AbstractResource): @classmethod def csv_header(self): return super().csv_header() + [ - "origin_id", "provider", "slug", "terraform_updated", "url" + "origin_id", "provider", "psg", "slug", "terraform_updated", "url" ] diff --git a/app/terraform/proxy/azure_cdn.py b/app/terraform/proxy/azure_cdn.py index 06308bd..ed2d139 100644 --- a/app/terraform/proxy/azure_cdn.py +++ b/app/terraform/proxy/azure_cdn.py @@ -1,6 +1,7 @@ import datetime import string import random +from collections import defaultdict from azure.identity import ClientSecretCredential from azure.mgmt.alertsmanagement import AlertsManagementClient @@ -10,7 +11,7 @@ from app import app from app.alarms import get_proxy_alarm from app.extensions import db from app.models.base import Group -from app.models.mirrors import Proxy +from app.models.mirrors import Proxy, Origin from app.models.alarms import AlarmState from app.terraform.proxy import ProxyAutomation @@ -70,8 +71,10 @@ class ProxyAzureCdnAutomation(ProxyAutomation): label_order = ["namespace", "tenant", "name", "attributes"] } - resource "azurerm_cdn_profile" "profile_{{ group.id }}" { + {% for subgroup in subgroups[group.id] %} + resource "azurerm_cdn_profile" "profile_{{ group.id }}_{{ subgroup }}" { name = module.label_{{ group.id }}.id + attributes = ["sub{{ subgroup }}"] location = "{{ azure_location }}" resource_group_name = data.azurerm_resource_group.this.name sku = "Standard_Microsoft" @@ -79,9 +82,9 @@ class ProxyAzureCdnAutomation(ProxyAutomation): tags = module.label_{{ group.id }}.tags } - resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}" { + resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}_{{ subgroup }}" { name = "cdn-diagnostics" - target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}.id + target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id storage_account_id = azurerm_storage_account.this.id log { @@ -105,10 +108,10 @@ class ProxyAzureCdnAutomation(ProxyAutomation): } } - resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}" { - name = "bandwidth-out-high-${module.label_{{ group.id }}.id}" + resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}_{{ subgroup }}" { + name = "bandwidth-out-high-${module.label_{{ group.id }}.id}-sub{{ subgroup }}" resource_group_name = data.azurerm_resource_group.this.name - scopes = [azurerm_cdn_profile.profile_{{ group.id }}.id] + scopes = [azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id] description = "Action will be triggered when response size is too high." criteria { @@ -122,11 +125,12 @@ class ProxyAzureCdnAutomation(ProxyAutomation): window_size = "PT1H" } {% endfor %} + {% endfor %} {% for proxy in proxies %} resource "azurerm_cdn_endpoint" "endpoint_{{ proxy.id }}" { name = "{{ proxy.slug }}" - profile_name = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}.name + profile_name = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}_{{ proxy.psg }}.name location = "{{ azure_location }}" resource_group_name = data.azurerm_resource_group.this.name @@ -164,25 +168,41 @@ class ProxyAzureCdnAutomation(ProxyAutomation): {% endfor %} """ + def get_subgroup_counts(self): + conn = db.engine.connect() + result = conn.execute(""" + SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin + WHERE proxy.origin_id = origin.id + AND proxy.destroyed IS NULL + AND proxy.provider = 'azure_cdn' + GROUP BY origin.group_id, proxy.psg; + """) + subgroups = defaultdict(lambda: defaultdict(lambda: 0)) + for row in result: + subgroups[row[0]][row[1]] = row[2] + return subgroups + def create_missing_proxies(self): groups = Group.query.all() + subgroups = self.get_subgroup_counts() for group in groups: - active_proxies = len([p for p in Proxy.query.filter( - Proxy.provider == 'azure_cdn', - Proxy.destroyed == None - ).all() if p.origin.group_id == group.id]) + subgroup = 0 for origin in group.origins: - if active_proxies == 25: - break - active_proxies += 1 + while True: + if subgroups[group.id][subgroup] >= 25: + subgroup += 1 + else: + break azure_cdn_proxies = [ x for x in origin.proxies if x.provider == "azure_cdn" and x.deprecated is None and x.destroyed is None ] if not azure_cdn_proxies: + subgroups[group.id][subgroup] += 1 proxy = Proxy() proxy.origin_id = origin.id proxy.provider = "azure_cdn" + proxy.psg = subgroup proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join( random.choices(string.ascii_lowercase, k=random.randint(10, 15))) proxy.url = f"https://{proxy.slug}.azureedge.net" @@ -230,8 +250,8 @@ if __name__ == "__main__": auto = ProxyAzureCdnAutomation() auto.create_missing_proxies() auto.destroy_expired_proxies() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply(refresh=False, parallelism=1) # Rate limits are problem - set_urls() - import_monitor_alerts() + # auto.generate_terraform() + # auto.terraform_init() + # auto.terraform_apply(refresh=False, parallelism=1) # Rate limits are problem + # set_urls() + # import_monitor_alerts() diff --git a/migrations/versions/22a33ecf3474_add_provider_subgroup_to_proxy.py b/migrations/versions/22a33ecf3474_add_provider_subgroup_to_proxy.py new file mode 100644 index 0000000..4756620 --- /dev/null +++ b/migrations/versions/22a33ecf3474_add_provider_subgroup_to_proxy.py @@ -0,0 +1,32 @@ +"""add provider subgroup to proxy + +Revision ID: 22a33ecf3474 +Revises: 25092034e059 +Create Date: 2022-04-25 13:33:21.231392 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '22a33ecf3474' +down_revision = '25092034e059' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('proxy', schema=None) as batch_op: + batch_op.add_column(sa.Column('psg', sa.Integer(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('proxy', schema=None) as batch_op: + batch_op.drop_column('psg') + + # ### end Alembic commands ###