azure_cdn: use subgroups, attempt 1

app/models/mirrors.py

@@ -26,6 +26,7 @@ class Origin(AbstractConfiguration):
 class Proxy(AbstractResource):
     origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)
     provider = db.Column(db.String(20), nullable=False)
+    psg = db.Column(db.Integer, nullable=True)
     slug = db.Column(db.String(20), nullable=True)
     terraform_updated = db.Column(db.DateTime(), nullable=True)
     url = db.Column(db.String(255), nullable=True)
@@ -36,7 +37,7 @@ class Proxy(AbstractResource):
     @classmethod
     def csv_header(self):
         return super().csv_header() + [
-            "origin_id", "provider", "slug", "terraform_updated", "url"
+            "origin_id", "provider", "psg", "slug", "terraform_updated", "url"
         ]
 
 

app/terraform/proxy/azure_cdn.py

@@ -1,6 +1,7 @@
 import datetime
 import string
 import random
+from collections import defaultdict
 
 from azure.identity import ClientSecretCredential
 from azure.mgmt.alertsmanagement import AlertsManagementClient
@@ -10,7 +11,7 @@ from app import app
 from app.alarms import get_proxy_alarm
 from app.extensions import db
 from app.models.base import Group
-from app.models.mirrors import Proxy
+from app.models.mirrors import Proxy, Origin
 from app.models.alarms import AlarmState
 from app.terraform.proxy import ProxyAutomation
 
@@ -70,8 +71,10 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
       label_order = ["namespace", "tenant", "name", "attributes"]
     }
     
-    resource "azurerm_cdn_profile" "profile_{{ group.id }}" {
+    {% for subgroup in subgroups[group.id] %}
+    resource "azurerm_cdn_profile" "profile_{{ group.id }}_{{ subgroup }}" {
       name                = module.label_{{ group.id }}.id
+      attributes          = ["sub{{ subgroup }}"]
       location            = "{{ azure_location }}"
       resource_group_name = data.azurerm_resource_group.this.name
       sku                 = "Standard_Microsoft"
@@ -79,9 +82,9 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
       tags = module.label_{{ group.id }}.tags
     }
     
-    resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}" {
+    resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}_{{ subgroup }}" {
       name               = "cdn-diagnostics"
-      target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}.id
+      target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id
       storage_account_id = azurerm_storage_account.this.id
     
       log {
@@ -105,10 +108,10 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
       }
     }
     
-    resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}" {
+    resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}_{{ subgroup }}" {
-      name                = "bandwidth-out-high-${module.label_{{ group.id }}.id}"
+      name                = "bandwidth-out-high-${module.label_{{ group.id }}.id}-sub{{ subgroup }}"
       resource_group_name = data.azurerm_resource_group.this.name
-      scopes              = [azurerm_cdn_profile.profile_{{ group.id }}.id]
+      scopes              = [azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id]
       description         = "Action will be triggered when response size is too high."
     
       criteria {
@@ -122,11 +125,12 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
       window_size = "PT1H"
     }
     {% endfor %}
+    {% endfor %}
     
     {% for proxy in proxies %}
     resource "azurerm_cdn_endpoint" "endpoint_{{ proxy.id }}" {
       name                = "{{ proxy.slug }}"
-      profile_name        = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}.name
+      profile_name        = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}_{{ proxy.psg }}.name
       location            = "{{ azure_location }}"
       resource_group_name = data.azurerm_resource_group.this.name
     
@@ -164,25 +168,41 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
     {% endfor %}
     """
 
+    def get_subgroup_counts(self):
+        conn = db.engine.connect()
+        result = conn.execute("""
+        SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin
+        WHERE proxy.origin_id = origin.id
+            AND proxy.destroyed IS NULL
+            AND proxy.provider = 'azure_cdn'
+        GROUP BY origin.group_id, proxy.psg;
+        """)
+        subgroups = defaultdict(lambda: defaultdict(lambda: 0))
+        for row in result:
+            subgroups[row[0]][row[1]] = row[2]
+        return subgroups
+
     def create_missing_proxies(self):
         groups = Group.query.all()
+        subgroups = self.get_subgroup_counts()
         for group in groups:
-            active_proxies = len([p for p in Proxy.query.filter(
+            subgroup = 0
-                Proxy.provider == 'azure_cdn',
-                Proxy.destroyed == None
-            ).all() if p.origin.group_id == group.id])
             for origin in group.origins:
-                if active_proxies == 25:
+                while True:
-                    break
+                    if subgroups[group.id][subgroup] >= 25:
-                active_proxies += 1
+                        subgroup += 1
+                    else:
+                        break
                 azure_cdn_proxies = [
                     x for x in origin.proxies
                     if x.provider == "azure_cdn" and x.deprecated is None and x.destroyed is None
                 ]
                 if not azure_cdn_proxies:
+                    subgroups[group.id][subgroup] += 1
                     proxy = Proxy()
                     proxy.origin_id = origin.id
                     proxy.provider = "azure_cdn"
+                    proxy.psg = subgroup
                     proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
                         random.choices(string.ascii_lowercase, k=random.randint(10, 15)))
                     proxy.url = f"https://{proxy.slug}.azureedge.net"
@@ -230,8 +250,8 @@ if __name__ == "__main__":
         auto = ProxyAzureCdnAutomation()
         auto.create_missing_proxies()
         auto.destroy_expired_proxies()
-        auto.generate_terraform()
+        # auto.generate_terraform()
-        auto.terraform_init()
+        # auto.terraform_init()
-        auto.terraform_apply(refresh=False, parallelism=1)  # Rate limits are problem
+        # auto.terraform_apply(refresh=False, parallelism=1)  # Rate limits are problem
-        set_urls()
+        # set_urls()
-        import_monitor_alerts()
+        # import_monitor_alerts()

migrations/versions/22a33ecf3474_add_provider_subgroup_to_proxy.py

@@ -0,0 +1,32 @@
+"""add provider subgroup to proxy
+
+Revision ID: 22a33ecf3474
+Revises: 25092034e059
+Create Date: 2022-04-25 13:33:21.231392
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '22a33ecf3474'
+down_revision = '25092034e059'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('proxy', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('psg', sa.Integer(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('proxy', schema=None) as batch_op:
+        batch_op.drop_column('psg')
+
+    # ### end Alembic commands ###