jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

azure_cdn: use subgroups, attempt 1

Browse source
This commit is contained in:
Iain Learmonth 2022-04-25 14:56:35 +01:00
parent b189aa8efd
commit b59d334689
3 changed files with 74 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

3
app/models/mirrors.py
View file

@ -26,6 +26,7 @@ class Origin(AbstractConfiguration):
class Proxy(AbstractResource):
origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)
provider = db.Column(db.String(20), nullable=False)
psg = db.Column(db.Integer, nullable=True)
slug = db.Column(db.String(20), nullable=True)
terraform_updated = db.Column(db.DateTime(), nullable=True)
url = db.Column(db.String(255), nullable=True)
@ -36,7 +37,7 @@ class Proxy(AbstractResource):
@classmethod
def csv_header(self):
return super().csv_header() + [
"origin_id", "provider", "slug", "terraform_updated", "url"
"origin_id", "provider", "psg", "slug", "terraform_updated", "url"
]

60
app/terraform/proxy/azure_cdn.py
View file

@ -1,6 +1,7 @@
import datetime
import string
import random
from collections import defaultdict
from azure.identity import ClientSecretCredential
from azure.mgmt.alertsmanagement import AlertsManagementClient
@ -10,7 +11,7 @@ from app import app
from app.alarms import get_proxy_alarm
from app.extensions import db
from app.models.base import Group
from app.models.mirrors import Proxy
from app.models.mirrors import Proxy, Origin
from app.models.alarms import AlarmState
from app.terraform.proxy import ProxyAutomation
@ -70,8 +71,10 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
label_order = ["namespace", "tenant", "name", "attributes"]
}
resource "azurerm_cdn_profile" "profile_{{ group.id }}" {
{% for subgroup in subgroups[group.id] %}
resource "azurerm_cdn_profile" "profile_{{ group.id }}_{{ subgroup }}" {
name = module.label_{{ group.id }}.id
attributes = ["sub{{ subgroup }}"]
location = "{{ azure_location }}"
resource_group_name = data.azurerm_resource_group.this.name
sku = "Standard_Microsoft"
@ -79,9 +82,9 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
tags = module.label_{{ group.id }}.tags
}
resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}" {
resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}_{{ subgroup }}" {
name = "cdn-diagnostics"
target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}.id
target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id
storage_account_id = azurerm_storage_account.this.id
log {
@ -105,10 +108,10 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
}
}
resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}" {
name = "bandwidth-out-high-${module.label_{{ group.id }}.id}"
resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}_{{ subgroup }}" {
name = "bandwidth-out-high-${module.label_{{ group.id }}.id}-sub{{ subgroup }}"
resource_group_name = data.azurerm_resource_group.this.name
scopes = [azurerm_cdn_profile.profile_{{ group.id }}.id]
scopes = [azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id]
description = "Action will be triggered when response size is too high."
criteria {
@ -122,11 +125,12 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
window_size = "PT1H"
}
{% endfor %}
{% endfor %}
{% for proxy in proxies %}
resource "azurerm_cdn_endpoint" "endpoint_{{ proxy.id }}" {
name = "{{ proxy.slug }}"
profile_name = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}.name
profile_name = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}_{{ proxy.psg }}.name
location = "{{ azure_location }}"
resource_group_name = data.azurerm_resource_group.this.name
@ -164,25 +168,41 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
{% endfor %}
"""
def get_subgroup_counts(self):
conn = db.engine.connect()
result = conn.execute("""
SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin
WHERE proxy.origin_id = origin.id
AND proxy.destroyed IS NULL
AND proxy.provider = 'azure_cdn'
GROUP BY origin.group_id, proxy.psg;
""")
subgroups = defaultdict(lambda: defaultdict(lambda: 0))
for row in result:
subgroups[row[0]][row[1]] = row[2]
return subgroups
def create_missing_proxies(self):
groups = Group.query.all()
subgroups = self.get_subgroup_counts()
for group in groups:
active_proxies = len([p for p in Proxy.query.filter(
Proxy.provider == 'azure_cdn',
Proxy.destroyed == None
).all() if p.origin.group_id == group.id])
subgroup = 0
for origin in group.origins:
if active_proxies == 25:
break
active_proxies += 1
while True:
if subgroups[group.id][subgroup] >= 25:
subgroup += 1
else:
break
azure_cdn_proxies = [
x for x in origin.proxies
if x.provider == "azure_cdn" and x.deprecated is None and x.destroyed is None
]
if not azure_cdn_proxies:
subgroups[group.id][subgroup] += 1
proxy = Proxy()
proxy.origin_id = origin.id
proxy.provider = "azure_cdn"
proxy.psg = subgroup
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
random.choices(string.ascii_lowercase, k=random.randint(10, 15)))
proxy.url = f"https://{proxy.slug}.azureedge.net"
@ -230,8 +250,8 @@ if __name__ == "__main__":
auto = ProxyAzureCdnAutomation()
auto.create_missing_proxies()
auto.destroy_expired_proxies()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply(refresh=False, parallelism=1) # Rate limits are problem
set_urls()
import_monitor_alerts()
# auto.generate_terraform()
# auto.terraform_init()
# auto.terraform_apply(refresh=False, parallelism=1) # Rate limits are problem
# set_urls()
# import_monitor_alerts()

32
migrations/versions/22a33ecf3474_add_provider_subgroup_to_proxy.py Normal file
View file

@ -0,0 +1,32 @@
"""add provider subgroup to proxy
Revision ID: 22a33ecf3474
Revises: 25092034e059
Create Date: 2022-04-25 13:33:21.231392
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '22a33ecf3474'
down_revision = '25092034e059'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('proxy', schema=None) as batch_op:
batch_op.add_column(sa.Column('psg', sa.Integer(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('proxy', schema=None) as batch_op:
batch_op.drop_column('psg')
# ### end Alembic commands ###