automation: establish an automation framework

app/terraform/proxy/__init__.py

@@ -37,6 +37,8 @@ class ProxyAutomation(TerraformAutomation):
         for group in groups:
             subgroup = 0
             for origin in group.origins:
+                if origin.destroyed is not None:
+                    continue
                 while True:
                     if subgroups[group.id][subgroup] >= self.subgroup_max:
                         subgroup += 1
@@ -87,7 +89,7 @@ class ProxyAutomation(TerraformAutomation):
         self.deprecate_orphaned_proxies()
         self.destroy_expired_proxies()
 
-    def tf_posthook(self):
+    def tf_posthook(self, *, prehook_result):
         self.import_state(self.tf_show())
 
     def tf_generate(self):

app/terraform/proxy/azure_cdn.py

@@ -1,16 +1,11 @@
-from azure.identity import ClientSecretCredential
-from azure.mgmt.alertsmanagement import AlertsManagementClient
-
-from app import app
-from app.alarms import get_proxy_alarm
 from app.extensions import db
 from app.models.mirrors import Proxy
-from app.models.alarms import AlarmState
 from app.terraform.proxy import ProxyAutomation
 
 
 class ProxyAzureCdnAutomation(ProxyAutomation):
     short_name = "proxy_azure_cdn"
+    description = "Deploy proxies to Azure CDN"
     provider = "azure_cdn"
     subgroup_max = 25
     parallelism = 1
@@ -170,33 +165,3 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
         for proxy in proxies:
             proxy.url = f"https://{proxy.slug}.azureedge.net"
         db.session.commit()
-
-
-def import_monitor_alerts():
-    credential = ClientSecretCredential(
-        tenant_id=app.config['AZURE_TENANT_ID'],
-        client_id=app.config['AZURE_CLIENT_ID'],
-        client_secret=app.config['AZURE_CLIENT_SECRET'])
-    client = AlertsManagementClient(
-        credential,
-        app.config['AZURE_SUBSCRIPTION_ID']
-    )
-    firing = [x.name[len("bandwidth-out-high-bc-"):]
-              for x in client.alerts.get_all()
-              if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
-    for proxy in Proxy.query.filter(
-        Proxy.provider == "azure_cdn",
-        Proxy.destroyed == None
-    ):
-        alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
-        if proxy.origin.group.group_name.lower() not in firing:
-            alarm.update_state(AlarmState.OK, "Azure monitor alert not firing")
-        else:
-            alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing")
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        auto = ProxyAzureCdnAutomation()
-        auto.automate()
-        import_monitor_alerts()

app/terraform/proxy/cloudfront.py

@@ -1,17 +1,13 @@
 import datetime
 
-import boto3
-
-from app import app
-from app.alarms import get_proxy_alarm
 from app.extensions import db
-from app.models.alarms import Alarm, AlarmState
 from app.models.mirrors import Proxy
 from app.terraform.proxy import ProxyAutomation
 
 
 class ProxyCloudfrontAutomation(ProxyAutomation):
     short_name = "proxy_cloudfront"
+    description = "Deploy proxies to AWS CloudFront"
     provider = "cloudfront"
 
     template_parameters = [
@@ -87,55 +83,3 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
                         proxy.terraform_updated = datetime.datetime.utcnow()
                         break
         db.session.commit()
-
-
-def import_cloudwatch_alarms():
-    cloudwatch = boto3.client('cloudwatch',
-                              aws_access_key_id=app.config['AWS_ACCESS_KEY'],
-                              aws_secret_access_key=app.config['AWS_SECRET_KEY'],
-                              region_name='us-east-2')
-    dist_paginator = cloudwatch.get_paginator('describe_alarms')
-    page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-")
-    for page in page_iterator:
-        for cw_alarm in page['MetricAlarms']:
-            dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):]
-            proxy = Proxy.query.filter(Proxy.slug == dist_id).first()
-            if proxy is None:
-                print("Skipping unknown proxy " + dist_id)
-                continue
-            alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
-            if cw_alarm['StateValue'] == "OK":
-                alarm.update_state(AlarmState.OK, "CloudWatch alarm OK")
-            elif cw_alarm['StateValue'] == "ALARM":
-                alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM")
-            else:
-                alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}")
-    alarm = Alarm.query.filter(
-        Alarm.alarm_type == "cloudfront-quota"
-    ).first()
-    if alarm is None:
-        alarm = Alarm()
-        alarm.target = "service/cloudfront"
-        alarm.alarm_type = "cloudfront-quota"
-        alarm.state_changed = datetime.datetime.utcnow()
-        db.session.add(alarm)
-    alarm.last_updated = datetime.datetime.utcnow()
-    deployed_count = len(Proxy.query.filter(
-        Proxy.destroyed == None).all())
-    old_state = alarm.alarm_state
-    if deployed_count > 370:
-        alarm.alarm_state = AlarmState.CRITICAL
-    elif deployed_count > 320:
-        alarm.alarm_state = AlarmState.WARNING
-    else:
-        alarm.alarm_state = AlarmState.OK
-    if alarm.alarm_state != old_state:
-        alarm.state_changed = datetime.datetime.utcnow()
-    db.session.commit()
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        auto = ProxyCloudfrontAutomation()
-        auto.automate()
-        import_cloudwatch_alarms()