automation: establish an automation framework

app/terraform/alarms/proxy_cloudfront.py

@@ -0,0 +1,60 @@
+import datetime
+
+import boto3
+
+from app import app
+from app.alarms import get_proxy_alarm
+from app.extensions import db
+from app.models.mirrors import Proxy
+from app.models.alarms import AlarmState, Alarm
+from app.terraform import BaseAutomation
+
+
+class AlarmProxyCloudfrontAutomation(BaseAutomation):
+    short_name = "monitor_proxy_cloudfront"
+    description = "Import alarms for AWS CloudFront proxies"
+
+    def automate(self):
+        cloudwatch = boto3.client('cloudwatch',
+                                  aws_access_key_id=app.config['AWS_ACCESS_KEY'],
+                                  aws_secret_access_key=app.config['AWS_SECRET_KEY'],
+                                  region_name='us-east-2')
+        dist_paginator = cloudwatch.get_paginator('describe_alarms')
+        page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-")
+        for page in page_iterator:
+            for cw_alarm in page['MetricAlarms']:
+                dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):]
+                proxy = Proxy.query.filter(Proxy.slug == dist_id).first()
+                if proxy is None:
+                    print("Skipping unknown proxy " + dist_id)
+                    continue
+                alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
+                if cw_alarm['StateValue'] == "OK":
+                    alarm.update_state(AlarmState.OK, "CloudWatch alarm OK")
+                elif cw_alarm['StateValue'] == "ALARM":
+                    alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM")
+                else:
+                    alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}")
+        alarm = Alarm.query.filter(
+            Alarm.alarm_type == "cloudfront-quota"
+        ).first()
+        if alarm is None:
+            alarm = Alarm()
+            alarm.target = "service/cloudfront"
+            alarm.alarm_type = "cloudfront-quota"
+            alarm.state_changed = datetime.datetime.utcnow()
+            db.session.add(alarm)
+        alarm.last_updated = datetime.datetime.utcnow()
+        deployed_count = len(Proxy.query.filter(
+            Proxy.destroyed == None).all())
+        old_state = alarm.alarm_state
+        if deployed_count > 370:
+            alarm.alarm_state = AlarmState.CRITICAL
+        elif deployed_count > 320:
+            alarm.alarm_state = AlarmState.WARNING
+        else:
+            alarm.alarm_state = AlarmState.OK
+        if alarm.alarm_state != old_state:
+            alarm.state_changed = datetime.datetime.utcnow()
+        db.session.commit()
+        return True, []