61 lines
2.6 KiB
Python
61 lines
2.6 KiB
Python
|
import datetime
|
||
|
|
|
||
|
|
import boto3
|
||
|
|
|
||
|
|
from app import app
|
||
|
|
from app.alarms import get_proxy_alarm
|
||
|
|
from app.extensions import db
|
||
|
|
from app.models.mirrors import Proxy
|
||
|
|
from app.models.alarms import AlarmState, Alarm
|
||
|
|
from app.terraform import BaseAutomation
|
||
|
|
|
||
|
|
|
||
|
|
class AlarmProxyCloudfrontAutomation(BaseAutomation):
|
||
|
|
short_name = "monitor_proxy_cloudfront"
|
||
|
|
description = "Import alarms for AWS CloudFront proxies"
|
||
|
|
|
||
|
|
def automate(self):
|
||
|
|
cloudwatch = boto3.client('cloudwatch',
|
||
|
|
aws_access_key_id=app.config['AWS_ACCESS_KEY'],
|
||
|
|
aws_secret_access_key=app.config['AWS_SECRET_KEY'],
|
||
|
|
region_name='us-east-2')
|
||
|
|
dist_paginator = cloudwatch.get_paginator('describe_alarms')
|
||
|
|
page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-")
|
||
|
|
for page in page_iterator:
|
||
|
|
for cw_alarm in page['MetricAlarms']:
|
||
|
|
dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):]
|
||
|
|
proxy = Proxy.query.filter(Proxy.slug == dist_id).first()
|
||
|
|
if proxy is None:
|
||
|
|
print("Skipping unknown proxy " + dist_id)
|
||
|
|
continue
|
||
|
|
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
|
||
|
|
if cw_alarm['StateValue'] == "OK":
|
||
|
|
alarm.update_state(AlarmState.OK, "CloudWatch alarm OK")
|
||
|
|
elif cw_alarm['StateValue'] == "ALARM":
|
||
|
|
alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM")
|
||
|
|
else:
|
||
|
|
alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}")
|
||
|
|
alarm = Alarm.query.filter(
|
||
|
|
Alarm.alarm_type == "cloudfront-quota"
|
||
|
|
).first()
|
||
|
|
if alarm is None:
|
||
|
|
alarm = Alarm()
|
||
|
|
alarm.target = "service/cloudfront"
|
||
|
|
alarm.alarm_type = "cloudfront-quota"
|
||
|
|
alarm.state_changed = datetime.datetime.utcnow()
|
||
|
|
db.session.add(alarm)
|
||
|
|
alarm.last_updated = datetime.datetime.utcnow()
|
||
|
|
deployed_count = len(Proxy.query.filter(
|
||
|
|
Proxy.destroyed == None).all())
|
||
|
|
old_state = alarm.alarm_state
|
||
|
|
if deployed_count > 370:
|
||
|
|
alarm.alarm_state = AlarmState.CRITICAL
|
||
|
|
elif deployed_count > 320:
|
||
|
|
alarm.alarm_state = AlarmState.WARNING
|
||
|
|
else:
|
||
|
|
alarm.alarm_state = AlarmState.OK
|
||
|
|
if alarm.alarm_state != old_state:
|
||
|
|
alarm.state_changed = datetime.datetime.utcnow()
|
||
|
|
db.session.commit()
|
||
|
|
return True, []
|