Initial import
This commit is contained in:
commit
09f0b0672d
64 changed files with 3735 additions and 0 deletions
51
app/terraform/proxy/__init__.py
Normal file
51
app/terraform/proxy/__init__.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import datetime
|
||||
|
||||
from app import app
|
||||
from app.extensions import db
|
||||
from app.models import Group, Origin, Proxy
|
||||
from app.terraform import BaseAutomation
|
||||
|
||||
|
||||
class ProxyAutomation(BaseAutomation):
|
||||
def create_missing_proxies(self):
|
||||
origins = Origin.query.all()
|
||||
for origin in origins:
|
||||
cloudfront_proxies = [
|
||||
x for x in origin.proxies
|
||||
if x.provider == self.provider and x.deprecated is None and x.destroyed is None
|
||||
]
|
||||
if not cloudfront_proxies:
|
||||
proxy = Proxy()
|
||||
proxy.origin_id = origin.id
|
||||
proxy.provider = self.provider
|
||||
proxy.added = datetime.datetime.utcnow()
|
||||
proxy.updated = datetime.datetime.utcnow()
|
||||
db.session.add(proxy)
|
||||
db.session.commit()
|
||||
|
||||
def destroy_expired_proxies(self):
|
||||
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
|
||||
proxies = Proxy.query.filter(
|
||||
Proxy.destroyed == None,
|
||||
Proxy.provider == self.provider,
|
||||
Proxy.deprecated < cutoff
|
||||
).all()
|
||||
for proxy in proxies:
|
||||
proxy.destroyed = datetime.datetime.utcnow()
|
||||
proxy.updated = datetime.datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
def generate_terraform(self):
|
||||
self.write_terraform_config(
|
||||
self.template,
|
||||
groups=Group.query.all(),
|
||||
proxies=Proxy.query.filter(
|
||||
Proxy.provider == self.provider,
|
||||
Proxy.destroyed == None
|
||||
).all(),
|
||||
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
||||
**{
|
||||
k: app.config[k.upper()]
|
||||
for k in self.template_parameters
|
||||
}
|
||||
)
|
||||
238
app/terraform/proxy/azure_cdn.py
Normal file
238
app/terraform/proxy/azure_cdn.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
import datetime
|
||||
import string
|
||||
import random
|
||||
|
||||
from azure.identity import ClientSecretCredential
|
||||
from azure.mgmt.alertsmanagement import AlertsManagementClient
|
||||
import tldextract
|
||||
|
||||
from app import app
|
||||
from app.alarms import get_proxy_alarm
|
||||
from app.extensions import db
|
||||
from app.models import Group, Proxy, Alarm, AlarmState
|
||||
from app.terraform.proxy import ProxyAutomation
|
||||
|
||||
|
||||
class ProxyAzureCdnAutomation(ProxyAutomation):
|
||||
short_name = "proxy_azure_cdn"
|
||||
provider = "azure_cdn"
|
||||
|
||||
template_parameters = [
|
||||
"azure_resource_group_name",
|
||||
"azure_storage_account_name",
|
||||
"azure_location",
|
||||
"azure_client_id",
|
||||
"azure_client_secret",
|
||||
"azure_subscription_id",
|
||||
"azure_tenant_id"
|
||||
]
|
||||
|
||||
template = """
|
||||
terraform {
|
||||
required_providers {
|
||||
azurerm = {
|
||||
source = "hashicorp/azurerm"
|
||||
version = "=2.99.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "azurerm" {
|
||||
features {}
|
||||
|
||||
client_id = "{{ azure_client_id }}"
|
||||
client_secret = "{{ azure_client_secret }}"
|
||||
subscription_id = "{{ azure_subscription_id }}"
|
||||
tenant_id = "{{ azure_tenant_id }}"
|
||||
skip_provider_registration = true
|
||||
}
|
||||
|
||||
data "azurerm_resource_group" "this" {
|
||||
name = "{{ azure_resource_group_name }}"
|
||||
}
|
||||
|
||||
resource "azurerm_storage_account" "this" {
|
||||
name = "{{ azure_storage_account_name }}"
|
||||
resource_group_name = data.azurerm_resource_group.this.name
|
||||
location = "{{ azure_location }}"
|
||||
account_tier = "Standard"
|
||||
account_replication_type = "RAGRS"
|
||||
}
|
||||
|
||||
{% for group in groups %}
|
||||
module "label_{{ group.id }}" {
|
||||
source = "cloudposse/label/null"
|
||||
version = "0.25.0"
|
||||
namespace = "{{ global_namespace }}"
|
||||
tenant = "{{ group.group_name }}"
|
||||
label_order = ["namespace", "tenant", "name", "attributes"]
|
||||
}
|
||||
|
||||
resource "azurerm_cdn_profile" "profile_{{ group.id }}" {
|
||||
name = module.label_{{ group.id }}.id
|
||||
location = "{{ azure_location }}"
|
||||
resource_group_name = data.azurerm_resource_group.this.name
|
||||
sku = "Standard_Microsoft"
|
||||
|
||||
tags = module.label_{{ group.id }}.tags
|
||||
}
|
||||
|
||||
resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}" {
|
||||
name = "cdn-diagnostics"
|
||||
target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}.id
|
||||
storage_account_id = azurerm_storage_account.this.id
|
||||
|
||||
log {
|
||||
category = "AzureCDNAccessLog"
|
||||
enabled = true
|
||||
|
||||
retention_policy {
|
||||
enabled = true
|
||||
days = 90
|
||||
}
|
||||
}
|
||||
|
||||
metric {
|
||||
category = "AllMetrics"
|
||||
enabled = true
|
||||
|
||||
retention_policy {
|
||||
enabled = true
|
||||
days = 90
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}" {
|
||||
name = "bandwidth-out-high-${module.label_{{ group.id }}.id}"
|
||||
resource_group_name = data.azurerm_resource_group.this.name
|
||||
scopes = [azurerm_cdn_profile.profile_{{ group.id }}.id]
|
||||
description = "Action will be triggered when response size is too high."
|
||||
|
||||
criteria {
|
||||
metric_namespace = "Microsoft.Cdn/profiles"
|
||||
metric_name = "ResponseSize"
|
||||
aggregation = "Total"
|
||||
operator = "GreaterThan"
|
||||
threshold = 21474836481
|
||||
}
|
||||
|
||||
window_size = "PT1H"
|
||||
}
|
||||
{% endfor %}
|
||||
|
||||
{% for proxy in proxies %}
|
||||
resource "azurerm_cdn_endpoint" "endpoint_{{ proxy.id }}" {
|
||||
name = "{{ proxy.slug }}"
|
||||
profile_name = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}.name
|
||||
location = "{{ azure_location }}"
|
||||
resource_group_name = data.azurerm_resource_group.this.name
|
||||
|
||||
origin {
|
||||
name = "upstream"
|
||||
host_name = "{{ proxy.origin.domain_name }}"
|
||||
}
|
||||
|
||||
global_delivery_rule {
|
||||
modify_request_header_action {
|
||||
action = "Overwrite"
|
||||
name = "User-Agent"
|
||||
value = "Amazon CloudFront"
|
||||
}
|
||||
modify_request_header_action {
|
||||
action = "Append"
|
||||
name = "X-Amz-Cf-Id"
|
||||
value = "dummystring"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "azurerm_monitor_diagnostic_setting" "diagnostic_{{ proxy.id }}" {
|
||||
name = "cdn-diagnostics"
|
||||
target_resource_id = azurerm_cdn_endpoint.endpoint_{{ proxy.id }}.id
|
||||
storage_account_id = azurerm_storage_account.this.id
|
||||
|
||||
log {
|
||||
category = "CoreAnalytics"
|
||||
enabled = true
|
||||
|
||||
retention_policy {
|
||||
enabled = true
|
||||
days = 90
|
||||
}
|
||||
}
|
||||
}
|
||||
{% endfor %}
|
||||
"""
|
||||
|
||||
def create_missing_proxies(self):
|
||||
groups = Group.query.all()
|
||||
for group in groups:
|
||||
active_proxies = len([p for p in Proxy.query.filter(
|
||||
Proxy.provider == 'azure_cdn',
|
||||
Proxy.destroyed == None
|
||||
).all() if p.origin.group_id == group.id])
|
||||
for origin in group.origins:
|
||||
if active_proxies == 25:
|
||||
break
|
||||
active_proxies += 1
|
||||
azure_cdn_proxies = [
|
||||
x for x in origin.proxies
|
||||
if x.provider == "azure_cdn" and x.deprecated is None and x.destroyed is None
|
||||
]
|
||||
if not azure_cdn_proxies:
|
||||
proxy = Proxy()
|
||||
proxy.origin_id = origin.id
|
||||
proxy.provider = "azure_cdn"
|
||||
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
|
||||
random.choices(string.ascii_lowercase, k=random.randint(10, 15)))
|
||||
proxy.url = f"https://{proxy.slug}.azureedge.net"
|
||||
proxy.added = datetime.datetime.utcnow()
|
||||
proxy.updated = datetime.datetime.utcnow()
|
||||
db.session.add(proxy)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def set_urls():
|
||||
proxies = Proxy.query.filter(
|
||||
Proxy.provider == 'azure_cdn',
|
||||
Proxy.destroyed == None
|
||||
).all()
|
||||
for proxy in proxies:
|
||||
proxy.url = f"https://{proxy.slug}.azureedge.net"
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def import_monitor_alerts():
|
||||
credential = ClientSecretCredential(
|
||||
tenant_id=app.config['AZURE_TENANT_ID'],
|
||||
client_id=app.config['AZURE_CLIENT_ID'],
|
||||
client_secret=app.config['AZURE_CLIENT_SECRET'])
|
||||
client = AlertsManagementClient(
|
||||
credential,
|
||||
app.config['AZURE_SUBSCRIPTION_ID']
|
||||
)
|
||||
firing = [x.name[len("bandwidth-out-high-bc-"):]
|
||||
for x in client.alerts.get_all()
|
||||
if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
|
||||
for proxy in Proxy.query.filter(
|
||||
Proxy.provider == "azure_cdn",
|
||||
Proxy.destroyed == None
|
||||
):
|
||||
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
|
||||
if proxy.origin.group.group_name.lower() not in firing:
|
||||
alarm.update_state(AlarmState.OK, "Azure monitor alert not firing")
|
||||
else:
|
||||
alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with app.app_context():
|
||||
auto = ProxyAzureCdnAutomation()
|
||||
auto.create_missing_proxies()
|
||||
auto.destroy_expired_proxies()
|
||||
auto.generate_terraform()
|
||||
auto.terraform_init()
|
||||
auto.terraform_apply(refresh=False, parallelism=1) # Rate limits are problem
|
||||
set_urls()
|
||||
import_monitor_alerts()
|
||||
156
app/terraform/proxy/cloudfront.py
Normal file
156
app/terraform/proxy/cloudfront.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import datetime
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import boto3
|
||||
|
||||
from app import app
|
||||
from app.alarms import get_proxy_alarm
|
||||
from app.extensions import db
|
||||
from app.models import Proxy, Alarm, AlarmState
|
||||
from app.terraform.proxy import ProxyAutomation
|
||||
|
||||
|
||||
class ProxyCloudfrontAutomation(ProxyAutomation):
|
||||
short_name = "proxy_cloudfront"
|
||||
provider = "cloudfront"
|
||||
|
||||
template_parameters = [
|
||||
"aws_access_key",
|
||||
"aws_secret_key"
|
||||
]
|
||||
|
||||
template = """
|
||||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
version = "~> 4.4.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
access_key = "{{ aws_access_key }}"
|
||||
secret_key = "{{ aws_secret_key }}"
|
||||
region = "us-east-1"
|
||||
}
|
||||
|
||||
{% for group in groups %}
|
||||
module "label_{{ group.id }}" {
|
||||
source = "cloudposse/label/null"
|
||||
version = "0.25.0"
|
||||
namespace = "{{ global_namespace }}"
|
||||
tenant = "{{ group.group_name }}"
|
||||
label_order = ["namespace", "tenant", "name", "attributes"]
|
||||
}
|
||||
|
||||
module "log_bucket_{{ group.id }}" {
|
||||
source = "cloudposse/s3-log-storage/aws"
|
||||
version = "0.28.0"
|
||||
context = module.label_{{ group.id }}.context
|
||||
name = "logs"
|
||||
attributes = ["cloudfront"]
|
||||
acl = "log-delivery-write"
|
||||
standard_transition_days = 30
|
||||
glacier_transition_days = 60
|
||||
expiration_days = 90
|
||||
}
|
||||
|
||||
resource "aws_sns_topic" "alarms_{{ group.id }}" {
|
||||
name = "${module.label_{{ group.id }}.id}-cloudfront-alarms"
|
||||
}
|
||||
{% endfor %}
|
||||
|
||||
{% for proxy in proxies %}
|
||||
module "cloudfront_{{ proxy.id }}" {
|
||||
source = "sr2c/bc-proxy/aws"
|
||||
version = "0.0.5"
|
||||
origin_domain = "{{ proxy.origin.domain_name }}"
|
||||
logging_bucket = module.log_bucket_{{ proxy.origin.group.id }}.bucket_domain_name
|
||||
sns_topic_arn = aws_sns_topic.alarms_{{ proxy.origin.group.id }}.arn
|
||||
low_bandwidth_alarm = false
|
||||
context = module.label_{{ proxy.origin.group.id }}.context
|
||||
name = "proxy"
|
||||
attributes = ["{{ proxy.origin.domain_name }}"]
|
||||
}
|
||||
{% endfor %}
|
||||
"""
|
||||
|
||||
|
||||
def import_cloudfront_values():
|
||||
terraform = subprocess.run(
|
||||
['terraform', 'show', '-json'],
|
||||
cwd=os.path.join(
|
||||
app.config['TERRAFORM_DIRECTORY'],
|
||||
"proxy_cloudfront"),
|
||||
stdout=subprocess.PIPE)
|
||||
state = json.loads(terraform.stdout)
|
||||
|
||||
for mod in state['values']['root_module']['child_modules']:
|
||||
if mod['address'].startswith('module.cloudfront_'):
|
||||
for res in mod['resources']:
|
||||
if res['address'].endswith('aws_cloudfront_distribution.this'):
|
||||
proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first()
|
||||
proxy.url = "https://" + res['values']['domain_name']
|
||||
proxy.slug = res['values']['id']
|
||||
proxy.terraform_updated = datetime.datetime.utcnow()
|
||||
db.session.commit()
|
||||
break
|
||||
|
||||
|
||||
def import_cloudwatch_alarms():
|
||||
cloudwatch = boto3.client('cloudwatch',
|
||||
aws_access_key_id=app.config['AWS_ACCESS_KEY'],
|
||||
aws_secret_access_key=app.config['AWS_SECRET_KEY'],
|
||||
region_name='us-east-1')
|
||||
dist_paginator = cloudwatch.get_paginator('describe_alarms')
|
||||
page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-")
|
||||
for page in page_iterator:
|
||||
for cw_alarm in page['MetricAlarms']:
|
||||
dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):]
|
||||
proxy = Proxy.query.filter(Proxy.slug == dist_id).first()
|
||||
if proxy is None:
|
||||
print("Skipping unknown proxy " + dist_id)
|
||||
continue
|
||||
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
|
||||
if cw_alarm['StateValue'] == "OK":
|
||||
alarm.update_state(AlarmState.OK, "CloudWatch alarm OK")
|
||||
elif cw_alarm['StateValue'] == "ALARM":
|
||||
alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM")
|
||||
else:
|
||||
alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}")
|
||||
alarm = Alarm.query.filter(
|
||||
Alarm.alarm_type == "cloudfront-quota"
|
||||
).first()
|
||||
if alarm is None:
|
||||
alarm = Alarm()
|
||||
alarm.target = "service/cloudfront"
|
||||
alarm.alarm_type = "cloudfront-quota"
|
||||
alarm.state_changed = datetime.datetime.utcnow()
|
||||
db.session.add(alarm)
|
||||
alarm.last_updated = datetime.datetime.utcnow()
|
||||
deployed_count = len(Proxy.query.filter(
|
||||
Proxy.destroyed == None).all())
|
||||
old_state = alarm.alarm_state
|
||||
if deployed_count > 370:
|
||||
alarm.alarm_state = AlarmState.CRITICAL
|
||||
elif deployed_count > 320:
|
||||
alarm.alarm_state = AlarmState.WARNING
|
||||
else:
|
||||
alarm.alarm_state = AlarmState.OK
|
||||
if alarm.alarm_state != old_state:
|
||||
alarm.state_changed = datetime.datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with app.app_context():
|
||||
auto = ProxyCloudfrontAutomation()
|
||||
auto.destroy_expired_proxies()
|
||||
auto.create_missing_proxies()
|
||||
auto.generate_terraform()
|
||||
auto.terraform_init()
|
||||
auto.terraform_apply()
|
||||
import_cloudfront_values()
|
||||
import_cloudwatch_alarms()
|
||||
Loading…
Add table
Add a link
Reference in a new issue