jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

automation: pull up terraform funcs to abstract class

Browse source 
see #1
This commit is contained in:
Iain Learmonth 2022-05-08 13:01:15 +01:00
parent 9b8ac493b1
commit 082de33b5d
7 changed files with 194 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

67
app/terraform/proxy/__init__.py
View file

@ -1,16 +1,22 @@
import datetime
from collections import defaultdict
import datetime
import math
import string
import random
from sqlalchemy import text
from tldextract import tldextract
from app import app
from app.extensions import db
from app.models.base import Group
from app.models.mirrors import Origin, Proxy
from app.terraform import BaseAutomation
from app.models.mirrors import Proxy
from app.terraform.terraform import TerraformAutomation
class ProxyAutomation(BaseAutomation):
class ProxyAutomation(TerraformAutomation):
subgroup_max = math.inf
def get_subgroups(self):
conn = db.engine.connect()
result = conn.execute(text("""
@ -26,26 +32,38 @@ class ProxyAutomation(BaseAutomation):
return subgroups
def create_missing_proxies(self):
origins = Origin.query.filter(Origin.destroyed == None).all()
for origin in origins:
cloudfront_proxies = [
x for x in origin.proxies
if x.provider == self.provider and x.deprecated is None and x.destroyed is None
]
if not cloudfront_proxies:
proxy = Proxy()
proxy.origin_id = origin.id
proxy.provider = self.provider
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
db.session.add(proxy)
db.session.commit()
groups = Group.query.all()
subgroups = self.get_subgroups()
for group in groups:
subgroup = 0
for origin in group.origins:
while True:
if subgroups[group.id][subgroup] >= self.subgroup_max:
subgroup += 1
else:
break
proxies = [
x for x in origin.proxies
if x.provider == self.provider and x.deprecated is None and x.destroyed is None
]
if not proxies:
subgroups[group.id][subgroup] += 1
proxy = Proxy()
proxy.origin_id = origin.id
proxy.provider = self.provider
proxy.psg = subgroup
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
random.choices(string.ascii_lowercase, k=12))
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
db.session.add(proxy)
db.session.commit()
def deprecate_orphaned_proxies(self):
proxies = Proxy.query.filter(
Proxy.deprecated == None,
Proxy.destroyed == None,
Proxy.provider == self.provider,
Proxy.deprecated == None
Proxy.provider == self.provider
).all()
for proxy in proxies:
if proxy.origin.destroyed is not None:
@ -64,13 +82,16 @@ class ProxyAutomation(BaseAutomation):
proxy.updated = datetime.datetime.utcnow()
db.session.commit()
def pre_housekeeping(self):
def tf_prehook(self):
self.create_missing_proxies()
self.deprecate_orphaned_proxies()
self.destroy_expired_proxies()
def generate_terraform(self):
self.write_terraform_config(
def tf_posthook(self):
self.import_state(self.tf_show())
def tf_generate(self):
self.tf_write(
self.template,
groups=Group.query.all(),
proxies=Proxy.query.filter(

60
app/terraform/proxy/azure_cdn.py
View file

@ -1,15 +1,9 @@
import datetime
import string
import random
from azure.identity import ClientSecretCredential
from azure.mgmt.alertsmanagement import AlertsManagementClient
import tldextract
from app import app
from app.alarms import get_proxy_alarm
from app.extensions import db
from app.models.base import Group
from app.models.mirrors import Proxy
from app.models.alarms import AlarmState
from app.terraform.proxy import ProxyAutomation
@ -18,6 +12,8 @@ from app.terraform.proxy import ProxyAutomation
class ProxyAzureCdnAutomation(ProxyAutomation):
short_name = "proxy_azure_cdn"
provider = "azure_cdn"
subgroup_max = 25
parallelism = 1
template_parameters = [
"azure_resource_group_name",
@ -166,44 +162,14 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
{% endfor %}
"""
def create_missing_proxies(self):
groups = Group.query.all()
subgroups = self.get_subgroups()
for group in groups:
subgroup = 0
for origin in group.origins:
while True:
if subgroups[group.id][subgroup] >= 25:
subgroup += 1
else:
break
azure_cdn_proxies = [
x for x in origin.proxies
if x.provider == "azure_cdn" and x.deprecated is None and x.destroyed is None
]
if not azure_cdn_proxies:
subgroups[group.id][subgroup] += 1
proxy = Proxy()
proxy.origin_id = origin.id
proxy.provider = "azure_cdn"
proxy.psg = subgroup
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
random.choices(string.ascii_lowercase, k=random.randint(10, 15)))
proxy.url = f"https://{proxy.slug}.azureedge.net"
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
db.session.add(proxy)
db.session.commit()
def set_urls():
proxies = Proxy.query.filter(
Proxy.provider == 'azure_cdn',
Proxy.destroyed == None
).all()
for proxy in proxies:
proxy.url = f"https://{proxy.slug}.azureedge.net"
db.session.commit()
def import_state(self, state):
proxies = Proxy.query.filter(
Proxy.provider == self.provider,
Proxy.destroyed == None
).all()
for proxy in proxies:
proxy.url = f"https://{proxy.slug}.azureedge.net"
db.session.commit()
def import_monitor_alerts():
@ -232,9 +198,5 @@ def import_monitor_alerts():
if __name__ == "__main__":
with app.app_context():
auto = ProxyAzureCdnAutomation()
auto.pre_housekeeping()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply(refresh=False, parallelism=1) # Rate limits are problem
set_urls()
auto.automate()
import_monitor_alerts()

40
app/terraform/proxy/cloudfront.py
View file

@ -1,7 +1,4 @@
import datetime
import json
import os
import subprocess
import boto3
@ -79,26 +76,17 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
{% endfor %}
"""
def import_cloudfront_values():
terraform = subprocess.run(
['terraform', 'show', '-json'],
cwd=os.path.join(
app.config['TERRAFORM_DIRECTORY'],
"proxy_cloudfront"),
stdout=subprocess.PIPE)
state = json.loads(terraform.stdout)
for mod in state['values']['root_module']['child_modules']:
if mod['address'].startswith('module.cloudfront_'):
for res in mod['resources']:
if res['address'].endswith('aws_cloudfront_distribution.this'):
proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first()
proxy.url = "https://" + res['values']['domain_name']
proxy.slug = res['values']['id']
proxy.terraform_updated = datetime.datetime.utcnow()
db.session.commit()
break
def import_state(self, state):
for mod in state['values']['root_module']['child_modules']:
if mod['address'].startswith('module.cloudfront_'):
for res in mod['resources']:
if res['address'].endswith('aws_cloudfront_distribution.this'):
proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first()
proxy.url = "https://" + res['values']['domain_name']
proxy.slug = res['values']['id']
proxy.terraform_updated = datetime.datetime.utcnow()
break
db.session.commit()
def import_cloudwatch_alarms():
@ -149,9 +137,5 @@ def import_cloudwatch_alarms():
if __name__ == "__main__":
with app.app_context():
auto = ProxyCloudfrontAutomation()
auto.pre_housekeeping()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply(refresh=False)
import_cloudfront_values()
auto.automate()
import_cloudwatch_alarms()