From 082de33b5d34438697be7a6621d98549a0520b71 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Sun, 8 May 2022 13:01:15 +0100 Subject: [PATCH] automation: pull up terraform funcs to abstract class see #1 --- app/terraform/__init__.py | 60 ++++++-------------- app/terraform/proxy/__init__.py | 67 ++++++++++++++-------- app/terraform/proxy/azure_cdn.py | 60 ++++---------------- app/terraform/proxy/cloudfront.py | 40 ++++--------- app/terraform/terraform.py | 93 +++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/tech/automation.rst | 16 ++++++ 7 files changed, 194 insertions(+), 143 deletions(-) create mode 100644 app/terraform/terraform.py create mode 100644 docs/tech/automation.rst diff --git a/app/terraform/__init__.py b/app/terraform/__init__.py index 76e4771..36d4ce8 100644 --- a/app/terraform/__init__.py +++ b/app/terraform/__init__.py @@ -1,56 +1,30 @@ -import json import os -import subprocess -from typing import Dict, Any - -import jinja2 from app import app class BaseAutomation: short_name = None + """ + The short name of the automation provider. This is used as an opaque token throughout + the portal system. + """ - def working_directory(self, filename=None): + def automate(self): + raise NotImplementedError() + + def working_directory(self, filename=None) -> str: + """ + Provides a filesystem path that can be used during the automation run. + This is currently a persistent path, but this should not be relied upon + as future versions may use disposable temporary paths instead. State that + is needed in subsequent runs should be stored elsewhere. + + :param filename: the filename inside the working directory to create a path for + :return: filesystem path for that filename + """ return os.path.join( app.config['TERRAFORM_DIRECTORY'], self.short_name or self.__class__.__name__.lower(), filename or "" ) - - def write_terraform_config(self, template: str, **kwargs): - tmpl = jinja2.Template(template) - with open(self.working_directory("main.tf"), 'w') as tf: - tf.write(tmpl.render(**kwargs)) - - def terraform_init(self): - subprocess.run( - ['terraform', 'init'], - cwd=self.working_directory()) - - def terraform_plan(self): - plan = subprocess.run( - ['terraform', 'plan'], - cwd=self.working_directory()) - - def terraform_apply(self, refresh: bool = True, parallelism: int = 10): - subprocess.run( - ['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve', - f'-parallelism={str(parallelism)}'], - cwd=self.working_directory()) - - def terraform_show(self) -> Dict[str, Any]: - terraform = subprocess.run( - ['terraform', 'show', '-json'], - cwd=os.path.join( - self.working_directory()), - stdout=subprocess.PIPE) - return json.loads(terraform.stdout) - - def terraform_output(self) -> Dict[str, Any]: - terraform = subprocess.run( - ['terraform', 'output', '-json'], - cwd=os.path.join( - self.working_directory()), - stdout=subprocess.PIPE) - return json.loads(terraform.stdout) diff --git a/app/terraform/proxy/__init__.py b/app/terraform/proxy/__init__.py index 0ed7768..f9674dd 100644 --- a/app/terraform/proxy/__init__.py +++ b/app/terraform/proxy/__init__.py @@ -1,16 +1,22 @@ -import datetime from collections import defaultdict +import datetime +import math +import string +import random from sqlalchemy import text +from tldextract import tldextract from app import app from app.extensions import db from app.models.base import Group -from app.models.mirrors import Origin, Proxy -from app.terraform import BaseAutomation +from app.models.mirrors import Proxy +from app.terraform.terraform import TerraformAutomation -class ProxyAutomation(BaseAutomation): +class ProxyAutomation(TerraformAutomation): + subgroup_max = math.inf + def get_subgroups(self): conn = db.engine.connect() result = conn.execute(text(""" @@ -26,26 +32,38 @@ class ProxyAutomation(BaseAutomation): return subgroups def create_missing_proxies(self): - origins = Origin.query.filter(Origin.destroyed == None).all() - for origin in origins: - cloudfront_proxies = [ - x for x in origin.proxies - if x.provider == self.provider and x.deprecated is None and x.destroyed is None - ] - if not cloudfront_proxies: - proxy = Proxy() - proxy.origin_id = origin.id - proxy.provider = self.provider - proxy.added = datetime.datetime.utcnow() - proxy.updated = datetime.datetime.utcnow() - db.session.add(proxy) - db.session.commit() + groups = Group.query.all() + subgroups = self.get_subgroups() + for group in groups: + subgroup = 0 + for origin in group.origins: + while True: + if subgroups[group.id][subgroup] >= self.subgroup_max: + subgroup += 1 + else: + break + proxies = [ + x for x in origin.proxies + if x.provider == self.provider and x.deprecated is None and x.destroyed is None + ] + if not proxies: + subgroups[group.id][subgroup] += 1 + proxy = Proxy() + proxy.origin_id = origin.id + proxy.provider = self.provider + proxy.psg = subgroup + proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join( + random.choices(string.ascii_lowercase, k=12)) + proxy.added = datetime.datetime.utcnow() + proxy.updated = datetime.datetime.utcnow() + db.session.add(proxy) + db.session.commit() def deprecate_orphaned_proxies(self): proxies = Proxy.query.filter( + Proxy.deprecated == None, Proxy.destroyed == None, - Proxy.provider == self.provider, - Proxy.deprecated == None + Proxy.provider == self.provider ).all() for proxy in proxies: if proxy.origin.destroyed is not None: @@ -64,13 +82,16 @@ class ProxyAutomation(BaseAutomation): proxy.updated = datetime.datetime.utcnow() db.session.commit() - def pre_housekeeping(self): + def tf_prehook(self): self.create_missing_proxies() self.deprecate_orphaned_proxies() self.destroy_expired_proxies() - def generate_terraform(self): - self.write_terraform_config( + def tf_posthook(self): + self.import_state(self.tf_show()) + + def tf_generate(self): + self.tf_write( self.template, groups=Group.query.all(), proxies=Proxy.query.filter( diff --git a/app/terraform/proxy/azure_cdn.py b/app/terraform/proxy/azure_cdn.py index c6fcf08..5acd8e3 100644 --- a/app/terraform/proxy/azure_cdn.py +++ b/app/terraform/proxy/azure_cdn.py @@ -1,15 +1,9 @@ -import datetime -import string -import random - from azure.identity import ClientSecretCredential from azure.mgmt.alertsmanagement import AlertsManagementClient -import tldextract from app import app from app.alarms import get_proxy_alarm from app.extensions import db -from app.models.base import Group from app.models.mirrors import Proxy from app.models.alarms import AlarmState from app.terraform.proxy import ProxyAutomation @@ -18,6 +12,8 @@ from app.terraform.proxy import ProxyAutomation class ProxyAzureCdnAutomation(ProxyAutomation): short_name = "proxy_azure_cdn" provider = "azure_cdn" + subgroup_max = 25 + parallelism = 1 template_parameters = [ "azure_resource_group_name", @@ -166,44 +162,14 @@ class ProxyAzureCdnAutomation(ProxyAutomation): {% endfor %} """ - def create_missing_proxies(self): - groups = Group.query.all() - subgroups = self.get_subgroups() - for group in groups: - subgroup = 0 - for origin in group.origins: - while True: - if subgroups[group.id][subgroup] >= 25: - subgroup += 1 - else: - break - azure_cdn_proxies = [ - x for x in origin.proxies - if x.provider == "azure_cdn" and x.deprecated is None and x.destroyed is None - ] - if not azure_cdn_proxies: - subgroups[group.id][subgroup] += 1 - proxy = Proxy() - proxy.origin_id = origin.id - proxy.provider = "azure_cdn" - proxy.psg = subgroup - proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join( - random.choices(string.ascii_lowercase, k=random.randint(10, 15))) - proxy.url = f"https://{proxy.slug}.azureedge.net" - proxy.added = datetime.datetime.utcnow() - proxy.updated = datetime.datetime.utcnow() - db.session.add(proxy) - db.session.commit() - - -def set_urls(): - proxies = Proxy.query.filter( - Proxy.provider == 'azure_cdn', - Proxy.destroyed == None - ).all() - for proxy in proxies: - proxy.url = f"https://{proxy.slug}.azureedge.net" - db.session.commit() + def import_state(self, state): + proxies = Proxy.query.filter( + Proxy.provider == self.provider, + Proxy.destroyed == None + ).all() + for proxy in proxies: + proxy.url = f"https://{proxy.slug}.azureedge.net" + db.session.commit() def import_monitor_alerts(): @@ -232,9 +198,5 @@ def import_monitor_alerts(): if __name__ == "__main__": with app.app_context(): auto = ProxyAzureCdnAutomation() - auto.pre_housekeeping() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply(refresh=False, parallelism=1) # Rate limits are problem - set_urls() + auto.automate() import_monitor_alerts() diff --git a/app/terraform/proxy/cloudfront.py b/app/terraform/proxy/cloudfront.py index 0c68b39..1c1fa42 100644 --- a/app/terraform/proxy/cloudfront.py +++ b/app/terraform/proxy/cloudfront.py @@ -1,7 +1,4 @@ import datetime -import json -import os -import subprocess import boto3 @@ -79,26 +76,17 @@ class ProxyCloudfrontAutomation(ProxyAutomation): {% endfor %} """ - -def import_cloudfront_values(): - terraform = subprocess.run( - ['terraform', 'show', '-json'], - cwd=os.path.join( - app.config['TERRAFORM_DIRECTORY'], - "proxy_cloudfront"), - stdout=subprocess.PIPE) - state = json.loads(terraform.stdout) - - for mod in state['values']['root_module']['child_modules']: - if mod['address'].startswith('module.cloudfront_'): - for res in mod['resources']: - if res['address'].endswith('aws_cloudfront_distribution.this'): - proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first() - proxy.url = "https://" + res['values']['domain_name'] - proxy.slug = res['values']['id'] - proxy.terraform_updated = datetime.datetime.utcnow() - db.session.commit() - break + def import_state(self, state): + for mod in state['values']['root_module']['child_modules']: + if mod['address'].startswith('module.cloudfront_'): + for res in mod['resources']: + if res['address'].endswith('aws_cloudfront_distribution.this'): + proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first() + proxy.url = "https://" + res['values']['domain_name'] + proxy.slug = res['values']['id'] + proxy.terraform_updated = datetime.datetime.utcnow() + break + db.session.commit() def import_cloudwatch_alarms(): @@ -149,9 +137,5 @@ def import_cloudwatch_alarms(): if __name__ == "__main__": with app.app_context(): auto = ProxyCloudfrontAutomation() - auto.pre_housekeeping() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply(refresh=False) - import_cloudfront_values() + auto.automate() import_cloudwatch_alarms() diff --git a/app/terraform/terraform.py b/app/terraform/terraform.py new file mode 100644 index 0000000..8311c85 --- /dev/null +++ b/app/terraform/terraform.py @@ -0,0 +1,93 @@ +import json +import subprocess +from typing import Dict, Any, Optional + +import jinja2 + +from app.terraform import BaseAutomation + + +class TerraformAutomation(BaseAutomation): + """ + An abstract class to be extended by automation plugins using Terraform + providers to deploy resources. + """ + + parallelism = 10 + """ + Default parallelism for remote API calls. + """ + + def automate(self): + self.tf_prehook() + self.tf_generate() + self.tf_init() + self.tf_apply(refresh=False) + self.tf_posthook() + + def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None): + if not parallelism: + parallelism = self.parallelism + subprocess.run( + ['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve', + f'-parallelism={str(parallelism)}'], + cwd=self.working_directory()) + + def tf_generate(self): + raise NotImplementedError() + + def tf_init(self): + subprocess.run( + ['terraform', 'init'], + cwd=self.working_directory()) + + def tf_output(self) -> Dict[str, Any]: + tf = subprocess.run( + ['terraform', 'output', '-json'], + cwd=self.working_directory(), + stdout=subprocess.PIPE) + return json.loads(tf.stdout) + + def tf_plan(self): + tf = subprocess.run( + ['terraform', 'plan'], + cwd=self.working_directory()) + # TODO: looks like terraform has a -json output mode here but it's + # more like JSON-ND, task is to figure out how to yield those records + # as plan runs, the same is probably also true for apply + + def tf_posthook(self, prehook_result: Any = None) -> None: + """ + This hook function is called as part of normal automation, after the + completion of :func:`tf_apply`. + + The default, if not overridden by a subclass, is to do nothing. + + :param prehook_result: the returned value of :func:`tf_prehook` + :return: None + """ + pass + + def tf_prehook(self) -> Optional[Any]: + """ + This hook function is called as part of normal automation, before generating + the terraform configuration file. The return value will be passed to + :func:`tf_posthook` but is otherwise ignored. + + The default, if not overridden by a subclass, is to do nothing. + + :return: state that is useful to :func:`tf_posthook`, if required + """ + pass + + def tf_show(self) -> Dict[str, Any]: + terraform = subprocess.run( + ['terraform', 'show', '-json'], + cwd=self.working_directory(), + stdout=subprocess.PIPE) + return json.loads(terraform.stdout) + + def tf_write(self, template: str, **kwargs): + tmpl = jinja2.Template(template) + with open(self.working_directory("main.tf"), 'w') as tf: + tf.write(tmpl.render(**kwargs)) diff --git a/docs/index.rst b/docs/index.rst index 851c3a6..b79282d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ Documentation Home tech/index.rst tech/conf.rst tech/resource.rst + tech/automation.rst tech/schemas.rst diff --git a/docs/tech/automation.rst b/docs/tech/automation.rst new file mode 100644 index 0000000..530e0fa --- /dev/null +++ b/docs/tech/automation.rst @@ -0,0 +1,16 @@ +Automation Plugins +================== + +Base +---- + +.. autoclass:: app.terraform.BaseAutomation + :members: + :undoc-members: + +Terraform +--------- + +.. autoclass:: app.terraform.terraform.TerraformAutomation + :members: + :undoc-members: