automation: pull up terraform funcs to abstract class

see #1

app/terraform/__init__.py

@@ -1,56 +1,30 @@
-import json
 import os
-import subprocess
-from typing import Dict, Any
-
-import jinja2
 
 from app import app
 
 
 class BaseAutomation:
     short_name = None
+    """
+    The short name of the automation provider. This is used as an opaque token throughout
+    the portal system.
+    """
 
-    def working_directory(self, filename=None):
+    def automate(self):
+        raise NotImplementedError()
+
+    def working_directory(self, filename=None) -> str:
+        """
+        Provides a filesystem path that can be used during the automation run.
+        This is currently a persistent path, but this should not be relied upon
+        as future versions may use disposable temporary paths instead. State that
+        is needed in subsequent runs should be stored elsewhere.
+
+        :param filename: the filename inside the working directory to create a path for
+        :return: filesystem path for that filename
+        """
         return os.path.join(
             app.config['TERRAFORM_DIRECTORY'],
             self.short_name or self.__class__.__name__.lower(),
             filename or ""
         )
-
-    def write_terraform_config(self, template: str, **kwargs):
-        tmpl = jinja2.Template(template)
-        with open(self.working_directory("main.tf"), 'w') as tf:
-            tf.write(tmpl.render(**kwargs))
-
-    def terraform_init(self):
-        subprocess.run(
-            ['terraform', 'init'],
-            cwd=self.working_directory())
-
-    def terraform_plan(self):
-        plan = subprocess.run(
-            ['terraform', 'plan'],
-            cwd=self.working_directory())
-
-    def terraform_apply(self, refresh: bool = True, parallelism: int = 10):
-        subprocess.run(
-            ['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve',
-             f'-parallelism={str(parallelism)}'],
-            cwd=self.working_directory())
-
-    def terraform_show(self) -> Dict[str, Any]:
-        terraform = subprocess.run(
-            ['terraform', 'show', '-json'],
-            cwd=os.path.join(
-                self.working_directory()),
-            stdout=subprocess.PIPE)
-        return json.loads(terraform.stdout)
-
-    def terraform_output(self) -> Dict[str, Any]:
-        terraform = subprocess.run(
-            ['terraform', 'output', '-json'],
-            cwd=os.path.join(
-                self.working_directory()),
-            stdout=subprocess.PIPE)
-        return json.loads(terraform.stdout)

app/terraform/proxy/__init__.py

@@ -1,16 +1,22 @@
-import datetime
 from collections import defaultdict
+import datetime
+import math
+import string
+import random
 
 from sqlalchemy import text
+from tldextract import tldextract
 
 from app import app
 from app.extensions import db
 from app.models.base import Group
-from app.models.mirrors import Origin, Proxy
+from app.models.mirrors import Proxy
-from app.terraform import BaseAutomation
+from app.terraform.terraform import TerraformAutomation
 
 
-class ProxyAutomation(BaseAutomation):
+class ProxyAutomation(TerraformAutomation):
+    subgroup_max = math.inf
+
     def get_subgroups(self):
         conn = db.engine.connect()
         result = conn.execute(text("""
@@ -26,26 +32,38 @@ class ProxyAutomation(BaseAutomation):
         return subgroups
 
     def create_missing_proxies(self):
-        origins = Origin.query.filter(Origin.destroyed == None).all()
+        groups = Group.query.all()
-        for origin in origins:
+        subgroups = self.get_subgroups()
-            cloudfront_proxies = [
+        for group in groups:
-                x for x in origin.proxies
+            subgroup = 0
-                if x.provider == self.provider and x.deprecated is None and x.destroyed is None
+            for origin in group.origins:
-            ]
+                while True:
-            if not cloudfront_proxies:
+                    if subgroups[group.id][subgroup] >= self.subgroup_max:
-                proxy = Proxy()
+                        subgroup += 1
-                proxy.origin_id = origin.id
+                    else:
-                proxy.provider = self.provider
+                        break
-                proxy.added = datetime.datetime.utcnow()
+                proxies = [
-                proxy.updated = datetime.datetime.utcnow()
+                    x for x in origin.proxies
-                db.session.add(proxy)
+                    if x.provider == self.provider and x.deprecated is None and x.destroyed is None
-                db.session.commit()
+                ]
+                if not proxies:
+                    subgroups[group.id][subgroup] += 1
+                    proxy = Proxy()
+                    proxy.origin_id = origin.id
+                    proxy.provider = self.provider
+                    proxy.psg = subgroup
+                    proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
+                        random.choices(string.ascii_lowercase, k=12))
+                    proxy.added = datetime.datetime.utcnow()
+                    proxy.updated = datetime.datetime.utcnow()
+                    db.session.add(proxy)
+        db.session.commit()
 
     def deprecate_orphaned_proxies(self):
         proxies = Proxy.query.filter(
+            Proxy.deprecated == None,
             Proxy.destroyed == None,
-            Proxy.provider == self.provider,
+            Proxy.provider == self.provider
-            Proxy.deprecated == None
         ).all()
         for proxy in proxies:
             if proxy.origin.destroyed is not None:
@@ -64,13 +82,16 @@ class ProxyAutomation(BaseAutomation):
             proxy.updated = datetime.datetime.utcnow()
         db.session.commit()
 
-    def pre_housekeeping(self):
+    def tf_prehook(self):
         self.create_missing_proxies()
         self.deprecate_orphaned_proxies()
         self.destroy_expired_proxies()
 
-    def generate_terraform(self):
+    def tf_posthook(self):
-        self.write_terraform_config(
+        self.import_state(self.tf_show())
+
+    def tf_generate(self):
+        self.tf_write(
             self.template,
             groups=Group.query.all(),
             proxies=Proxy.query.filter(

app/terraform/proxy/azure_cdn.py

@@ -1,15 +1,9 @@
-import datetime
-import string
-import random
-
 from azure.identity import ClientSecretCredential
 from azure.mgmt.alertsmanagement import AlertsManagementClient
-import tldextract
 
 from app import app
 from app.alarms import get_proxy_alarm
 from app.extensions import db
-from app.models.base import Group
 from app.models.mirrors import Proxy
 from app.models.alarms import AlarmState
 from app.terraform.proxy import ProxyAutomation
@@ -18,6 +12,8 @@ from app.terraform.proxy import ProxyAutomation
 class ProxyAzureCdnAutomation(ProxyAutomation):
     short_name = "proxy_azure_cdn"
     provider = "azure_cdn"
+    subgroup_max = 25
+    parallelism = 1
 
     template_parameters = [
         "azure_resource_group_name",
@@ -166,44 +162,14 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
     {% endfor %}
     """
 
-    def create_missing_proxies(self):
+    def import_state(self, state):
-        groups = Group.query.all()
+        proxies = Proxy.query.filter(
-        subgroups = self.get_subgroups()
+            Proxy.provider == self.provider,
-        for group in groups:
+            Proxy.destroyed == None
-            subgroup = 0
+        ).all()
-            for origin in group.origins:
+        for proxy in proxies:
-                while True:
+            proxy.url = f"https://{proxy.slug}.azureedge.net"
-                    if subgroups[group.id][subgroup] >= 25:
+        db.session.commit()
-                        subgroup += 1
-                    else:
-                        break
-                azure_cdn_proxies = [
-                    x for x in origin.proxies
-                    if x.provider == "azure_cdn" and x.deprecated is None and x.destroyed is None
-                ]
-                if not azure_cdn_proxies:
-                    subgroups[group.id][subgroup] += 1
-                    proxy = Proxy()
-                    proxy.origin_id = origin.id
-                    proxy.provider = "azure_cdn"
-                    proxy.psg = subgroup
-                    proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
-                        random.choices(string.ascii_lowercase, k=random.randint(10, 15)))
-                    proxy.url = f"https://{proxy.slug}.azureedge.net"
-                    proxy.added = datetime.datetime.utcnow()
-                    proxy.updated = datetime.datetime.utcnow()
-                    db.session.add(proxy)
-            db.session.commit()
-
-
-def set_urls():
-    proxies = Proxy.query.filter(
-        Proxy.provider == 'azure_cdn',
-        Proxy.destroyed == None
-    ).all()
-    for proxy in proxies:
-        proxy.url = f"https://{proxy.slug}.azureedge.net"
-    db.session.commit()
 
 
 def import_monitor_alerts():
@@ -232,9 +198,5 @@ def import_monitor_alerts():
 if __name__ == "__main__":
     with app.app_context():
         auto = ProxyAzureCdnAutomation()
-        auto.pre_housekeeping()
+        auto.automate()
-        auto.generate_terraform()
-        auto.terraform_init()
-        auto.terraform_apply(refresh=False, parallelism=1)  # Rate limits are problem
-        set_urls()
         import_monitor_alerts()

app/terraform/proxy/cloudfront.py

@@ -1,7 +1,4 @@
 import datetime
-import json
-import os
-import subprocess
 
 import boto3
 
@@ -79,26 +76,17 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
     {% endfor %}
     """
 
-
+    def import_state(self, state):
-def import_cloudfront_values():
+        for mod in state['values']['root_module']['child_modules']:
-    terraform = subprocess.run(
+            if mod['address'].startswith('module.cloudfront_'):
-        ['terraform', 'show', '-json'],
+                for res in mod['resources']:
-        cwd=os.path.join(
+                    if res['address'].endswith('aws_cloudfront_distribution.this'):
-            app.config['TERRAFORM_DIRECTORY'],
+                        proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first()
-            "proxy_cloudfront"),
+                        proxy.url = "https://" + res['values']['domain_name']
-        stdout=subprocess.PIPE)
+                        proxy.slug = res['values']['id']
-    state = json.loads(terraform.stdout)
+                        proxy.terraform_updated = datetime.datetime.utcnow()
-
+                        break
-    for mod in state['values']['root_module']['child_modules']:
+        db.session.commit()
-        if mod['address'].startswith('module.cloudfront_'):
-            for res in mod['resources']:
-                if res['address'].endswith('aws_cloudfront_distribution.this'):
-                    proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first()
-                    proxy.url = "https://" + res['values']['domain_name']
-                    proxy.slug = res['values']['id']
-                    proxy.terraform_updated = datetime.datetime.utcnow()
-                    db.session.commit()
-                    break
 
 
 def import_cloudwatch_alarms():
@@ -149,9 +137,5 @@ def import_cloudwatch_alarms():
 if __name__ == "__main__":
     with app.app_context():
         auto = ProxyCloudfrontAutomation()
-        auto.pre_housekeeping()
+        auto.automate()
-        auto.generate_terraform()
-        auto.terraform_init()
-        auto.terraform_apply(refresh=False)
-        import_cloudfront_values()
         import_cloudwatch_alarms()

app/terraform/terraform.py

@@ -0,0 +1,93 @@
+import json
+import subprocess
+from typing import Dict, Any, Optional
+
+import jinja2
+
+from app.terraform import BaseAutomation
+
+
+class TerraformAutomation(BaseAutomation):
+    """
+    An abstract class to be extended by automation plugins using Terraform
+    providers to deploy resources.
+    """
+
+    parallelism = 10
+    """
+    Default parallelism for remote API calls.
+    """
+
+    def automate(self):
+        self.tf_prehook()
+        self.tf_generate()
+        self.tf_init()
+        self.tf_apply(refresh=False)
+        self.tf_posthook()
+
+    def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None):
+        if not parallelism:
+            parallelism = self.parallelism
+        subprocess.run(
+            ['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve',
+             f'-parallelism={str(parallelism)}'],
+            cwd=self.working_directory())
+
+    def tf_generate(self):
+        raise NotImplementedError()
+
+    def tf_init(self):
+        subprocess.run(
+            ['terraform', 'init'],
+            cwd=self.working_directory())
+
+    def tf_output(self) -> Dict[str, Any]:
+        tf = subprocess.run(
+            ['terraform', 'output', '-json'],
+            cwd=self.working_directory(),
+            stdout=subprocess.PIPE)
+        return json.loads(tf.stdout)
+
+    def tf_plan(self):
+        tf = subprocess.run(
+            ['terraform', 'plan'],
+            cwd=self.working_directory())
+        # TODO: looks like terraform has a -json output mode here but it's
+        # more like JSON-ND, task is to figure out how to yield those records
+        # as plan runs, the same is probably also true for apply
+
+    def tf_posthook(self, prehook_result: Any = None) -> None:
+        """
+        This hook function is called as part of normal automation, after the
+        completion of :func:`tf_apply`.
+
+        The default, if not overridden by a subclass, is to do nothing.
+
+        :param prehook_result: the returned value of :func:`tf_prehook`
+        :return: None
+        """
+        pass
+
+    def tf_prehook(self) -> Optional[Any]:
+        """
+        This hook function is called as part of normal automation, before generating
+        the terraform configuration file. The return value will be passed to
+        :func:`tf_posthook` but is otherwise ignored.
+
+        The default, if not overridden by a subclass, is to do nothing.
+
+        :return: state that is useful to :func:`tf_posthook`, if required
+        """
+        pass
+
+    def tf_show(self) -> Dict[str, Any]:
+        terraform = subprocess.run(
+            ['terraform', 'show', '-json'],
+            cwd=self.working_directory(),
+            stdout=subprocess.PIPE)
+        return json.loads(terraform.stdout)
+
+    def tf_write(self, template: str, **kwargs):
+        tmpl = jinja2.Template(template)
+        with open(self.working_directory("main.tf"), 'w') as tf:
+            tf.write(tmpl.render(**kwargs))

docs/index.rst

@@ -31,6 +31,7 @@ Documentation Home
    tech/index.rst
    tech/conf.rst
    tech/resource.rst
+   tech/automation.rst
    tech/schemas.rst
 
 

docs/tech/automation.rst

@@ -0,0 +1,16 @@
+Automation Plugins
+==================
+
+Base
+----
+
+.. autoclass:: app.terraform.BaseAutomation
+   :members:
+   :undoc-members:
+
+Terraform
+---------
+
+.. autoclass:: app.terraform.terraform.TerraformAutomation
+   :members:
+   :undoc-members: