jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

automation: pull up terraform funcs to abstract class

Browse source 
see #1
This commit is contained in:
Iain Learmonth 2022-05-08 13:01:15 +01:00
parent 9b8ac493b1
commit 082de33b5d
7 changed files with 194 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

60
app/terraform/__init__.py
View file

@ -1,56 +1,30 @@
import json
import os
import subprocess
from typing import Dict, Any
import jinja2
from app import app
class BaseAutomation:
short_name = None
"""
The short name of the automation provider. This is used as an opaque token throughout
the portal system.
"""
def working_directory(self, filename=None):
def automate(self):
raise NotImplementedError()
def working_directory(self, filename=None) -> str:
"""
Provides a filesystem path that can be used during the automation run.
This is currently a persistent path, but this should not be relied upon
as future versions may use disposable temporary paths instead. State that
is needed in subsequent runs should be stored elsewhere.
:param filename: the filename inside the working directory to create a path for
:return: filesystem path for that filename
"""
return os.path.join(
app.config['TERRAFORM_DIRECTORY'],
self.short_name or self.__class__.__name__.lower(),
filename or ""
)
def write_terraform_config(self, template: str, **kwargs):
tmpl = jinja2.Template(template)
with open(self.working_directory("main.tf"), 'w') as tf:
tf.write(tmpl.render(**kwargs))
def terraform_init(self):
subprocess.run(
['terraform', 'init'],
cwd=self.working_directory())
def terraform_plan(self):
plan = subprocess.run(
['terraform', 'plan'],
cwd=self.working_directory())
def terraform_apply(self, refresh: bool = True, parallelism: int = 10):
subprocess.run(
['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve',
f'-parallelism={str(parallelism)}'],
cwd=self.working_directory())
def terraform_show(self) -> Dict[str, Any]:
terraform = subprocess.run(
['terraform', 'show', '-json'],
cwd=os.path.join(
self.working_directory()),
stdout=subprocess.PIPE)
return json.loads(terraform.stdout)
def terraform_output(self) -> Dict[str, Any]:
terraform = subprocess.run(
['terraform', 'output', '-json'],
cwd=os.path.join(
self.working_directory()),
stdout=subprocess.PIPE)
return json.loads(terraform.stdout)

67
app/terraform/proxy/__init__.py
View file

@ -1,16 +1,22 @@
import datetime
from collections import defaultdict
import datetime
import math
import string
import random
from sqlalchemy import text
from tldextract import tldextract
from app import app
from app.extensions import db
from app.models.base import Group
from app.models.mirrors import Origin, Proxy
from app.terraform import BaseAutomation
from app.models.mirrors import Proxy
from app.terraform.terraform import TerraformAutomation
class ProxyAutomation(BaseAutomation):
class ProxyAutomation(TerraformAutomation):
subgroup_max = math.inf
def get_subgroups(self):
conn = db.engine.connect()
result = conn.execute(text("""
@ -26,26 +32,38 @@ class ProxyAutomation(BaseAutomation):
return subgroups
def create_missing_proxies(self):
origins = Origin.query.filter(Origin.destroyed == None).all()
for origin in origins:
cloudfront_proxies = [
x for x in origin.proxies
if x.provider == self.provider and x.deprecated is None and x.destroyed is None
]
if not cloudfront_proxies:
proxy = Proxy()
proxy.origin_id = origin.id
proxy.provider = self.provider
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
db.session.add(proxy)
db.session.commit()
groups = Group.query.all()
subgroups = self.get_subgroups()
for group in groups:
subgroup = 0
for origin in group.origins:
while True:
if subgroups[group.id][subgroup] >= self.subgroup_max:
subgroup += 1
else:
break
proxies = [
x for x in origin.proxies
if x.provider == self.provider and x.deprecated is None and x.destroyed is None
]
if not proxies:
subgroups[group.id][subgroup] += 1
proxy = Proxy()
proxy.origin_id = origin.id
proxy.provider = self.provider
proxy.psg = subgroup
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
random.choices(string.ascii_lowercase, k=12))
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
db.session.add(proxy)
db.session.commit()
def deprecate_orphaned_proxies(self):
proxies = Proxy.query.filter(
Proxy.deprecated == None,
Proxy.destroyed == None,
Proxy.provider == self.provider,
Proxy.deprecated == None
Proxy.provider == self.provider
).all()
for proxy in proxies:
if proxy.origin.destroyed is not None:
@ -64,13 +82,16 @@ class ProxyAutomation(BaseAutomation):
proxy.updated = datetime.datetime.utcnow()
db.session.commit()
def pre_housekeeping(self):
def tf_prehook(self):
self.create_missing_proxies()
self.deprecate_orphaned_proxies()
self.destroy_expired_proxies()
def generate_terraform(self):
self.write_terraform_config(
def tf_posthook(self):
self.import_state(self.tf_show())
def tf_generate(self):
self.tf_write(
self.template,
groups=Group.query.all(),
proxies=Proxy.query.filter(

60
app/terraform/proxy/azure_cdn.py
View file

@ -1,15 +1,9 @@
import datetime
import string
import random
from azure.identity import ClientSecretCredential
from azure.mgmt.alertsmanagement import AlertsManagementClient
import tldextract
from app import app
from app.alarms import get_proxy_alarm
from app.extensions import db
from app.models.base import Group
from app.models.mirrors import Proxy
from app.models.alarms import AlarmState
from app.terraform.proxy import ProxyAutomation
@ -18,6 +12,8 @@ from app.terraform.proxy import ProxyAutomation
class ProxyAzureCdnAutomation(ProxyAutomation):
short_name = "proxy_azure_cdn"
provider = "azure_cdn"
subgroup_max = 25
parallelism = 1
template_parameters = [
"azure_resource_group_name",
@ -166,44 +162,14 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
{% endfor %}
"""
def create_missing_proxies(self):
groups = Group.query.all()
subgroups = self.get_subgroups()
for group in groups:
subgroup = 0
for origin in group.origins:
while True:
if subgroups[group.id][subgroup] >= 25:
subgroup += 1
else:
break
azure_cdn_proxies = [
x for x in origin.proxies
if x.provider == "azure_cdn" and x.deprecated is None and x.destroyed is None
]
if not azure_cdn_proxies:
subgroups[group.id][subgroup] += 1
proxy = Proxy()
proxy.origin_id = origin.id
proxy.provider = "azure_cdn"
proxy.psg = subgroup
proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
random.choices(string.ascii_lowercase, k=random.randint(10, 15)))
proxy.url = f"https://{proxy.slug}.azureedge.net"
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
db.session.add(proxy)
db.session.commit()
def set_urls():
proxies = Proxy.query.filter(
Proxy.provider == 'azure_cdn',
Proxy.destroyed == None
).all()
for proxy in proxies:
proxy.url = f"https://{proxy.slug}.azureedge.net"
db.session.commit()
def import_state(self, state):
proxies = Proxy.query.filter(
Proxy.provider == self.provider,
Proxy.destroyed == None
).all()
for proxy in proxies:
proxy.url = f"https://{proxy.slug}.azureedge.net"
db.session.commit()
def import_monitor_alerts():
@ -232,9 +198,5 @@ def import_monitor_alerts():
if __name__ == "__main__":
with app.app_context():
auto = ProxyAzureCdnAutomation()
auto.pre_housekeeping()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply(refresh=False, parallelism=1) # Rate limits are problem
set_urls()
auto.automate()
import_monitor_alerts()

40
app/terraform/proxy/cloudfront.py
View file

@ -1,7 +1,4 @@
import datetime
import json
import os
import subprocess
import boto3
@ -79,26 +76,17 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
{% endfor %}
"""
def import_cloudfront_values():
terraform = subprocess.run(
['terraform', 'show', '-json'],
cwd=os.path.join(
app.config['TERRAFORM_DIRECTORY'],
"proxy_cloudfront"),
stdout=subprocess.PIPE)
state = json.loads(terraform.stdout)
for mod in state['values']['root_module']['child_modules']:
if mod['address'].startswith('module.cloudfront_'):
for res in mod['resources']:
if res['address'].endswith('aws_cloudfront_distribution.this'):
proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first()
proxy.url = "https://" + res['values']['domain_name']
proxy.slug = res['values']['id']
proxy.terraform_updated = datetime.datetime.utcnow()
db.session.commit()
break
def import_state(self, state):
for mod in state['values']['root_module']['child_modules']:
if mod['address'].startswith('module.cloudfront_'):
for res in mod['resources']:
if res['address'].endswith('aws_cloudfront_distribution.this'):
proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first()
proxy.url = "https://" + res['values']['domain_name']
proxy.slug = res['values']['id']
proxy.terraform_updated = datetime.datetime.utcnow()
break
db.session.commit()
def import_cloudwatch_alarms():
@ -149,9 +137,5 @@ def import_cloudwatch_alarms():
if __name__ == "__main__":
with app.app_context():
auto = ProxyCloudfrontAutomation()
auto.pre_housekeeping()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply(refresh=False)
import_cloudfront_values()
auto.automate()
import_cloudwatch_alarms()

93
app/terraform/terraform.py Normal file
View file

@ -0,0 +1,93 @@
import json
import subprocess
from typing import Dict, Any, Optional
import jinja2
from app.terraform import BaseAutomation
class TerraformAutomation(BaseAutomation):
"""
An abstract class to be extended by automation plugins using Terraform
providers to deploy resources.
"""
parallelism = 10
"""
Default parallelism for remote API calls.
"""
def automate(self):
self.tf_prehook()
self.tf_generate()
self.tf_init()
self.tf_apply(refresh=False)
self.tf_posthook()
def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None):
if not parallelism:
parallelism = self.parallelism
subprocess.run(
['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve',
f'-parallelism={str(parallelism)}'],
cwd=self.working_directory())
def tf_generate(self):
raise NotImplementedError()
def tf_init(self):
subprocess.run(
['terraform', 'init'],
cwd=self.working_directory())
def tf_output(self) -> Dict[str, Any]:
tf = subprocess.run(
['terraform', 'output', '-json'],
cwd=self.working_directory(),
stdout=subprocess.PIPE)
return json.loads(tf.stdout)
def tf_plan(self):
tf = subprocess.run(
['terraform', 'plan'],
cwd=self.working_directory())
# TODO: looks like terraform has a -json output mode here but it's
# more like JSON-ND, task is to figure out how to yield those records
# as plan runs, the same is probably also true for apply
def tf_posthook(self, prehook_result: Any = None) -> None:
"""
This hook function is called as part of normal automation, after the
completion of :func:`tf_apply`.
The default, if not overridden by a subclass, is to do nothing.
:param prehook_result: the returned value of :func:`tf_prehook`
:return: None
"""
pass
def tf_prehook(self) -> Optional[Any]:
"""
This hook function is called as part of normal automation, before generating
the terraform configuration file. The return value will be passed to
:func:`tf_posthook` but is otherwise ignored.
The default, if not overridden by a subclass, is to do nothing.
:return: state that is useful to :func:`tf_posthook`, if required
"""
pass
def tf_show(self) -> Dict[str, Any]:
terraform = subprocess.run(
['terraform', 'show', '-json'],
cwd=self.working_directory(),
stdout=subprocess.PIPE)
return json.loads(terraform.stdout)
def tf_write(self, template: str, **kwargs):
tmpl = jinja2.Template(template)
with open(self.working_directory("main.tf"), 'w') as tf:
tf.write(tmpl.render(**kwargs))

1
docs/index.rst
View file

@ -31,6 +31,7 @@ Documentation Home
tech/index.rst
tech/conf.rst
tech/resource.rst
tech/automation.rst
tech/schemas.rst

16
docs/tech/automation.rst Normal file
View file

@ -0,0 +1,16 @@
Automation Plugins
==================
Base
----
.. autoclass:: app.terraform.BaseAutomation
:members:
:undoc-members:
Terraform
---------
.. autoclass:: app.terraform.terraform.TerraformAutomation
:members:
:undoc-members: