From 51f580a304c8f30b41ff8e147716cde1c71410a3 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Mon, 16 May 2022 10:08:18 +0100 Subject: [PATCH] auto/terraform: typing hints for base terraform module --- app/cli/automate.py | 6 ++++-- app/terraform/__init__.py | 4 +++- app/terraform/proxy/__init__.py | 3 ++- app/terraform/terraform.py | 38 ++++++++++++++++++++++----------- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/app/cli/automate.py b/app/cli/automate.py index 48581b6..a21d22b 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -60,7 +60,7 @@ jobs = { def run_all(**kwargs: bool) -> None: for job in jobs.values(): - run_job(job, **kwargs) + run_job(job, **kwargs) # type: ignore def run_job(job_cls: Type[BaseAutomation], *, @@ -136,7 +136,9 @@ class AutomateCliHandler: def run(self) -> None: with app.app_context(): if self.args.job: - run_job(jobs[self.args.job], force=self.args.force, ignore_schedule=self.args.ignore_schedule) + run_job(jobs[self.args.job], # type: ignore + force=self.args.force, + ignore_schedule=self.args.ignore_schedule) elif self.args.all: run_all(force=self.args.force, ignore_schedule=self.args.ignore_schedule) else: diff --git a/app/terraform/__init__.py b/app/terraform/__init__.py index 18ccc9a..844812c 100644 --- a/app/terraform/__init__.py +++ b/app/terraform/__init__.py @@ -1,10 +1,11 @@ +from abc import ABCMeta, abstractmethod import os from typing import Tuple from app import app -class BaseAutomation: +class BaseAutomation(metaclass=ABCMeta): short_name: str = "base" description: str = "Abstract base automation." frequency: int @@ -14,6 +15,7 @@ class BaseAutomation: the portal system. """ + @abstractmethod def automate(self, full: bool = False) -> Tuple[bool, str]: raise NotImplementedError() diff --git a/app/terraform/proxy/__init__.py b/app/terraform/proxy/__init__.py index 23bd576..30b2403 100644 --- a/app/terraform/proxy/__init__.py +++ b/app/terraform/proxy/__init__.py @@ -3,6 +3,7 @@ import datetime import math import string import random +from typing import Dict from sqlalchemy import text from tldextract import tldextract @@ -17,7 +18,7 @@ from app.terraform.terraform import TerraformAutomation class ProxyAutomation(TerraformAutomation): subgroup_max = math.inf - def get_subgroups(self): + def get_subgroups(self) -> Dict[int, Dict[int, int]]: conn = db.engine.connect() result = conn.execute(text(""" SELECT origin.group_id, proxy.psg, COUNT(proxy.id) FROM proxy, origin diff --git a/app/terraform/terraform.py b/app/terraform/terraform.py index 199f2f2..55a1ba5 100644 --- a/app/terraform/terraform.py +++ b/app/terraform/terraform.py @@ -1,5 +1,6 @@ import json import subprocess +from abc import abstractmethod from typing import Any, Dict, List, Optional, Tuple import jinja2 @@ -18,7 +19,23 @@ class TerraformAutomation(BaseAutomation): Default parallelism for remote API calls. """ - def automate(self, full: bool = False): + def automate(self, full: bool = False) -> Tuple[bool, str]: + """ + Runs the Terraform automation module. The run will follow these steps: + + 1. The :func:`tf_prehook` hook is run. + 2. Generate a Terraform configuration and write it to a single ``main.tf`` file in the working directory + (see :func:`working_directory `). + 3. Run ``terraform init``. + 4. Run ``terraform apply``. This will only include a refresh if *full* is **True**. The apply will wait up + to *lock_timeout* minutes for a lock to be released before failing. Up to *parallelism* requests will be + sent to remote APIs concurrently. + 5. The :func:`tf_posthook` hook is run. + 6. The logs from the apply step are returned as a string. + + :param full: include a Terraform refresh in the automation module run + :return: success status and Terraform apply logs + """ prehook_result = self.tf_prehook() self.tf_generate() self.tf_init() @@ -45,11 +62,12 @@ class TerraformAutomation(BaseAutomation): stdout=subprocess.PIPE) return tf.returncode, tf.stdout.decode('utf-8') - def tf_generate(self): + @abstractmethod + def tf_generate(self) -> None: raise NotImplementedError() def tf_init(self, *, - lock_timeout: int = 15): + lock_timeout: int = 15) -> None: # The init command does not support JSON output subprocess.run( ['terraform', @@ -58,7 +76,7 @@ class TerraformAutomation(BaseAutomation): ], cwd=self.working_directory()) - def tf_output(self) -> Dict[str, Any]: + def tf_output(self) -> Any: tf = subprocess.run( ['terraform', 'output', '-json'], cwd=self.working_directory(), @@ -68,7 +86,7 @@ class TerraformAutomation(BaseAutomation): def tf_plan(self, *, refresh: bool = True, parallelism: Optional[int] = None, - lock_timeout: int = 15): + lock_timeout: int = 15) -> Tuple[int, str]: tf = subprocess.run( ['terraform', 'plan', @@ -78,11 +96,7 @@ class TerraformAutomation(BaseAutomation): f'-lock-timeout={str(lock_timeout)}m', ], cwd=self.working_directory()) - logs = [] - for line in tf.stdout.decode('utf-8').split('\n'): - if line.strip(): - logs.append(json.loads(line)) - return tf.returncode, str(logs) + return tf.returncode, tf.stdout.decode('utf-8') def tf_posthook(self, *, prehook_result: Any = None) -> None: """ @@ -108,14 +122,14 @@ class TerraformAutomation(BaseAutomation): """ pass - def tf_show(self) -> Dict[str, Any]: + def tf_show(self) -> Any: terraform = subprocess.run( ['terraform', 'show', '-json'], cwd=self.working_directory(), stdout=subprocess.PIPE) return json.loads(terraform.stdout) - def tf_write(self, template: str, **kwargs): + def tf_write(self, template: str, **kwargs: Any) -> None: tmpl = jinja2.Template(template) with open(self.working_directory("main.tf"), 'w') as tf: tf.write(tmpl.render(**kwargs))