From 32239c379aa4dc3999791fb6e1bdf39935472c60 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Tue, 30 Aug 2022 10:05:12 +0100 Subject: [PATCH] terraform: generate conf with http backend --- app/cli/automate.py | 1 + app/terraform/bridge/__init__.py | 7 +++++++ app/terraform/bridge/aws.py | 1 + app/terraform/bridge/gandi.py | 1 + app/terraform/bridge/hcloud.py | 1 + app/terraform/bridge/ovh.py | 1 + app/terraform/eotk/aws.py | 8 ++++++++ app/terraform/list/__init__.py | 6 ++++++ app/terraform/list/github.py | 1 + app/terraform/list/gitlab.py | 1 + app/terraform/list/s3.py | 1 + app/terraform/proxy/__init__.py | 14 ++++++++++---- app/terraform/proxy/azure_cdn.py | 1 + app/terraform/proxy/cloudfront.py | 1 + app/terraform/proxy/fastly.py | 1 + app/tfstate.py | 1 + 16 files changed, 43 insertions(+), 4 deletions(-) diff --git a/app/cli/automate.py b/app/cli/automate.py index af33232..82635b4 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -105,6 +105,7 @@ def run_job(job_cls: Type[BaseAutomation], *, # to be logged for investigation. Catching more specific exceptions would just mean that # others go unrecorded and are difficult to debug. except Exception as exc: # pylint: disable=broad-except + raise exc trace = TracebackException.from_exception(exc) success = False logs = "\n".join(trace.format()) diff --git a/app/terraform/bridge/__init__.py b/app/terraform/bridge/__init__.py index feabd35..2a78c6f 100644 --- a/app/terraform/bridge/__init__.py +++ b/app/terraform/bridge/__init__.py @@ -1,4 +1,5 @@ import datetime +import os from typing import Iterable, Optional, Any, List from app import app @@ -70,6 +71,12 @@ class BridgeAutomation(TerraformAutomation): BridgeConf.provider == self.provider ).all(), global_namespace=app.config['GLOBAL_NAMESPACE'], + terraform_modules_path=os.path.join(*list(os.path.split(app.root_path))[:-1], 'terraform-modules'), + backend_config=f"""backend "http" {{ + lock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + unlock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + }}""", **{ k: app.config[k.upper()] for k in self.template_parameters diff --git a/app/terraform/bridge/aws.py b/app/terraform/bridge/aws.py index 65f16d8..2199d65 100644 --- a/app/terraform/bridge/aws.py +++ b/app/terraform/bridge/aws.py @@ -14,6 +14,7 @@ class BridgeAWSAutomation(BridgeAutomation): template = """ terraform { + {{ backend_config }} required_providers { aws = { version = "~> 4.2.0" diff --git a/app/terraform/bridge/gandi.py b/app/terraform/bridge/gandi.py index 111082b..a9b9277 100644 --- a/app/terraform/bridge/gandi.py +++ b/app/terraform/bridge/gandi.py @@ -16,6 +16,7 @@ class BridgeGandiAutomation(BridgeAutomation): template = """ terraform { + {{ backend_config }} required_providers { openstack = { source = "terraform-provider-openstack/openstack" diff --git a/app/terraform/bridge/hcloud.py b/app/terraform/bridge/hcloud.py index bb5b148..dfe51f8 100644 --- a/app/terraform/bridge/hcloud.py +++ b/app/terraform/bridge/hcloud.py @@ -13,6 +13,7 @@ class BridgeHcloudAutomation(BridgeAutomation): template = """ terraform { + {{ backend_config }} required_providers { random = { source = "hashicorp/random" diff --git a/app/terraform/bridge/ovh.py b/app/terraform/bridge/ovh.py index 323f8e9..6cb0bb3 100644 --- a/app/terraform/bridge/ovh.py +++ b/app/terraform/bridge/ovh.py @@ -19,6 +19,7 @@ class BridgeOvhAutomation(BridgeAutomation): template = """ terraform { + {{ backend_config }} required_providers { random = { source = "hashicorp/random" diff --git a/app/terraform/eotk/aws.py b/app/terraform/eotk/aws.py index 1315ff3..34cb5de 100644 --- a/app/terraform/eotk/aws.py +++ b/app/terraform/eotk/aws.py @@ -1,4 +1,5 @@ import datetime +import os from typing import Any from app import app @@ -39,6 +40,7 @@ class EotkAWSAutomation(TerraformAutomation): template = """ terraform { + {{ backend_config }} required_providers { aws = { version = "~> 4.4.0" @@ -84,6 +86,12 @@ class EotkAWSAutomation(TerraformAutomation): Group.destroyed.is_(None) ).all(), global_namespace=app.config['GLOBAL_NAMESPACE'], + terraform_modules_path=os.path.join(*list(os.path.split(app.root_path))[:-1], 'terraform-modules'), + backend_config=f"""backend "http" {{ + lock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + unlock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + }}""", **{ k: app.config[k.upper()] for k in self.template_parameters diff --git a/app/terraform/list/__init__.py b/app/terraform/list/__init__.py index ac3b249..d3c9114 100644 --- a/app/terraform/list/__init__.py +++ b/app/terraform/list/__init__.py @@ -50,6 +50,12 @@ class ListAutomation(TerraformAutomation): MirrorList.provider == self.provider, ).all(), global_namespace=app.config['GLOBAL_NAMESPACE'], + terraform_modules_path=os.path.join(*list(os.path.split(app.root_path))[:-1], 'terraform-modules'), + backend_config=f"""backend "http" {{ + lock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + unlock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + }}""", **{ k: app.config[k.upper()] for k in self.template_parameters diff --git a/app/terraform/list/github.py b/app/terraform/list/github.py index c46a500..694ab04 100644 --- a/app/terraform/list/github.py +++ b/app/terraform/list/github.py @@ -17,6 +17,7 @@ class ListGithubAutomation(ListAutomation): template = """ terraform { + {{ backend_config }} required_providers { github = { source = "integrations/github" diff --git a/app/terraform/list/gitlab.py b/app/terraform/list/gitlab.py index 9993d06..6fc6e1a 100644 --- a/app/terraform/list/gitlab.py +++ b/app/terraform/list/gitlab.py @@ -16,6 +16,7 @@ class ListGitlabAutomation(ListAutomation): template = """ terraform { + {{ backend_config }} required_providers { gitlab = { source = "gitlabhq/gitlab" diff --git a/app/terraform/list/s3.py b/app/terraform/list/s3.py index 898819c..9d722d9 100644 --- a/app/terraform/list/s3.py +++ b/app/terraform/list/s3.py @@ -13,6 +13,7 @@ class ListS3Automation(ListAutomation): template = """ terraform { + {{ backend_config }} required_providers { aws = { version = "~> 4.4.0" diff --git a/app/terraform/proxy/__init__.py b/app/terraform/proxy/__init__.py index 9f2d27c..76d44ba 100644 --- a/app/terraform/proxy/__init__.py +++ b/app/terraform/proxy/__init__.py @@ -39,6 +39,10 @@ def update_smart_proxy_instance(group_id: int, instance.instance_id = instance_id +def sp_trusted_prefixes() -> str: + return "\n".join([f"geoip2_proxy {p};" for p in all_cdn_prefixes()]) + + class ProxyAutomation(TerraformAutomation): subgroup_max = math.inf """ @@ -154,14 +158,16 @@ class ProxyAutomation(TerraformAutomation): Proxy.provider == self.provider, Proxy.destroyed.is_(None)).all(), subgroups=self.get_subgroups(), global_namespace=app.config['GLOBAL_NAMESPACE'], bypass_token=app.config['BYPASS_TOKEN'], terraform_modules_path=os.path.join(*list(os.path.split(app.root_path))[:-1], 'terraform-modules'), + backend_config=f"""backend "http" {{ + lock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + unlock_address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + address = "{app.config['TFSTATE_BACKEND']}/{self.short_name}" + }}""", **{k: app.config[k.upper()] for k in self.template_parameters}) if self.smart_proxies: for group in groups: self.sp_config(group) - def sp_trusted_prefixes(self) -> str: - return "\n".join([f"geoip2_proxy {p};" for p in all_cdn_prefixes()]) - def sp_config(self, group: Group) -> None: group_origins: List[Origin] = Origin.query.filter( Origin.group_id == group.id, @@ -174,7 +180,7 @@ class ProxyAutomation(TerraformAutomation): $geoip2_metadata_country_build metadata build_epoch; $geoip2_data_country_code default=US country iso_code; } - """ + self.sp_trusted_prefixes() + """ + """ + sp_trusted_prefixes() + """ geoip2_proxy_recursive on; map $geoip2_data_country_code $redirect_country { default yes; diff --git a/app/terraform/proxy/azure_cdn.py b/app/terraform/proxy/azure_cdn.py index e016ca2..fcd8849 100644 --- a/app/terraform/proxy/azure_cdn.py +++ b/app/terraform/proxy/azure_cdn.py @@ -25,6 +25,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation): template = """ terraform { + {{ backend_config }} required_providers { azurerm = { source = "hashicorp/azurerm" diff --git a/app/terraform/proxy/cloudfront.py b/app/terraform/proxy/cloudfront.py index 739d04d..041fd72 100644 --- a/app/terraform/proxy/cloudfront.py +++ b/app/terraform/proxy/cloudfront.py @@ -25,6 +25,7 @@ class ProxyCloudfrontAutomation(ProxyAutomation): template = """ terraform { + {{ backend_config }} required_providers { acme = { source = "vancluever/acme" diff --git a/app/terraform/proxy/fastly.py b/app/terraform/proxy/fastly.py index 7db721d..18d59d6 100644 --- a/app/terraform/proxy/fastly.py +++ b/app/terraform/proxy/fastly.py @@ -20,6 +20,7 @@ class ProxyFastlyAutomation(ProxyAutomation): template = """ terraform { + {{ backend_config }} required_providers { aws = { version = "~> 4.4.0" diff --git a/app/tfstate.py b/app/tfstate.py index bfa58dd..d079450 100644 --- a/app/tfstate.py +++ b/app/tfstate.py @@ -24,6 +24,7 @@ def handle_update(key): return "OK", 200 state = TerraformState(key=key) if state.lock and not (request.method == "UNLOCK" and request.args.get('ID') is None): + # force-unlock seems to not give an ID to verify so accept no ID being present if json.loads(state.lock)['ID'] != request.args.get('ID'): return Response(state.lock, status=409, content_type="application/json") if request.method == "POST":