fix: auto-resolve stuck distribution deletions

app/terraform/proxy/__init__.py

@@ -83,7 +83,7 @@ class ProxyAutomation(TerraformAutomation):
     def tf_prehook(self) -> Optional[Any]:  # pylint: disable=useless-return
         return None
 
-    def tf_posthook(self, *, prehook_result: Any = None) -> None:
+    def tf_posthook(self, *, prehook_result: Any = None, logs: Optional[str] = None) -> None:
         self.import_state(self.tf_show())
 
     def tf_generate(self) -> None:

app/terraform/proxy/cloudfront.py

@@ -1,5 +1,10 @@
+import json
+import re
 from datetime import datetime, timezone
-from typing import Any
+from typing import Any, Optional
+
+import boto3
+from flask import current_app
 
 from app.extensions import db
 from app.models.mirrors import Proxy
@@ -108,6 +113,29 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
     {% endfor %}
     """
 
+    def tf_posthook(self, *, prehook_result: Any = None, logs: Optional[str] = None) -> None:
+        self.import_state(self.tf_show())
+        failed_ids = []
+        for line in logs.strip().split('\n'):
+            try:
+                log_entry = json.loads(line)
+                if log_entry.get("@level") == "error" and "CloudFront Distribution" in log_entry.get("@message", ""):
+                    match = re.search(r'CloudFront Distribution (\w+) cannot be deleted', log_entry["@message"])
+                    if match:
+                        failed_ids.append(match.group(1))
+            except json.JSONDecodeError:
+                continue
+        client = boto3.client(
+            'cloudfront',
+            aws_access_key_id=current_app.config["AWS_ACCESS_KEY"],
+            aws_secret_access_key=current_app.config["AWS_SECRET_KEY"],
+            region_name="us-east-1"
+        )
+        for failed_id in failed_ids:
+            response = client.get_distribution_config(Id=failed_id)
+            etag = response['ETag']
+            client.delete_distribution(Id=failed_id, IfMatch=etag)
+
     def import_state(self, state: Any) -> None:
         if not isinstance(state, dict):
             raise RuntimeError("The Terraform state object returned was not a dict.")

app/terraform/terraform.py

@@ -54,7 +54,7 @@ class TerraformAutomation(BaseAutomation):
         returncode, logs = self.tf_apply(
             self.working_dir, refresh=self.always_refresh or full
         )
-        self.tf_posthook(prehook_result=prehook_result)
+        self.tf_posthook(prehook_result=prehook_result, logs=logs)
         return returncode == 0, logs
 
     def tf_apply(
@@ -143,7 +143,7 @@ class TerraformAutomation(BaseAutomation):
         )
         return tfcmd.returncode, tfcmd.stdout.decode("utf-8")
 
-    def tf_posthook(self, *, prehook_result: Any = None) -> None:
+    def tf_posthook(self, *, prehook_result: Any = None, logs: Optional[str] = None) -> None:
         """
         This hook function is called as part of normal automation, after the
         completion of :func:`tf_apply`.
@@ -151,6 +151,7 @@ class TerraformAutomation(BaseAutomation):
         The default, if not overridden by a subclass, is to do nothing.
 
         :param prehook_result: the returned value of :func:`tf_prehook`
+        :param logs: any available logs from the apply stage
         :return: None
         """