feat(eotk): switch to new autonomous eotk instances

.gitmodules

@@ -22,3 +22,6 @@
 [submodule "terraform-modules/terraform-aws-tor-bridge"]
 	path = terraform-modules/terraform-aws-tor-bridge
 	url = https://github.com/sr2c/terraform-aws-tor-bridge.git
+[submodule "terraform-modules/terraform-aws-bc-eotk"]
+	path = terraform-modules/terraform-aws-bc-eotk
+	url = https://github.com/sr2c/terraform-aws-bc-eotk.git

app/portal/templates/tables.html.j2

@@ -98,7 +98,7 @@
                             <a href="{{ url_for("portal." + application + "." + application + "_conf", group_id=instance.group_id) }}"
                                class="btn btn-primary btn-sm">Preview Configuration</a>
                             {% endif %}
-                            <a href="https://{{ instance.region }}.console.aws.amazon.com/systems-manager/session-manager/{{ instance.instance_id }}?region={{ instance.region }}" class="btn btn-outline-secondary btn-sm" target="_ssm">
+                            <a href="https://{{ instance.region[:-1] }}.console.aws.amazon.com/systems-manager/session-manager/{{ instance.instance_id }}?region={{ instance.region[:-1] }}" class="btn btn-outline-secondary btn-sm" target="_ssm">
                                 {{ icon("terminal") }}
                             </a>
                         </td>

app/terraform/__init__.py

@@ -1,9 +1,42 @@
 import os
+import stat
 from typing import Tuple, Any, Optional
+from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
 
 import jinja2
 
 
+class DeterministicZip:
+    """
+    Create a zip file deterministically.
+
+    Heavily inspired by https://github.com/bboe/deterministic_zip.
+    """
+    zipfile: ZipFile
+
+    def __init__(self, filename: str):
+        self.zipfile = ZipFile(filename, "w")
+
+    def __enter__(self) -> "DeterministicZip":
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.zipfile.close()
+
+    def add_file(self, path: str, contents: bytes) -> None:
+        permission = 0o555 if os.access(path, os.X_OK) else 0o444
+        zip_info = ZipInfo()
+        zip_info.filename = path
+        zip_info.date_time = (2022, 1, 1, 0, 0, 0)
+        zip_info.external_attr = (stat.S_IFREG | permission) << 16
+        self.zipfile.writestr(
+            zip_info,
+            contents,
+            compress_type=ZIP_DEFLATED,
+            compresslevel=9,
+        )
+
+
 class BaseAutomation:
     short_name: str = "base"
     description: str = "Abstract base automation."
@@ -37,3 +70,13 @@ class BaseAutomation:
         tmpl = jinja2.Template(template)
         with open(os.path.join(self.working_dir, filename), 'w', encoding="utf-8") as tfconf:
             tfconf.write(tmpl.render(**kwargs))
+
+    def bin_write(self, filename: str, data: bytes, group_id: Optional[int] = None) -> None:
+        if not self.working_dir:
+            raise RuntimeError("No working directory specified.")
+        try:
+            os.mkdir(os.path.join(self.working_dir, str(group_id)))
+        except FileExistsError:
+            pass
+        with open(os.path.join(self.working_dir, str(group_id) if group_id else "", filename), 'wb') as binfile:
+            binfile.write(data)

app/terraform/eotk/__init__.py

@@ -0,0 +1,36 @@
+import jinja2
+from flask import current_app
+
+from app.models.base import Group
+
+EOTK_CONFIG_TEMPLATE = """
+set log_separate 1
+
+set nginx_resolver 127.0.0.53 ipv6=off
+
+set nginx_cache_seconds 60
+set nginx_cache_size 64m
+set nginx_tmpfile_size 8m
+
+set x_from_onion_value 1
+set inject_headers_upstream Bypass-Rate-Limit-Token,{{ bypass_token }}
+
+foreignmap facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion facebook.com
+foreignmap twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion twitter.com
+
+set project sites
+{% for o in group.onions %}
+hardmap {{ o.onion_name }} {{ o.domain_name }}
+{%- endfor %}
+"""
+
+
+def eotk_configuration(group: Group) -> str:
+    """
+    Generate an EOTK project configuration for an origin group.
+
+    :param group: the origin group
+    :return: the configuration
+    """
+    tmpl = jinja2.Template(EOTK_CONFIG_TEMPLATE)
+    return tmpl.render(bypass_token=current_app.config["BYPASS_TOKEN"], group=group)

app/terraform/eotk/aws.py

@@ -6,6 +6,8 @@ from app import app
 from app.extensions import db
 from app.models.base import Group
 from app.models.onions import Eotk
+from app.terraform import DeterministicZip
+from app.terraform.eotk import eotk_configuration
 from app.terraform.terraform import TerraformAutomation
 
 
@@ -54,31 +56,25 @@ class EotkAWSAutomation(TerraformAutomation):
       region = "us-east-2"
     }
 
-    provider "aws" {
-      access_key = "{{ aws_access_key }}"
-      secret_key = "{{ aws_secret_key }}"
-      region = "eu-central-1"
-      alias = "second_region"
-    }
-
     {% for group in groups %}
     module "eotk_{{ group.id }}" {
-      providers = {
+      source = "{{ terraform_modules_path }}/terraform-aws-bc-eotk"
-        aws = aws,
-        aws.second_region = aws.second_region
-      }
-      source = "sr2c/aws/eotk"
-      version = "0.0.6"
       namespace = "{{ global_namespace }}"
       tenant = "{{ group.group_name }}"
       name = "eotk"
       label_order = ["namespace", "tenant", "name", "attributes"]
-      disable_api_termination = true
+      configuration_bundle = "{{ group.id }}.zip"
+    }
+
+    output "eotk_instances_{{ group.id }}" {
+      value = module.eotk_{{ group.id }}.instances
     }
     {% endfor %}
     """
 
     def tf_generate(self) -> None:
+        if not self.working_dir:
+            raise RuntimeError("No working directory specified.")
         self.tf_write(
             self.template,
             groups=Group.query.filter(
@@ -97,19 +93,28 @@ class EotkAWSAutomation(TerraformAutomation):
                 for k in self.template_parameters
             }
         )
+        for group in Group.query.filter(
+                Group.eotk.is_(True),
+                Group.destroyed.is_(None)
+        ).order_by(Group.id).all():
+            with DeterministicZip(os.path.join(self.working_dir, f"{group.id}.zip")) as dzip:
+                dzip.add_file("sites.conf", eotk_configuration(group).encode('utf-8'))
+                for onion in sorted(group.onions, key=lambda o: o.onion_name):  # type: ignore[no-any-return]
+                    dzip.add_file(f"{onion.onion_name}.v3pub.key", onion.onion_public_key)
+                    dzip.add_file(f"{onion.onion_name}.v3sec.key", onion.onion_private_key)
+                    dzip.add_file(f"{onion.onion_name[:20]}-v3.cert", onion.tls_public_key)
+                    dzip.add_file(f"{onion.onion_name[:20]}-v3.pem", onion.tls_private_key)
 
     def tf_posthook(self, *, prehook_result: Any = None) -> None:
-        state = self.tf_show()
+        for e in Eotk.query.all():
-        for g in state["values"]["root_module"]["child_modules"]:
+            db.session.delete(e)
-            if g["address"].startswith("module.eotk_"):
+        outputs = self.tf_output()
-                group_id = int(g["address"][len("module.eotk_"):])
+        for output in outputs:
-                for i in g["child_modules"]:
+            if output.startswith("eotk_instances_"):
-                    if ".module.instance_" in i["address"]:
+                try:
-                        instance = int(i["address"][-1])
+                    group_id = int(output[len("eotk_instance_") + 1:])
-                        region = "us-east-2" if instance == 1 else "eu-central-1"
+                    for az in outputs[output]['value']:
-                        for s in i["child_modules"]:
+                        update_eotk_instance(group_id, az, outputs[output]['value'][az])
-                            if s["address"].endswith(".module.instance"):
+                except ValueError:
-                                for x in s["resources"]:
+                    pass
-                                    if x["address"].endswith(".module.instance.aws_instance.default[0]"):
-                                        update_eotk_instance(group_id, region, x['values']['id'])
         db.session.commit()

app/terraform/terraform.py

@@ -150,9 +150,9 @@ class TerraformAutomation(BaseAutomation):
         """
 
     def tf_show(self) -> Any:
-        # This subprocess call doesn't take any user input.
         if not self.working_dir:
             raise RuntimeError("No working directory specified.")
+        # This subprocess call doesn't take any user input.
         terraform = subprocess.run(  # nosec
             ['terraform', 'show', '-json'],
             cwd=self.working_dir,

terraform-modules/terraform-aws-bc-eotk

@@ -0,0 +1 @@
+Subproject commit fbbf804da7528470558d3e887a6bd44fed5990c4

terraform-modules/terraform-aws-eotk

@@ -1 +0,0 @@
-Subproject commit 4bd8d9b7a72d6aab918f210358c183943ee2d64d