jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(eotk): switch to new autonomous eotk instances

Browse source
This commit is contained in:
Iain Learmonth 2022-12-07 18:13:01 +00:00
parent e28fcc6061
commit c584aa0e90
8 changed files with 117 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitmodules vendored
View file

@ -22,3 +22,6 @@
[submodule "terraform-modules/terraform-aws-tor-bridge"]
path = terraform-modules/terraform-aws-tor-bridge
url = https://github.com/sr2c/terraform-aws-tor-bridge.git
[submodule "terraform-modules/terraform-aws-bc-eotk"]
path = terraform-modules/terraform-aws-bc-eotk
url = https://github.com/sr2c/terraform-aws-bc-eotk.git

2
app/portal/templates/tables.html.j2
View file

@ -98,7 +98,7 @@
<a href="{{ url_for("portal." + application + "." + application + "_conf", group_id=instance.group_id) }}"
class="btn btn-primary btn-sm">Preview Configuration</a>
{% endif %}
<a href="https://{{ instance.region }}.console.aws.amazon.com/systems-manager/session-manager/{{ instance.instance_id }}?region={{ instance.region }}" class="btn btn-outline-secondary btn-sm" target="_ssm">
<a href="https://{{ instance.region[:-1] }}.console.aws.amazon.com/systems-manager/session-manager/{{ instance.instance_id }}?region={{ instance.region[:-1] }}" class="btn btn-outline-secondary btn-sm" target="_ssm">
{{ icon("terminal") }}
</a>
</td>

43
app/terraform/__init__.py
View file

@ -1,9 +1,42 @@
import os
import stat
from typing import Tuple, Any, Optional
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
import jinja2
class DeterministicZip:
"""
Create a zip file deterministically.
Heavily inspired by https://github.com/bboe/deterministic_zip.
"""
zipfile: ZipFile
def __init__(self, filename: str):
self.zipfile = ZipFile(filename, "w")
def __enter__(self) -> "DeterministicZip":
return self
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
self.zipfile.close()
def add_file(self, path: str, contents: bytes) -> None:
permission = 0o555 if os.access(path, os.X_OK) else 0o444
zip_info = ZipInfo()
zip_info.filename = path
zip_info.date_time = (2022, 1, 1, 0, 0, 0)
zip_info.external_attr = (stat.S_IFREG | permission) << 16
self.zipfile.writestr(
zip_info,
contents,
compress_type=ZIP_DEFLATED,
compresslevel=9,
)
class BaseAutomation:
short_name: str = "base"
description: str = "Abstract base automation."
@ -37,3 +70,13 @@ class BaseAutomation:
tmpl = jinja2.Template(template)
with open(os.path.join(self.working_dir, filename), 'w', encoding="utf-8") as tfconf:
tfconf.write(tmpl.render(**kwargs))
def bin_write(self, filename: str, data: bytes, group_id: Optional[int] = None) -> None:
if not self.working_dir:
raise RuntimeError("No working directory specified.")
try:
os.mkdir(os.path.join(self.working_dir, str(group_id)))
except FileExistsError:
pass
with open(os.path.join(self.working_dir, str(group_id) if group_id else "", filename), 'wb') as binfile:
binfile.write(data)

36
app/terraform/eotk/__init__.py
View file

@ -0,0 +1,36 @@
import jinja2
from flask import current_app
from app.models.base import Group
EOTK_CONFIG_TEMPLATE = """
set log_separate 1
set nginx_resolver 127.0.0.53 ipv6=off
set nginx_cache_seconds 60
set nginx_cache_size 64m
set nginx_tmpfile_size 8m
set x_from_onion_value 1
set inject_headers_upstream Bypass-Rate-Limit-Token,{{ bypass_token }}
foreignmap facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion facebook.com
foreignmap twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion twitter.com
set project sites
{% for o in group.onions %}
hardmap {{ o.onion_name }} {{ o.domain_name }}
{%- endfor %}
"""
def eotk_configuration(group: Group) -> str:
"""
Generate an EOTK project configuration for an origin group.
:param group: the origin group
:return: the configuration
"""
tmpl = jinja2.Template(EOTK_CONFIG_TEMPLATE)
return tmpl.render(bypass_token=current_app.config["BYPASS_TOKEN"], group=group)

59
app/terraform/eotk/aws.py
View file

@ -6,6 +6,8 @@ from app import app
from app.extensions import db
from app.models.base import Group
from app.models.onions import Eotk
from app.terraform import DeterministicZip
from app.terraform.eotk import eotk_configuration
from app.terraform.terraform import TerraformAutomation
@ -54,31 +56,25 @@ class EotkAWSAutomation(TerraformAutomation):
region = "us-east-2"
}
provider "aws" {
access_key = "{{ aws_access_key }}"
secret_key = "{{ aws_secret_key }}"
region = "eu-central-1"
alias = "second_region"
}
{% for group in groups %}
module "eotk_{{ group.id }}" {
providers = {
aws = aws,
aws.second_region = aws.second_region
}
source = "sr2c/aws/eotk"
version = "0.0.6"
source = "{{ terraform_modules_path }}/terraform-aws-bc-eotk"
namespace = "{{ global_namespace }}"
tenant = "{{ group.group_name }}"
name = "eotk"
label_order = ["namespace", "tenant", "name", "attributes"]
disable_api_termination = true
configuration_bundle = "{{ group.id }}.zip"
}
output "eotk_instances_{{ group.id }}" {
value = module.eotk_{{ group.id }}.instances
}
{% endfor %}
"""
def tf_generate(self) -> None:
if not self.working_dir:
raise RuntimeError("No working directory specified.")
self.tf_write(
self.template,
groups=Group.query.filter(
@ -97,19 +93,28 @@ class EotkAWSAutomation(TerraformAutomation):
for k in self.template_parameters
}
)
for group in Group.query.filter(
Group.eotk.is_(True),
Group.destroyed.is_(None)
).order_by(Group.id).all():
with DeterministicZip(os.path.join(self.working_dir, f"{group.id}.zip")) as dzip:
dzip.add_file("sites.conf", eotk_configuration(group).encode('utf-8'))
for onion in sorted(group.onions, key=lambda o: o.onion_name): # type: ignore[no-any-return]
dzip.add_file(f"{onion.onion_name}.v3pub.key", onion.onion_public_key)
dzip.add_file(f"{onion.onion_name}.v3sec.key", onion.onion_private_key)
dzip.add_file(f"{onion.onion_name[:20]}-v3.cert", onion.tls_public_key)
dzip.add_file(f"{onion.onion_name[:20]}-v3.pem", onion.tls_private_key)
def tf_posthook(self, *, prehook_result: Any = None) -> None:
state = self.tf_show()
for g in state["values"]["root_module"]["child_modules"]:
if g["address"].startswith("module.eotk_"):
group_id = int(g["address"][len("module.eotk_"):])
for i in g["child_modules"]:
if ".module.instance_" in i["address"]:
instance = int(i["address"][-1])
region = "us-east-2" if instance == 1 else "eu-central-1"
for s in i["child_modules"]:
if s["address"].endswith(".module.instance"):
for x in s["resources"]:
if x["address"].endswith(".module.instance.aws_instance.default[0]"):
update_eotk_instance(group_id, region, x['values']['id'])
for e in Eotk.query.all():
db.session.delete(e)
outputs = self.tf_output()
for output in outputs:
if output.startswith("eotk_instances_"):
try:
group_id = int(output[len("eotk_instance_") + 1:])
for az in outputs[output]['value']:
update_eotk_instance(group_id, az, outputs[output]['value'][az])
except ValueError:
pass
db.session.commit()

2
app/terraform/terraform.py
View file

@ -150,9 +150,9 @@ class TerraformAutomation(BaseAutomation):
"""
def tf_show(self) -> Any:
# This subprocess call doesn't take any user input.
if not self.working_dir:
raise RuntimeError("No working directory specified.")
# This subprocess call doesn't take any user input.
terraform = subprocess.run( # nosec
['terraform', 'show', '-json'],
cwd=self.working_dir,

1
terraform-modules/terraform-aws-bc-eotk Submodule

@ -0,0 +1 @@
Subproject commit fbbf804da7528470558d3e887a6bd44fed5990c4

1
terraform-modules/terraform-aws-eotk

@ -1 +0,0 @@
Subproject commit 4bd8d9b7a72d6aab918f210358c183943ee2d64d