diff --git a/app/cli/list.py b/app/cli/list.py index 5e4794e..9b86091 100644 --- a/app/cli/list.py +++ b/app/cli/list.py @@ -2,24 +2,16 @@ import argparse import json import logging import sys -from typing import Callable, TYPE_CHECKING, Any, Dict +from typing import Callable, TYPE_CHECKING, Any from app import app -from app.lists.bc2 import mirror_sites -from app.lists.bridgelines import bridgelines -from app.lists.mirror_mapping import mirror_mapping +from app.lists import lists if TYPE_CHECKING: _SubparserType = argparse._SubParsersAction[argparse.ArgumentParser] else: _SubparserType = Any -lists: Dict[str, Callable[[], Any]] = { - "mirror_mapping": mirror_mapping, - "bc2": mirror_sites, - "bridgelines": bridgelines, -} - def dump(list_f: Callable[[], Any]) -> None: json.dump(list_f(), sys.stdout, indent=2) diff --git a/app/lists/__init__.py b/app/lists/__init__.py index e69de29..fde31ad 100644 --- a/app/lists/__init__.py +++ b/app/lists/__init__.py @@ -0,0 +1,11 @@ +from typing import Dict, Callable, Any + +from app.lists.bc2 import mirror_sites +from app.lists.bridgelines import bridgelines +from app.lists.mirror_mapping import mirror_mapping + +lists: Dict[str, Callable[[], Any]] = { + "mirror_mapping": mirror_mapping, + "bc2": mirror_sites, + "bridgelines": bridgelines, +} diff --git a/app/models/base.py b/app/models/base.py index 17b6373..ef611a6 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -25,7 +25,7 @@ class Group(AbstractConfiguration): class MirrorList(AbstractConfiguration): provider = db.Column(db.String(255), nullable=False) format = db.Column(db.String(20), nullable=False) - # obfuscate = db.Column(db.Boolean(), nullable=False) + encoding = db.Column(db.String(20), nullable=False) container = db.Column(db.String(255), nullable=False) branch = db.Column(db.String(255), nullable=False) role = db.Column(db.String(255), nullable=True) @@ -44,6 +44,13 @@ class MirrorList(AbstractConfiguration): "bridgelines": "Tor Bridge Lines" } + encodings_supported = { + "json": "JSON (Plain)", + "jsno": "JSON (Obfuscated)", + "js": "JavaScript (Plain)", + "jso": "JavaScript (Obfuscated)" + } + def destroy(self) -> None: self.destroyed = datetime.utcnow() self.updated = datetime.utcnow() diff --git a/app/portal/list.py b/app/portal/list.py index 1f89f4a..4e80c66 100644 --- a/app/portal/list.py +++ b/app/portal/list.py @@ -29,6 +29,11 @@ def list_format_name(s: str) -> str: return MirrorList.formats_supported.get(s, "Unknown") +@bp.app_template_filter("list_encoding_name") +def list_encoding_name(s: str) -> str: + return MirrorList.encodings_supported.get(s, "Unknown") + + @bp.route('/list') def list_list() -> ResponseReturnValue: lists = MirrorList.query.filter(MirrorList.destroyed.is_(None)).all() @@ -82,6 +87,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue: form = NewMirrorListForm() form.provider.choices = [(k, v) for k, v in MirrorList.providers_supported] # type: ignore form.format.choices = [(k, v) for k, v in MirrorList.formats_supported] # type: ignore + form.encoding.choices = [(k, v) for k, v in MirrorList.encodings_supported] # type: ignore if form.validate_on_submit(): list_ = MirrorList() list_.provider = form.provider.data @@ -110,6 +116,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue: class NewMirrorListForm(FlaskForm): # type: ignore provider = SelectField('Provider', validators=[DataRequired()]) format = SelectField('Distribution Method', validators=[DataRequired()]) + encoding = SelectField('Encoding', validators=[DataRequired()]) description = StringField('Description', validators=[DataRequired()]) container = StringField('Container', validators=[DataRequired()], description="GitHub Project, GitLab Project or AWS S3 bucket name.") diff --git a/app/portal/templates/tables.html.j2 b/app/portal/templates/tables.html.j2 index 32473c8..35ef30c 100644 --- a/app/portal/templates/tables.html.j2 +++ b/app/portal/templates/tables.html.j2 @@ -506,6 +506,7 @@ Provider Format + Encoding URI Description Actions @@ -517,6 +518,7 @@ {{ list.provider | provider_name }} {{ list.format | format_name }} + {{ list.encoding | list_encoding_name }} {{ list.url() }} {{ list.description }} diff --git a/app/terraform/list/__init__.py b/app/terraform/list/__init__.py index 063c7bf..bceaae1 100644 --- a/app/terraform/list/__init__.py +++ b/app/terraform/list/__init__.py @@ -1,14 +1,35 @@ +from collections.abc import Mapping, Sequence import json -from typing import List +from typing import List, Any, IO from app import app -from app.lists.mirror_mapping import mirror_mapping -from app.lists.bc2 import mirror_sites -from app.lists.bridgelines import bridgelines +from app.lists import lists from app.models.base import MirrorList from app.terraform.terraform import TerraformAutomation +def obfuscator(obj: Any) -> Any: + if isinstance(obj, str): + return "".join([f"!AAA!{hex(ord(c))[2:].zfill(4)}" for c in obj]) + if isinstance(obj, Mapping): + return {obfuscator(k): obfuscator(v) for k, v in obj.items()} + if isinstance(obj, Sequence): + return [obfuscator(i) for i in obj] + return obj + + +def json_encode(obj: Any, obfuscate: bool) -> str: + if obfuscate: + obj = obfuscator(obj) + s = json.dumps(obj).replace("!AAA!", "\\u") + return s + return json.dumps(obj, indent=2, sort_keys=True) + + +def javascript_encode(obj: Any, obfuscate: bool) -> str: + return "mirrors = " + json_encode(obj, obfuscate) + ";" + + class ListAutomation(TerraformAutomation): template: str """ @@ -34,9 +55,9 @@ class ListAutomation(TerraformAutomation): for k in self.template_parameters } ) - with open(self.working_directory('bc2.json'), 'w') as out: - json.dump(mirror_sites(), out, indent=2, sort_keys=True) - with open(self.working_directory('bca.json'), 'w') as out: - json.dump(mirror_mapping(), out, indent=2, sort_keys=True) - with open(self.working_directory('bridgelines.json'), 'w') as out: - json.dump(bridgelines(), out, indent=2, sort_keys=True) + for format_ in lists: + for obfuscate in [True, False]: + with open(self.working_directory(f"{format_}.{'.jsno' if obfuscate else '.json'}"), 'w') as out: + out.write(json_encode(lists[format_](), obfuscate)) + with open(self.working_directory(f"{format_}.{'.jso' if obfuscate else '.js'}"), 'w') as out: + out.write(javascript_encode(lists[format_](), obfuscate)) diff --git a/app/terraform/list/github.py b/app/terraform/list/github.py index 60573dc..e2d70e0 100644 --- a/app/terraform/list/github.py +++ b/app/terraform/list/github.py @@ -37,7 +37,7 @@ class ListGithubAutomation(ListAutomation): repository = data.github_repository.repository_{{ list.id }}.name branch = "{{ list.branch }}" file = "{{ list.filename }}" - content = file("{{ list.format }}.json") + content = file("{{ list.format }}.{{ list.encoding }}") commit_message = "Managed by Terraform" commit_author = "Terraform User" commit_email = "terraform@api.otf.is" diff --git a/app/terraform/list/gitlab.py b/app/terraform/list/gitlab.py index ee3572c..9b51e88 100644 --- a/app/terraform/list/gitlab.py +++ b/app/terraform/list/gitlab.py @@ -36,7 +36,7 @@ class ListGitlabAutomation(ListAutomation): project = data.gitlab_project.project_{{ list.id }}.id file_path = "{{ list.filename }}" branch = "{{ list.branch }}" - content = base64encode(file("{{ list.format }}.json")) + content = base64encode(file("{{ list.format }}.{{ list.encoding }}")) author_email = "{{ gitlab_author_email }}" author_name = "{{ gitlab_author_name }}" commit_message = "{{ gitlab_commit_message }}" diff --git a/app/terraform/list/s3.py b/app/terraform/list/s3.py index cdef597..898819c 100644 --- a/app/terraform/list/s3.py +++ b/app/terraform/list/s3.py @@ -39,7 +39,7 @@ class ListS3Automation(ListAutomation): key = "{{ list.filename }}" source = "{{ list.format }}.json" content_type = "application/json" - etag = filemd5("{{ list.format }}.json") + etag = filemd5("{{ list.format }}.{{ list.encoding }}") } {% endfor %} """