lists: introduce obfuscated formats

app/cli/list.py

@@ -2,24 +2,16 @@ import argparse
 import json
 import logging
 import sys
-from typing import Callable, TYPE_CHECKING, Any, Dict
+from typing import Callable, TYPE_CHECKING, Any
 
 from app import app
-from app.lists.bc2 import mirror_sites
+from app.lists import lists
-from app.lists.bridgelines import bridgelines
-from app.lists.mirror_mapping import mirror_mapping
 
 if TYPE_CHECKING:
     _SubparserType = argparse._SubParsersAction[argparse.ArgumentParser]
 else:
     _SubparserType = Any
 
-lists: Dict[str, Callable[[], Any]] = {
-    "mirror_mapping": mirror_mapping,
-    "bc2": mirror_sites,
-    "bridgelines": bridgelines,
-}
-
 
 def dump(list_f: Callable[[], Any]) -> None:
     json.dump(list_f(), sys.stdout, indent=2)

app/lists/__init__.py

@@ -0,0 +1,11 @@
+from typing import Dict, Callable, Any
+
+from app.lists.bc2 import mirror_sites
+from app.lists.bridgelines import bridgelines
+from app.lists.mirror_mapping import mirror_mapping
+
+lists: Dict[str, Callable[[], Any]] = {
+    "mirror_mapping": mirror_mapping,
+    "bc2": mirror_sites,
+    "bridgelines": bridgelines,
+}

app/models/base.py

@@ -25,7 +25,7 @@ class Group(AbstractConfiguration):
 class MirrorList(AbstractConfiguration):
     provider = db.Column(db.String(255), nullable=False)
     format = db.Column(db.String(20), nullable=False)
-    # obfuscate = db.Column(db.Boolean(), nullable=False)
+    encoding = db.Column(db.String(20), nullable=False)
     container = db.Column(db.String(255), nullable=False)
     branch = db.Column(db.String(255), nullable=False)
     role = db.Column(db.String(255), nullable=True)
@@ -44,6 +44,13 @@ class MirrorList(AbstractConfiguration):
         "bridgelines": "Tor Bridge Lines"
     }
 
+    encodings_supported = {
+        "json": "JSON (Plain)",
+        "jsno": "JSON (Obfuscated)",
+        "js": "JavaScript (Plain)",
+        "jso": "JavaScript (Obfuscated)"
+    }
+
     def destroy(self) -> None:
         self.destroyed = datetime.utcnow()
         self.updated = datetime.utcnow()

app/portal/list.py

@@ -29,6 +29,11 @@ def list_format_name(s: str) -> str:
     return MirrorList.formats_supported.get(s, "Unknown")
 
 
+@bp.app_template_filter("list_encoding_name")
+def list_encoding_name(s: str) -> str:
+    return MirrorList.encodings_supported.get(s, "Unknown")
+
+
 @bp.route('/list')
 def list_list() -> ResponseReturnValue:
     lists = MirrorList.query.filter(MirrorList.destroyed.is_(None)).all()
@@ -82,6 +87,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue:
     form = NewMirrorListForm()
     form.provider.choices = [(k, v) for k, v in MirrorList.providers_supported]  # type: ignore
     form.format.choices = [(k, v) for k, v in MirrorList.formats_supported]  # type: ignore
+    form.encoding.choices = [(k, v) for k, v in MirrorList.encodings_supported]  # type: ignore
     if form.validate_on_submit():
         list_ = MirrorList()
         list_.provider = form.provider.data
@@ -110,6 +116,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue:
 class NewMirrorListForm(FlaskForm):  # type: ignore
     provider = SelectField('Provider', validators=[DataRequired()])
     format = SelectField('Distribution Method', validators=[DataRequired()])
+    encoding = SelectField('Encoding', validators=[DataRequired()])
     description = StringField('Description', validators=[DataRequired()])
     container = StringField('Container', validators=[DataRequired()],
                             description="GitHub Project, GitLab Project or AWS S3 bucket name.")

app/portal/templates/tables.html.j2

@@ -506,6 +506,7 @@
             <tr>
                 <th scope="col">Provider</th>
                 <th scope="col">Format</th>
+                <th scope="col">Encoding</th>
                 <th scope="col">URI</th>
                 <th scope="col">Description</th>
                 <th scope="col">Actions</th>
@@ -517,6 +518,7 @@
                     <tr class="align-middle">
                         <td>{{ list.provider | provider_name }}</td>
                         <td>{{ list.format | format_name }}</td>
+                        <td>{{ list.encoding | list_encoding_name }}</td>
                         <td><a href="{{ list.url() }}">{{ list.url() }}</a></td>
                         <td>{{ list.description }}</td>
                         <td>

app/terraform/list/__init__.py

@@ -1,14 +1,35 @@
+from collections.abc import Mapping, Sequence
 import json
-from typing import List
+from typing import List, Any, IO
 
 from app import app
-from app.lists.mirror_mapping import mirror_mapping
+from app.lists import lists
-from app.lists.bc2 import mirror_sites
-from app.lists.bridgelines import bridgelines
 from app.models.base import MirrorList
 from app.terraform.terraform import TerraformAutomation
 
 
+def obfuscator(obj: Any) -> Any:
+    if isinstance(obj, str):
+        return "".join([f"!AAA!{hex(ord(c))[2:].zfill(4)}" for c in obj])
+    if isinstance(obj, Mapping):
+        return {obfuscator(k): obfuscator(v) for k, v in obj.items()}
+    if isinstance(obj, Sequence):
+        return [obfuscator(i) for i in obj]
+    return obj
+
+
+def json_encode(obj: Any, obfuscate: bool) -> str:
+    if obfuscate:
+        obj = obfuscator(obj)
+        s = json.dumps(obj).replace("!AAA!", "\\u")
+        return s
+    return json.dumps(obj, indent=2, sort_keys=True)
+
+
+def javascript_encode(obj: Any, obfuscate: bool) -> str:
+    return "mirrors = " + json_encode(obj, obfuscate) + ";"
+
+
 class ListAutomation(TerraformAutomation):
     template: str
     """
@@ -34,9 +55,9 @@ class ListAutomation(TerraformAutomation):
                 for k in self.template_parameters
             }
         )
-        with open(self.working_directory('bc2.json'), 'w') as out:
+        for format_ in lists:
-            json.dump(mirror_sites(), out, indent=2, sort_keys=True)
+            for obfuscate in [True, False]:
-        with open(self.working_directory('bca.json'), 'w') as out:
+                with open(self.working_directory(f"{format_}.{'.jsno' if obfuscate else '.json'}"), 'w') as out:
-            json.dump(mirror_mapping(), out, indent=2, sort_keys=True)
+                    out.write(json_encode(lists[format_](), obfuscate))
-        with open(self.working_directory('bridgelines.json'), 'w') as out:
+                with open(self.working_directory(f"{format_}.{'.jso' if obfuscate else '.js'}"), 'w') as out:
-            json.dump(bridgelines(), out, indent=2, sort_keys=True)
+                    out.write(javascript_encode(lists[format_](), obfuscate))

app/terraform/list/github.py

@@ -37,7 +37,7 @@ class ListGithubAutomation(ListAutomation):
       repository          = data.github_repository.repository_{{ list.id }}.name
       branch              = "{{ list.branch }}"
       file                = "{{ list.filename }}"
-      content             = file("{{ list.format }}.json")
+      content             = file("{{ list.format }}.{{ list.encoding }}")
       commit_message      = "Managed by Terraform"
       commit_author       = "Terraform User"
       commit_email        = "terraform@api.otf.is"

app/terraform/list/gitlab.py

@@ -36,7 +36,7 @@ class ListGitlabAutomation(ListAutomation):
       project        = data.gitlab_project.project_{{ list.id }}.id
       file_path      = "{{ list.filename }}"
       branch         = "{{ list.branch }}"
-      content        = base64encode(file("{{ list.format }}.json"))
+      content        = base64encode(file("{{ list.format }}.{{ list.encoding }}"))
       author_email   = "{{ gitlab_author_email }}"
       author_name    = "{{ gitlab_author_name }}"
       commit_message = "{{ gitlab_commit_message }}"

app/terraform/list/s3.py

@@ -39,7 +39,7 @@ class ListS3Automation(ListAutomation):
       key                 = "{{ list.filename }}"
       source              = "{{ list.format }}.json"
       content_type        = "application/json"
-      etag                = filemd5("{{ list.format }}.json")
+      etag                = filemd5("{{ list.format }}.{{ list.encoding }}")
     }
     {% endfor %}
     """