lists: introduce obfuscated formats

app/terraform/list/__init__.py

@@ -1,14 +1,35 @@
+from collections.abc import Mapping, Sequence
 import json
-from typing import List
+from typing import List, Any, IO
 
 from app import app
-from app.lists.mirror_mapping import mirror_mapping
-from app.lists.bc2 import mirror_sites
-from app.lists.bridgelines import bridgelines
+from app.lists import lists
 from app.models.base import MirrorList
 from app.terraform.terraform import TerraformAutomation
 
 
+def obfuscator(obj: Any) -> Any:
+    if isinstance(obj, str):
+        return "".join([f"!AAA!{hex(ord(c))[2:].zfill(4)}" for c in obj])
+    if isinstance(obj, Mapping):
+        return {obfuscator(k): obfuscator(v) for k, v in obj.items()}
+    if isinstance(obj, Sequence):
+        return [obfuscator(i) for i in obj]
+    return obj
+
+
+def json_encode(obj: Any, obfuscate: bool) -> str:
+    if obfuscate:
+        obj = obfuscator(obj)
+        s = json.dumps(obj).replace("!AAA!", "\\u")
+        return s
+    return json.dumps(obj, indent=2, sort_keys=True)
+
+
+def javascript_encode(obj: Any, obfuscate: bool) -> str:
+    return "mirrors = " + json_encode(obj, obfuscate) + ";"
+
+
 class ListAutomation(TerraformAutomation):
     template: str
     """
@@ -34,9 +55,9 @@ class ListAutomation(TerraformAutomation):
                 for k in self.template_parameters
             }
         )
-        with open(self.working_directory('bc2.json'), 'w') as out:
-            json.dump(mirror_sites(), out, indent=2, sort_keys=True)
-        with open(self.working_directory('bca.json'), 'w') as out:
-            json.dump(mirror_mapping(), out, indent=2, sort_keys=True)
-        with open(self.working_directory('bridgelines.json'), 'w') as out:
-            json.dump(bridgelines(), out, indent=2, sort_keys=True)
+        for format_ in lists:
+            for obfuscate in [True, False]:
+                with open(self.working_directory(f"{format_}.{'.jsno' if obfuscate else '.json'}"), 'w') as out:
+                    out.write(json_encode(lists[format_](), obfuscate))
+                with open(self.working_directory(f"{format_}.{'.jso' if obfuscate else '.js'}"), 'w') as out:
+                    out.write(javascript_encode(lists[format_](), obfuscate))

app/terraform/list/github.py

@@ -37,7 +37,7 @@ class ListGithubAutomation(ListAutomation):
       repository          = data.github_repository.repository_{{ list.id }}.name
       branch              = "{{ list.branch }}"
       file                = "{{ list.filename }}"
-      content             = file("{{ list.format }}.json")
+      content             = file("{{ list.format }}.{{ list.encoding }}")
       commit_message      = "Managed by Terraform"
       commit_author       = "Terraform User"
       commit_email        = "terraform@api.otf.is"

app/terraform/list/gitlab.py

@@ -36,7 +36,7 @@ class ListGitlabAutomation(ListAutomation):
       project        = data.gitlab_project.project_{{ list.id }}.id
       file_path      = "{{ list.filename }}"
       branch         = "{{ list.branch }}"
-      content        = base64encode(file("{{ list.format }}.json"))
+      content        = base64encode(file("{{ list.format }}.{{ list.encoding }}"))
       author_email   = "{{ gitlab_author_email }}"
       author_name    = "{{ gitlab_author_name }}"
       commit_message = "{{ gitlab_commit_message }}"

app/terraform/list/s3.py

@@ -39,7 +39,7 @@ class ListS3Automation(ListAutomation):
       key                 = "{{ list.filename }}"
       source              = "{{ list.format }}.json"
       content_type        = "application/json"
-      etag                = filemd5("{{ list.format }}.json")
+      etag                = filemd5("{{ list.format }}.{{ list.encoding }}")
     }
     {% endfor %}
     """