jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lists: introduce obfuscated formats

Browse source
This commit is contained in:
Iain Learmonth 2022-05-16 17:09:33 +01:00
parent b1f7426beb
commit 6fe633bf0a
9 changed files with 64 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

12
app/cli/list.py
View file

@ -2,24 +2,16 @@ import argparse
import json
import logging
import sys
from typing import Callable, TYPE_CHECKING, Any, Dict
from typing import Callable, TYPE_CHECKING, Any
from app import app
from app.lists.bc2 import mirror_sites
from app.lists.bridgelines import bridgelines
from app.lists.mirror_mapping import mirror_mapping
from app.lists import lists
if TYPE_CHECKING:
_SubparserType = argparse._SubParsersAction[argparse.ArgumentParser]
else:
_SubparserType = Any
lists: Dict[str, Callable[[], Any]] = {
"mirror_mapping": mirror_mapping,
"bc2": mirror_sites,
"bridgelines": bridgelines,
}
def dump(list_f: Callable[[], Any]) -> None:
json.dump(list_f(), sys.stdout, indent=2)

11
app/lists/__init__.py
View file

@ -0,0 +1,11 @@
from typing import Dict, Callable, Any
from app.lists.bc2 import mirror_sites
from app.lists.bridgelines import bridgelines
from app.lists.mirror_mapping import mirror_mapping
lists: Dict[str, Callable[[], Any]] = {
"mirror_mapping": mirror_mapping,
"bc2": mirror_sites,
"bridgelines": bridgelines,
}

9
app/models/base.py
View file

@ -25,7 +25,7 @@ class Group(AbstractConfiguration):
class MirrorList(AbstractConfiguration):
provider = db.Column(db.String(255), nullable=False)
format = db.Column(db.String(20), nullable=False)
# obfuscate = db.Column(db.Boolean(), nullable=False)
encoding = db.Column(db.String(20), nullable=False)
container = db.Column(db.String(255), nullable=False)
branch = db.Column(db.String(255), nullable=False)
role = db.Column(db.String(255), nullable=True)
@ -44,6 +44,13 @@ class MirrorList(AbstractConfiguration):
"bridgelines": "Tor Bridge Lines"
}
encodings_supported = {
"json": "JSON (Plain)",
"jsno": "JSON (Obfuscated)",
"js": "JavaScript (Plain)",
"jso": "JavaScript (Obfuscated)"
}
def destroy(self) -> None:
self.destroyed = datetime.utcnow()
self.updated = datetime.utcnow()

7
app/portal/list.py
View file

@ -29,6 +29,11 @@ def list_format_name(s: str) -> str:
return MirrorList.formats_supported.get(s, "Unknown")
@bp.app_template_filter("list_encoding_name")
def list_encoding_name(s: str) -> str:
return MirrorList.encodings_supported.get(s, "Unknown")
@bp.route('/list')
def list_list() -> ResponseReturnValue:
lists = MirrorList.query.filter(MirrorList.destroyed.is_(None)).all()
@ -82,6 +87,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue:
form = NewMirrorListForm()
form.provider.choices = [(k, v) for k, v in MirrorList.providers_supported] # type: ignore
form.format.choices = [(k, v) for k, v in MirrorList.formats_supported] # type: ignore
form.encoding.choices = [(k, v) for k, v in MirrorList.encodings_supported] # type: ignore
if form.validate_on_submit():
list_ = MirrorList()
list_.provider = form.provider.data
@ -110,6 +116,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue:
class NewMirrorListForm(FlaskForm): # type: ignore
provider = SelectField('Provider', validators=[DataRequired()])
format = SelectField('Distribution Method', validators=[DataRequired()])
encoding = SelectField('Encoding', validators=[DataRequired()])
description = StringField('Description', validators=[DataRequired()])
container = StringField('Container', validators=[DataRequired()],
description="GitHub Project, GitLab Project or AWS S3 bucket name.")

2
app/portal/templates/tables.html.j2
View file

@ -506,6 +506,7 @@
<tr>
<th scope="col">Provider</th>
<th scope="col">Format</th>
<th scope="col">Encoding</th>
<th scope="col">URI</th>
<th scope="col">Description</th>
<th scope="col">Actions</th>
@ -517,6 +518,7 @@
<tr class="align-middle">
<td>{{ list.provider | provider_name }}</td>
<td>{{ list.format | format_name }}</td>
<td>{{ list.encoding | list_encoding_name }}</td>
<td><a href="{{ list.url() }}">{{ list.url() }}</a></td>
<td>{{ list.description }}</td>
<td>

41
app/terraform/list/__init__.py
View file

@ -1,14 +1,35 @@
from collections.abc import Mapping, Sequence
import json
from typing import List
from typing import List, Any, IO
from app import app
from app.lists.mirror_mapping import mirror_mapping
from app.lists.bc2 import mirror_sites
from app.lists.bridgelines import bridgelines
from app.lists import lists
from app.models.base import MirrorList
from app.terraform.terraform import TerraformAutomation
def obfuscator(obj: Any) -> Any:
if isinstance(obj, str):
return "".join([f"!AAA!{hex(ord(c))[2:].zfill(4)}" for c in obj])
if isinstance(obj, Mapping):
return {obfuscator(k): obfuscator(v) for k, v in obj.items()}
if isinstance(obj, Sequence):
return [obfuscator(i) for i in obj]
return obj
def json_encode(obj: Any, obfuscate: bool) -> str:
if obfuscate:
obj = obfuscator(obj)
s = json.dumps(obj).replace("!AAA!", "\\u")
return s
return json.dumps(obj, indent=2, sort_keys=True)
def javascript_encode(obj: Any, obfuscate: bool) -> str:
return "mirrors = " + json_encode(obj, obfuscate) + ";"
class ListAutomation(TerraformAutomation):
template: str
"""
@ -34,9 +55,9 @@ class ListAutomation(TerraformAutomation):
for k in self.template_parameters
}
)
with open(self.working_directory('bc2.json'), 'w') as out:
json.dump(mirror_sites(), out, indent=2, sort_keys=True)
with open(self.working_directory('bca.json'), 'w') as out:
json.dump(mirror_mapping(), out, indent=2, sort_keys=True)
with open(self.working_directory('bridgelines.json'), 'w') as out:
json.dump(bridgelines(), out, indent=2, sort_keys=True)
for format_ in lists:
for obfuscate in [True, False]:
with open(self.working_directory(f"{format_}.{'.jsno' if obfuscate else '.json'}"), 'w') as out:
out.write(json_encode(lists[format_](), obfuscate))
with open(self.working_directory(f"{format_}.{'.jso' if obfuscate else '.js'}"), 'w') as out:
out.write(javascript_encode(lists[format_](), obfuscate))

2
app/terraform/list/github.py
View file

@ -37,7 +37,7 @@ class ListGithubAutomation(ListAutomation):
repository = data.github_repository.repository_{{ list.id }}.name
branch = "{{ list.branch }}"
file = "{{ list.filename }}"
content = file("{{ list.format }}.json")
content = file("{{ list.format }}.{{ list.encoding }}")
commit_message = "Managed by Terraform"
commit_author = "Terraform User"
commit_email = "terraform@api.otf.is"

2
app/terraform/list/gitlab.py
View file

@ -36,7 +36,7 @@ class ListGitlabAutomation(ListAutomation):
project = data.gitlab_project.project_{{ list.id }}.id
file_path = "{{ list.filename }}"
branch = "{{ list.branch }}"
content = base64encode(file("{{ list.format }}.json"))
content = base64encode(file("{{ list.format }}.{{ list.encoding }}"))
author_email = "{{ gitlab_author_email }}"
author_name = "{{ gitlab_author_name }}"
commit_message = "{{ gitlab_commit_message }}"

2
app/terraform/list/s3.py
View file

@ -39,7 +39,7 @@ class ListS3Automation(ListAutomation):
key = "{{ list.filename }}"
source = "{{ list.format }}.json"
content_type = "application/json"
etag = filemd5("{{ list.format }}.json")
etag = filemd5("{{ list.format }}.{{ list.encoding }}")
}
{% endfor %}
"""