From 076de1bc4423ec2b5adf6d0e34501dadadcc7f47 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Mon, 26 Sep 2022 14:51:11 +0100 Subject: [PATCH] list/gen: generate lists based on pools --- app/cli/list.py | 5 +++-- app/lists/__init__.py | 3 ++- app/lists/bc2.py | 5 +++-- app/lists/bridgelines.py | 3 ++- app/lists/mirror_mapping.py | 4 ++-- app/portal/list.py | 13 ++++++++----- app/terraform/list/__init__.py | 19 ++++++++++--------- 7 files changed, 30 insertions(+), 22 deletions(-) diff --git a/app/cli/list.py b/app/cli/list.py index 5abe454..07ca1b6 100644 --- a/app/cli/list.py +++ b/app/cli/list.py @@ -6,10 +6,11 @@ from typing import Callable, Any from app import app from app.cli import _SubparserType, BaseCliHandler from app.lists import lists +from app.models.base import Pool -def dump(list_f: Callable[[], Any]) -> None: - json.dump(list_f(), sys.stdout, indent=2) +def dump(list_f: Callable[[Pool], Any]) -> None: + json.dump(list_f(Pool.query.first()), sys.stdout, indent=2) class ListCliHandler(BaseCliHandler): diff --git a/app/lists/__init__.py b/app/lists/__init__.py index 27f8dea..9b4a1f7 100644 --- a/app/lists/__init__.py +++ b/app/lists/__init__.py @@ -3,8 +3,9 @@ from typing import Dict, Callable, Any from app.lists.bc2 import mirror_sites from app.lists.bridgelines import bridgelines from app.lists.mirror_mapping import mirror_mapping +from app.models.base import Pool -lists: Dict[str, Callable[[], Any]] = { +lists: Dict[str, Callable[[Pool], Any]] = { "bca": mirror_mapping, "bc2": mirror_sites, "bridgelines": bridgelines, diff --git a/app/lists/bc2.py b/app/lists/bc2.py index 5e151cf..2e6a5e8 100644 --- a/app/lists/bc2.py +++ b/app/lists/bc2.py @@ -6,6 +6,7 @@ from typing import List, Dict, Union, Any, Optional from pydantic import BaseModel, Field +from app.models.base import Pool from app.models.mirrors import Origin, Proxy @@ -69,12 +70,12 @@ def active_proxies(origin: Origin, provider: str) -> List[Proxy]: return list(filter(_filter_fn, origin.proxies)) -def mirror_sites(provider: str = "cloudfront") -> Dict[ +def mirror_sites(pool: Pool) -> Dict[ str, Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]: return {"version": "2.0", "sites": [{"main_domain": main_domain(origin), "available_alternatives": onion_alternative(origin) + [ proxy_alternative(proxy) for proxy in - active_proxies(origin, provider)]} for origin in + active_proxies(origin, pool)]} for origin in Origin.query.order_by(Origin.domain_name).all() if origin.destroyed is None]} diff --git a/app/lists/bridgelines.py b/app/lists/bridgelines.py index c7eda69..3a15f24 100644 --- a/app/lists/bridgelines.py +++ b/app/lists/bridgelines.py @@ -5,6 +5,7 @@ from typing import List, Iterable, Dict, Any, Optional from pydantic import BaseModel, Field +from app.models.base import Pool from app.models.bridges import Bridge @@ -31,7 +32,7 @@ class Bridgelines(BaseModel): title = "Bridgelines Version 1" -def bridgelines(*, distribution_method: Optional[str] = None) -> Dict[str, Any]: +def bridgelines(pool: Pool, *, distribution_method: Optional[str] = None) -> Dict[str, Any]: bridges: Iterable[Bridge] = Bridge.query.filter( Bridge.destroyed.is_(None), Bridge.deprecated.is_(None), diff --git a/app/lists/mirror_mapping.py b/app/lists/mirror_mapping.py index 1d8318d..659326d 100644 --- a/app/lists/mirror_mapping.py +++ b/app/lists/mirror_mapping.py @@ -7,7 +7,7 @@ from flask import current_app from pydantic import BaseModel, Field from tldextract import extract -from app.models.base import Group +from app.models.base import Group, Pool from app.models.mirrors import Proxy @@ -32,7 +32,7 @@ class MirrorMapping(BaseModel): title = "Mirror Mapping Version 1.1" -def mirror_mapping() -> Dict[str, Union[str, Dict[str, str]]]: +def mirror_mapping(pool: Pool) -> Dict[str, Union[str, Dict[str, str]]]: return MirrorMapping( version="1.1", mappings={ diff --git a/app/portal/list.py b/app/portal/list.py index b476d25..20836c1 100644 --- a/app/portal/list.py +++ b/app/portal/list.py @@ -60,14 +60,17 @@ def list_list() -> ResponseReturnValue: ) -@bp.route('/preview/') -def list_preview(format_: str) -> ResponseReturnValue: +@bp.route('/preview//') +def list_preview(format_: str, pool_id: int) -> ResponseReturnValue: + pool = Pool.query.filter(Pool.id == pool_id).first() + if not pool: + return response_404(message="Pool not found") if format_ == "bca": - return Response(json.dumps(mirror_mapping()), content_type="application/json") + return Response(json.dumps(mirror_mapping(pool)), content_type="application/json") if format_ == "bc2": - return Response(json.dumps(mirror_sites()), content_type="application/json") + return Response(json.dumps(mirror_sites(pool)), content_type="application/json") if format_ == "bridgelines": - return Response(json.dumps(bridgelines()), content_type="application/json") + return Response(json.dumps(bridgelines(pool)), content_type="application/json") return response_404(message="Format not found") diff --git a/app/terraform/list/__init__.py b/app/terraform/list/__init__.py index e214901..bd40050 100644 --- a/app/terraform/list/__init__.py +++ b/app/terraform/list/__init__.py @@ -5,7 +5,7 @@ from typing import List, Any from app import app from app.lists import lists -from app.models.base import MirrorList +from app.models.base import MirrorList, Pool from app.terraform.terraform import TerraformAutomation @@ -62,11 +62,12 @@ class ListAutomation(TerraformAutomation): for k in self.template_parameters } ) - for key, formatter in lists.items(): - for obfuscate in [True, False]: - with open(self.working_directory(f"{key}{'.jsno' if obfuscate else '.json'}"), - 'w', encoding="utf-8") as out: - out.write(json_encode(formatter(), obfuscate)) - with open(self.working_directory(f"{key}{'.jso' if obfuscate else '.js'}"), - 'w', encoding="utf-8") as out: - out.write(javascript_encode(formatter(), obfuscate)) + for pool in Pool.query.filter(Pool.destroyed.is_(None)).all(): + for key, formatter in lists.items(): + for obfuscate in [True, False]: + with open(self.working_directory(f"{key}.{pool.pool_name}{'.jsno' if obfuscate else '.json'}"), + 'w', encoding="utf-8") as out: + out.write(json_encode(formatter(pool), obfuscate)) + with open(self.working_directory(f"{key}.{pool.pool_name}{'.jso' if obfuscate else '.js'}"), + 'w', encoding="utf-8") as out: + out.write(javascript_encode(formatter(pool), obfuscate))