list/gen: generate lists based on pools

app/cli/list.py

@@ -6,10 +6,11 @@ from typing import Callable, Any
 from app import app
 from app.cli import _SubparserType, BaseCliHandler
 from app.lists import lists
+from app.models.base import Pool
 
 
-def dump(list_f: Callable[[], Any]) -> None:
-    json.dump(list_f(), sys.stdout, indent=2)
+def dump(list_f: Callable[[Pool], Any]) -> None:
+    json.dump(list_f(Pool.query.first()), sys.stdout, indent=2)
 
 
 class ListCliHandler(BaseCliHandler):

app/lists/__init__.py

@@ -3,8 +3,9 @@ from typing import Dict, Callable, Any
 from app.lists.bc2 import mirror_sites
 from app.lists.bridgelines import bridgelines
 from app.lists.mirror_mapping import mirror_mapping
+from app.models.base import Pool
 
-lists: Dict[str, Callable[[], Any]] = {
+lists: Dict[str, Callable[[Pool], Any]] = {
     "bca": mirror_mapping,
     "bc2": mirror_sites,
     "bridgelines": bridgelines,

app/lists/bc2.py

@@ -6,6 +6,7 @@ from typing import List, Dict, Union, Any, Optional
 
 from pydantic import BaseModel, Field
 
+from app.models.base import Pool
 from app.models.mirrors import Origin, Proxy
 
 
@@ -69,12 +70,12 @@ def active_proxies(origin: Origin, provider: str) -> List[Proxy]:
     return list(filter(_filter_fn, origin.proxies))
 
 
-def mirror_sites(provider: str = "cloudfront") -> Dict[
+def mirror_sites(pool: Pool) -> Dict[
         str, Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]:
     return {"version": "2.0", "sites": [{"main_domain": main_domain(origin),
                                          "available_alternatives": onion_alternative(origin) + [
                                              proxy_alternative(proxy) for proxy in
-                                             active_proxies(origin, provider)]} for origin in
+                                             active_proxies(origin, pool)]} for origin in
                                         Origin.query.order_by(Origin.domain_name).all() if
                                         origin.destroyed is None]}
 

app/lists/bridgelines.py

@@ -5,6 +5,7 @@ from typing import List, Iterable, Dict, Any, Optional
 
 from pydantic import BaseModel, Field
 
+from app.models.base import Pool
 from app.models.bridges import Bridge
 
 
@@ -31,7 +32,7 @@ class Bridgelines(BaseModel):
         title = "Bridgelines Version 1"
 
 
-def bridgelines(*, distribution_method: Optional[str] = None) -> Dict[str, Any]:
+def bridgelines(pool: Pool, *, distribution_method: Optional[str] = None) -> Dict[str, Any]:
     bridges: Iterable[Bridge] = Bridge.query.filter(
         Bridge.destroyed.is_(None),
         Bridge.deprecated.is_(None),

app/lists/mirror_mapping.py

@@ -7,7 +7,7 @@ from flask import current_app
 from pydantic import BaseModel, Field
 from tldextract import extract
 
-from app.models.base import Group
+from app.models.base import Group, Pool
 from app.models.mirrors import Proxy
 
 
@@ -32,7 +32,7 @@ class MirrorMapping(BaseModel):
         title = "Mirror Mapping Version 1.1"
 
 
-def mirror_mapping() -> Dict[str, Union[str, Dict[str, str]]]:
+def mirror_mapping(pool: Pool) -> Dict[str, Union[str, Dict[str, str]]]:
     return MirrorMapping(
         version="1.1",
         mappings={

app/portal/list.py

@@ -60,14 +60,17 @@ def list_list() -> ResponseReturnValue:
                            )
 
 
-@bp.route('/preview/<format_>')
-def list_preview(format_: str) -> ResponseReturnValue:
+@bp.route('/preview/<format_>/<pool_id>')
+def list_preview(format_: str, pool_id: int) -> ResponseReturnValue:
+    pool = Pool.query.filter(Pool.id == pool_id).first()
+    if not pool:
+        return response_404(message="Pool not found")
     if format_ == "bca":
-        return Response(json.dumps(mirror_mapping()), content_type="application/json")
+        return Response(json.dumps(mirror_mapping(pool)), content_type="application/json")
     if format_ == "bc2":
-        return Response(json.dumps(mirror_sites()), content_type="application/json")
+        return Response(json.dumps(mirror_sites(pool)), content_type="application/json")
     if format_ == "bridgelines":
-        return Response(json.dumps(bridgelines()), content_type="application/json")
+        return Response(json.dumps(bridgelines(pool)), content_type="application/json")
     return response_404(message="Format not found")
 
 

app/terraform/list/__init__.py

@@ -5,7 +5,7 @@ from typing import List, Any
 
 from app import app
 from app.lists import lists
-from app.models.base import MirrorList
+from app.models.base import MirrorList, Pool
 from app.terraform.terraform import TerraformAutomation
 
 
@@ -62,11 +62,12 @@ class ListAutomation(TerraformAutomation):
                 for k in self.template_parameters
             }
         )
-        for key, formatter in lists.items():
-            for obfuscate in [True, False]:
-                with open(self.working_directory(f"{key}{'.jsno' if obfuscate else '.json'}"),
-                          'w', encoding="utf-8") as out:
-                    out.write(json_encode(formatter(), obfuscate))
-                with open(self.working_directory(f"{key}{'.jso' if obfuscate else '.js'}"),
-                          'w', encoding="utf-8") as out:
-                    out.write(javascript_encode(formatter(), obfuscate))
+        for pool in Pool.query.filter(Pool.destroyed.is_(None)).all():
+            for key, formatter in lists.items():
+                for obfuscate in [True, False]:
+                    with open(self.working_directory(f"{key}.{pool.pool_name}{'.jsno' if obfuscate else '.json'}"),
+                              'w', encoding="utf-8") as out:
+                        out.write(json_encode(formatter(pool), obfuscate))
+                    with open(self.working_directory(f"{key}.{pool.pool_name}{'.jso' if obfuscate else '.js'}"),
+                              'w', encoding="utf-8") as out:
+                        out.write(javascript_encode(formatter(pool), obfuscate))