onions: switch bc2 over to new onion table, drop mirrors table

fixes: #4

app/cli/db.py

@@ -9,7 +9,7 @@ from app import app
 from app.extensions import db
 from app.models.base import Group, MirrorList
 from app.models.bridges import Bridge, BridgeConf
-from app.models.mirrors import Mirror, Origin, Proxy
+from app.models.mirrors import Origin, Proxy
 from app.models.alarms import Alarm, AlarmState
 
 if TYPE_CHECKING:
@@ -23,7 +23,6 @@ models = {
     "alarm": Alarm,
     "group": Group,
     "list": MirrorList,
-    "mirror": Mirror,
     "origin": Origin,
     "proxy": Proxy
 }

app/lists/bc2.py

@@ -1,10 +1,10 @@
 import builtins
 from datetime import datetime
-from typing import List, Dict, Union, Any
+from typing import List, Dict, Union, Any, Optional
 
 from pydantic import BaseModel, Field
 
-from app.models.mirrors import Origin, Proxy, Mirror
+from app.models.mirrors import Origin, Proxy
 
 
 class BC2Alternative(BaseModel):
@@ -16,9 +16,8 @@ class BC2Alternative(BaseModel):
 
 
 class BC2Site(BaseModel):
-    main_domain: str = Field(
+    main_domain: str = Field(description="The main domain name of the website, excluding \"www.\" if present.",
-        description="The main domain name of the website, excluding \"www.\" if present.",
+                             examples=["bbc.co.uk", "bbc.com", "guardianproject.info"])
-        examples=["bbc.co.uk", "bbc.com", "guardianproject.info"])
     available_alternatives: List[BC2Alternative]
 
 
@@ -30,14 +29,18 @@ class BypassCensorship2(BaseModel):
         title = "Bypass Censorship Version 2"
 
 
-def mirror_alternative(mirror: Mirror) -> Dict[str, Any]:
+def onion_alternative(origin: Origin) -> List[Dict[str, Any]]:
-    return {
+    url: Optional[str] = origin.onion()
-        "proto": "tor" if ".onion" in mirror.url else "https",
+    if url is None:
-        "type": "eotk" if ".onion" in mirror.url else "mirror",
+        return []
-        "created_at": str(mirror.added),
+    else:
-        "updated_at": str(mirror.updated),
+        return [{
-        "url": mirror.url
+            "proto": "tor",
-    }
+            "type": "eotk",
+            "created_at": str(origin.added),
+            "updated_at": str(origin.updated),
+            "url": url}
+        ]
 
 
 def proxy_alternative(proxy: Proxy) -> Dict[str, Any]:
@@ -50,15 +53,29 @@ def proxy_alternative(proxy: Proxy) -> Dict[str, Any]:
     }
 
 
-def mirror_sites(provider: str = "cloudfront") -> Dict[str,
+def main_domain(origin: Origin) -> str:
-                                                       Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]:
+    # Both description and domain_name are required to be not null in the database schema
-    return {"version": "2.0", "sites": [{
+    description: str = origin.description
-        "main_domain": x.description[len("proxy:"):].replace("www.", "") if x.description.startswith(
+    if description.startswith("proxy:"):
-            "proxy:") else x.domain_name.replace("www.", ""),
+        return description[len("proxy:"):].replace("www.", "")
-        "available_alternatives": [mirror_alternative(a) for a in x.mirrors if not a.deprecated and not a.destroyed] + [
+    domain_name: str = origin.domain_name
-            proxy_alternative(a) for a in x.proxies if
+    return domain_name.replace("www.", "")
-            a.url is not None and not a.deprecated and not a.destroyed and a.provider == provider]} for x in
+
-        Origin.query.order_by(Origin.domain_name).all() if x.destroyed is None]}
+
+def active_proxies(origin: Origin, provider: str) -> List[Proxy]:
+    def _filter_fn(proxy: Proxy) -> bool:
+        return proxy.url is not None and not proxy.deprecated and not proxy.destroyed and proxy.provider == provider
+    return list(filter(_filter_fn, origin.proxies))
+
+
+def mirror_sites(provider: str = "cloudfront") -> Dict[
+        str, Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]:
+    return {"version": "2.0", "sites": [{"main_domain": main_domain(origin),
+                                         "available_alternatives": onion_alternative(origin) + [
+                                             proxy_alternative(proxy) for proxy in
+                                             active_proxies(origin, provider)]} for origin in
+                                        Origin.query.order_by(Origin.domain_name).all() if
+                                        origin.destroyed is None]}
 
 
 if getattr(builtins, "__sphinx_build__", False):

app/models/mirrors.py

@@ -13,7 +13,6 @@ class Origin(AbstractConfiguration):
     auto_rotation = db.Column(db.Boolean, nullable=False)
 
     group = db.relationship("Group", back_populates="origins")
-    mirrors = db.relationship("Mirror", back_populates="origin")
     proxies = db.relationship("Proxy", back_populates="origin")
     alarms = db.relationship("Alarm", back_populates="origin")
 
@@ -34,7 +33,7 @@ class Origin(AbstractConfiguration):
         if not onion:
             return None
         domain_name: str = self.domain_name
-        return domain_name.replace(tld, f"{onion.onion_name}")
+        return f"https://{domain_name.replace(tld, onion.onion_name)}.onion"
 
 
 class Proxy(AbstractResource):
@@ -53,16 +52,3 @@ class Proxy(AbstractResource):
         return super().csv_header() + [
             "origin_id", "provider", "psg", "slug", "terraform_updated", "url"
         ]
-
-
-class Mirror(AbstractResource):
-    origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)
-    url = db.Column(db.String(255), unique=True, nullable=False)
-
-    origin = db.relationship("Origin", back_populates="mirrors")
-
-    @classmethod
-    def csv_header(cls) -> List[str]:
-        return super().csv_header() + [
-            "origin_id", "url"
-        ]

migrations/versions/1842ba85a5c7_drop_mirror_table.py

@@ -0,0 +1,40 @@
+"""drop mirror table
+
+Revision ID: 1842ba85a5c7
+Revises: 9f5525e84960
+Create Date: 2022-05-17 09:28:13.172061
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '1842ba85a5c7'
+down_revision = '9f5525e84960'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('mirror')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('mirror',
+    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('origin_id', sa.INTEGER(), autoincrement=False, nullable=False),
+    sa.Column('url', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('added', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.Column('updated', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.Column('deprecated', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('destroyed', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('deprecation_reason', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.ForeignKeyConstraint(['origin_id'], ['origin.id'], name='fk_mirror_origin_id_origin'),
+    sa.PrimaryKeyConstraint('id', name='pk_mirror'),
+    sa.UniqueConstraint('url', name='uq_mirror_url')
+    )
+    # ### end Alembic commands ###