diff --git a/app/cli/db.py b/app/cli/db.py index 59a20e2..0691951 100644 --- a/app/cli/db.py +++ b/app/cli/db.py @@ -9,7 +9,7 @@ from app import app from app.extensions import db from app.models.base import Group, MirrorList from app.models.bridges import Bridge, BridgeConf -from app.models.mirrors import Mirror, Origin, Proxy +from app.models.mirrors import Origin, Proxy from app.models.alarms import Alarm, AlarmState if TYPE_CHECKING: @@ -23,7 +23,6 @@ models = { "alarm": Alarm, "group": Group, "list": MirrorList, - "mirror": Mirror, "origin": Origin, "proxy": Proxy } diff --git a/app/lists/bc2.py b/app/lists/bc2.py index 83bac8d..93c1a8c 100644 --- a/app/lists/bc2.py +++ b/app/lists/bc2.py @@ -1,10 +1,10 @@ import builtins from datetime import datetime -from typing import List, Dict, Union, Any +from typing import List, Dict, Union, Any, Optional from pydantic import BaseModel, Field -from app.models.mirrors import Origin, Proxy, Mirror +from app.models.mirrors import Origin, Proxy class BC2Alternative(BaseModel): @@ -16,9 +16,8 @@ class BC2Alternative(BaseModel): class BC2Site(BaseModel): - main_domain: str = Field( - description="The main domain name of the website, excluding \"www.\" if present.", - examples=["bbc.co.uk", "bbc.com", "guardianproject.info"]) + main_domain: str = Field(description="The main domain name of the website, excluding \"www.\" if present.", + examples=["bbc.co.uk", "bbc.com", "guardianproject.info"]) available_alternatives: List[BC2Alternative] @@ -30,14 +29,18 @@ class BypassCensorship2(BaseModel): title = "Bypass Censorship Version 2" -def mirror_alternative(mirror: Mirror) -> Dict[str, Any]: - return { - "proto": "tor" if ".onion" in mirror.url else "https", - "type": "eotk" if ".onion" in mirror.url else "mirror", - "created_at": str(mirror.added), - "updated_at": str(mirror.updated), - "url": mirror.url - } +def onion_alternative(origin: Origin) -> List[Dict[str, Any]]: + url: Optional[str] = origin.onion() + if url is None: + return [] + else: + return [{ + "proto": "tor", + "type": "eotk", + "created_at": str(origin.added), + "updated_at": str(origin.updated), + "url": url} + ] def proxy_alternative(proxy: Proxy) -> Dict[str, Any]: @@ -50,15 +53,29 @@ def proxy_alternative(proxy: Proxy) -> Dict[str, Any]: } -def mirror_sites(provider: str = "cloudfront") -> Dict[str, - Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]: - return {"version": "2.0", "sites": [{ - "main_domain": x.description[len("proxy:"):].replace("www.", "") if x.description.startswith( - "proxy:") else x.domain_name.replace("www.", ""), - "available_alternatives": [mirror_alternative(a) for a in x.mirrors if not a.deprecated and not a.destroyed] + [ - proxy_alternative(a) for a in x.proxies if - a.url is not None and not a.deprecated and not a.destroyed and a.provider == provider]} for x in - Origin.query.order_by(Origin.domain_name).all() if x.destroyed is None]} +def main_domain(origin: Origin) -> str: + # Both description and domain_name are required to be not null in the database schema + description: str = origin.description + if description.startswith("proxy:"): + return description[len("proxy:"):].replace("www.", "") + domain_name: str = origin.domain_name + return domain_name.replace("www.", "") + + +def active_proxies(origin: Origin, provider: str) -> List[Proxy]: + def _filter_fn(proxy: Proxy) -> bool: + return proxy.url is not None and not proxy.deprecated and not proxy.destroyed and proxy.provider == provider + return list(filter(_filter_fn, origin.proxies)) + + +def mirror_sites(provider: str = "cloudfront") -> Dict[ + str, Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]: + return {"version": "2.0", "sites": [{"main_domain": main_domain(origin), + "available_alternatives": onion_alternative(origin) + [ + proxy_alternative(proxy) for proxy in + active_proxies(origin, provider)]} for origin in + Origin.query.order_by(Origin.domain_name).all() if + origin.destroyed is None]} if getattr(builtins, "__sphinx_build__", False): diff --git a/app/models/mirrors.py b/app/models/mirrors.py index 3ea1d4d..7a7f281 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -13,7 +13,6 @@ class Origin(AbstractConfiguration): auto_rotation = db.Column(db.Boolean, nullable=False) group = db.relationship("Group", back_populates="origins") - mirrors = db.relationship("Mirror", back_populates="origin") proxies = db.relationship("Proxy", back_populates="origin") alarms = db.relationship("Alarm", back_populates="origin") @@ -34,7 +33,7 @@ class Origin(AbstractConfiguration): if not onion: return None domain_name: str = self.domain_name - return domain_name.replace(tld, f"{onion.onion_name}") + return f"https://{domain_name.replace(tld, onion.onion_name)}.onion" class Proxy(AbstractResource): @@ -53,16 +52,3 @@ class Proxy(AbstractResource): return super().csv_header() + [ "origin_id", "provider", "psg", "slug", "terraform_updated", "url" ] - - -class Mirror(AbstractResource): - origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False) - url = db.Column(db.String(255), unique=True, nullable=False) - - origin = db.relationship("Origin", back_populates="mirrors") - - @classmethod - def csv_header(cls) -> List[str]: - return super().csv_header() + [ - "origin_id", "url" - ] diff --git a/migrations/versions/1842ba85a5c7_drop_mirror_table.py b/migrations/versions/1842ba85a5c7_drop_mirror_table.py new file mode 100644 index 0000000..700452d --- /dev/null +++ b/migrations/versions/1842ba85a5c7_drop_mirror_table.py @@ -0,0 +1,40 @@ +"""drop mirror table + +Revision ID: 1842ba85a5c7 +Revises: 9f5525e84960 +Create Date: 2022-05-17 09:28:13.172061 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '1842ba85a5c7' +down_revision = '9f5525e84960' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('mirror') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('mirror', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('origin_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('url', sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.Column('added', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('updated', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('deprecated', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('destroyed', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('deprecation_reason', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['origin_id'], ['origin.id'], name='fk_mirror_origin_id_origin'), + sa.PrimaryKeyConstraint('id', name='pk_mirror'), + sa.UniqueConstraint('url', name='uq_mirror_url') + ) + # ### end Alembic commands ###