feat: draw the rest of the owl

src/mirrors/models.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+
+from sqlalchemy.orm import Mapped
+
+from src.models import CustomBase, IdMixin
+
+
+class Mirror(CustomBase, IdMixin):
+    __tablename__ = "mirror"
+
+    origin: Mapped[str]
+    pool: Mapped[int]
+    mirror: Mapped[str]
+    first_seen: Mapped[datetime]
+    last_seen: Mapped[datetime]
+    # TODO: Record hits when a redirect goes to the mirror
+    hits: Mapped[int] = 0

src/mirrors/router.py

@@ -0,0 +1,27 @@
+from urllib.parse import urlparse
+
+from fastapi import APIRouter
+
+from src.database import DbSession
+from src.mirrors.schemas import MirrorLinks, RedirectorData
+from src.mirrors.service import refresh_mirrors
+from src.security import ApiKey
+
+router = APIRouter()
+
+
+@router.post("/api/v1/mirrors")
+def update_mirrors(db: DbSession, auth: ApiKey, data: RedirectorData):
+    for pool, data in enumerate(data.pools):
+        refresh_mirrors(db, pool, data.origins)
+    db.commit()
+
+
+@router.get("/api/v1/resolve", response_model=MirrorLinks)
+def resolve_mirror(db: DbSession, auth: ApiKey, url: str):
+    parsed = urlparse(url)
+    try:
+        mirror = resolve_mirror(db, parsed.netloc)
+        return {"url": parsed._replace(netloc=mirror)}
+    except ValueError:
+        return {"mirrors": []}

src/mirrors/schemas.py

@@ -0,0 +1,18 @@
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+
+class RedirectorDataPool(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    origins: dict[str, str]
+
+
+class RedirectorData(BaseModel):
+    version: Literal["1.0"]
+    pools: list[RedirectorDataPool]
+
+
+class MirrorLinks(BaseModel):
+    mirrors: list[str]

src/mirrors/service.py

@@ -0,0 +1,75 @@
+import random
+from datetime import datetime, timedelta
+from urllib.parse import urlparse, urlunparse
+
+from sqlalchemy import func
+from sqlalchemy.orm import Session
+
+from src.kldscp.client import KLDSCP_SUPPORTED_ORIGINS, get_kaleidoscope_mirror
+from src.mirrors.models import Mirror
+
+
+def _refresh_mirror(db: Session, mirror: str, origin: str, pool: int):
+    if mirror.startswith("https://"):
+        mirror = mirror[8:]
+    existing = (
+        db.query(Mirror)
+        .filter(Mirror.origin == origin, Mirror.mirror == mirror, Mirror.pool == pool)
+        .first()
+    )
+    if existing:
+        existing.last_seen = func.now()
+    else:
+        db.add(
+            Mirror(
+                origin=origin,
+                mirror=mirror,
+                pool=pool,
+                first_seen=func.now(),
+                last_seen=func.now(),
+            )
+        )
+
+
+def refresh_mirrors(db: Session, pool: int, data: dict[str, str | list[str]]):
+    for key in data:
+        if key.startswith("https://") and key.endswith("/"):
+            origin = key[8:-1]
+        else:
+            origin = key
+        if "/" in origin:
+            # TODO: flag this to operator
+            continue
+        if isinstance(data[key], list):
+            for mirror in data[key]:
+                _refresh_mirror(db, mirror, origin, pool)
+        elif isinstance(data[key], str):
+            _refresh_mirror(db, data[key], origin, pool)
+        else:
+            raise TypeError("data must be dict[str, str | list[str]]")
+    db.query(Mirror).filter(
+        Mirror.pool == pool, Mirror.last_seen < datetime.now() - timedelta(minutes=5)
+    ).delete()
+
+
+def get_mirrors(db: Session, origin: str, pool=None) -> list[str]:
+    if pool is None:
+        pool = [0, -2]
+    elif isinstance(pool, int):
+        pool = [pool]
+    result = db.query(Mirror).filter(Mirror.origin == origin, Mirror.pool.in_(pool)).all()
+    mirrors = [m.mirror for m in result]
+    if not mirrors:
+        if origin in KLDSCP_SUPPORTED_ORIGINS:
+            if (k_mirror := get_kaleidoscope_mirror(origin)) is not None:
+                mirrors.append(k_mirror)
+    return mirrors
+
+
+def resolve_mirror(db: Session, url: str) -> str | None:
+    parsed = urlparse(url)
+    try:
+        mirror = random.choice(get_mirrors(db, parsed.netloc))
+        return urlunparse(parsed._replace(netloc=f"{mirror}"))
+    except IndexError:
+        return None

src/mirrors/tasks.py

@@ -0,0 +1,16 @@
+import requests
+
+from src.database import get_db_session
+from src.mirrors.service import refresh_mirrors
+from src.utils import repeat_every
+
+
+@repeat_every(seconds=600)
+def update_rsf_mirrors():
+    with get_db_session() as db:
+        r = requests.get(
+            "https://raw.githubusercontent.com/RSF-RWB/collateralfreedom/refs/heads/main/sites.json"
+        )
+        mirrors = r.json()
+        refresh_mirrors(db, -2, mirrors)  # Tracking as hardcoded pool -2
+        db.commit()