feat: geo risk scores

app/models/__init__.py

@@ -1,7 +1,7 @@
 import logging
 from abc import abstractmethod
 from datetime import datetime
-from typing import Union, List, Optional, Any
+from typing import Union, List, Optional, Any, Dict
 
 from app.brm.brn import BRN
 from app.extensions import db
@@ -37,6 +37,21 @@ class AbstractConfiguration(db.Model):  # type: ignore
         ]
 
 
+class Deprecation(db.Model):  # type: ignore[name-defined,misc]
+    id = db.Column(db.Integer, primary_key=True)
+    resource_type = db.Column(db.String(50))
+    resource_id = db.Column(db.Integer)
+    deprecated_at = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
+    meta = db.Column(db.JSON())
+    reason = db.Column(db.String(), nullable=False)
+
+    @property
+    def resource(self) -> "AbstractResource":
+        from app.models.mirrors import Proxy
+        model = {'Proxy': Proxy}[self.resource_type]
+        return model.query.get(self.resource_id)  # type: ignore[no-any-return]
+
+
 class AbstractResource(db.Model):  # type: ignore
     __abstract__ = True
 
@@ -72,12 +87,14 @@ class AbstractResource(db.Model):  # type: ignore
     def brn(self) -> BRN:
         raise NotImplementedError()
 
-    def deprecate(self, *, reason: str) -> bool:
+    def deprecate(self, *, reason: str, meta: Optional[Dict[str, Any]] = None) -> bool:
         """
         Marks the resource as deprecated. In the event that the resource was already
         deprecated, no change will be recorded and the function will return False.
 
         :param reason: an opaque string that records the deprecation reason
+        :param meta: metadata associated with the deprecation reason, such as the circumstances in which censorship was
+                     detected
         :return: if the proxy was deprecated
         """
         if self.deprecated is None:
@@ -85,10 +102,26 @@ class AbstractResource(db.Model):  # type: ignore
             self.deprecated = datetime.utcnow()
             self.deprecation_reason = reason
             self.updated = datetime.utcnow()
+        if reason not in [d.reason for d in self.deprecations]:
+            new_deprecation = Deprecation(
+                resource_type=type(self).__name__,
+                resource_id=self.id,
+                reason=reason,
+                meta=meta
+            )
+            db.session.add(new_deprecation)
             return True
-        logging.info("Not deprecating %s (reason=%s) because it's already deprecated", self.brn, reason)
+        logging.info("Not deprecating %s (reason=%s) because it's already deprecated with that reason.",
+                     self.brn, reason)
         return False
 
+    @property
+    def deprecations(self) -> List[Deprecation]:
+        return Deprecation.query.filter_by(  # type: ignore[no-any-return]
+            resource_type='Proxy',
+            resource_id=self.id
+        ).all()
+
     def destroy(self) -> None:
         """
         Marks the resource for destruction.

app/models/mirrors.py

@@ -1,5 +1,5 @@
 import json
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Optional, List, Union, Any, Dict
 
 from tldextract import extract
@@ -8,9 +8,17 @@ from werkzeug.datastructures import FileStorage
 from app.brm.brn import BRN
 from app.brm.utils import thumbnail_uploaded_image, create_data_uri, normalize_color
 from app.extensions import db
-from app.models import AbstractConfiguration, AbstractResource
+from app.models import AbstractConfiguration, AbstractResource, Deprecation
 from app.models.onions import Onion
 
+country_origin = db.Table(
+    'country_origin',
+    db.metadata,
+    db.Column('country_id', db.ForeignKey('country.id'), primary_key=True),
+    db.Column('origin_id', db.ForeignKey('origin.id'), primary_key=True),
+    extend_existing=True,
+)
+
 
 class Origin(AbstractConfiguration):
     group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
@@ -18,9 +26,11 @@ class Origin(AbstractConfiguration):
     auto_rotation = db.Column(db.Boolean, nullable=False)
     smart = db.Column(db.Boolean(), nullable=False)
     assets = db.Column(db.Boolean(), nullable=False)
+    risk_level_override = db.Column(db.Integer(), nullable=True)
 
     group = db.relationship("Group", back_populates="origins")
     proxies = db.relationship("Proxy", back_populates="origin")
+    countries = db.relationship("Country", secondary=country_origin, back_populates='origins')
 
     @property
     def brn(self) -> BRN:
@@ -35,7 +45,7 @@ class Origin(AbstractConfiguration):
     @classmethod
     def csv_header(cls) -> List[str]:
         return super().csv_header() + [
-            "group_id", "domain_name", "auto_rotation", "smart"
+            "group_id", "domain_name", "auto_rotation", "smart", "assets", "country"
         ]
 
     def destroy(self) -> None:
@@ -51,6 +61,75 @@ class Origin(AbstractConfiguration):
         domain_name: str = self.domain_name
         return f"https://{domain_name.replace(tld, onion.onion_name)}.onion"
 
+    @property
+    def risk_level(self) -> Dict[str, int]:
+        if self.risk_level_override:
+            return {country.country_code: self.risk_level_override for country in self.countries}
+        frequency_factor = 0
+        recency_factor = 0
+        recent_deprecations = (
+            db.session.query(Deprecation)  # type: ignore[no-untyped-call]
+            .join(Proxy,
+                  Deprecation.resource_id == Proxy.id)
+            .join(Origin, Origin.id == Proxy.origin_id)
+            .filter(
+                Origin.id == self.id,
+                Deprecation.resource_type == 'Proxy',
+                Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168)
+            )
+            .distinct(Proxy.id)
+            .all()
+        )
+        for deprecation in recent_deprecations:
+            recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1)
+            frequency_factor += 1
+        risk_levels: Dict[str, int] = {}
+        for country in self.countries:
+            risk_levels[country.country_code.upper()] = int(max(1, min(10, frequency_factor * recency_factor))) + country.risk_level
+        return risk_levels
+
+
+class Country(AbstractConfiguration):
+    @property
+    def brn(self) -> BRN:
+        return BRN(
+            group_id=0,
+            product="country",
+            provider="iso3166-1",
+            resource_type="alpha2",
+            resource_id=self.country_code
+        )
+
+    country_code = db.Column(db.String(2), nullable=False)
+    risk_level_override = db.Column(db.Integer(), nullable=True)
+
+    origins = db.relationship("Origin", secondary=country_origin, back_populates='countries')
+
+    @property
+    def risk_level(self) -> int:
+        if self.risk_level_override:
+            return int(self.risk_level_override // 2)
+        frequency_factor = 0
+        recency_factor = 0
+        recent_deprecations = (
+            db.session.query(Deprecation)  # type: ignore[no-untyped-call]
+            .join(Proxy,
+                  Deprecation.resource_id == Proxy.id)
+            .join(Origin, Origin.id == Proxy.origin_id)
+            .join(Origin.countries)
+            .filter(
+                Country.id == self.id,
+                Deprecation.resource_type == 'Proxy',
+                Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168)
+            )
+            .distinct(Proxy.id)
+            .all()
+        )
+        for deprecation in recent_deprecations:
+            recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1)
+            frequency_factor += 1
+        return int(max(1, min(10, frequency_factor * recency_factor)))
+
 
 class StaticOrigin(AbstractConfiguration):
     group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)