feat: geo risk scores

app/lists/mirror_mapping.py

@@ -1,12 +1,15 @@
 #  pylint: disable=too-few-public-methods
 
 import builtins
+from datetime import datetime, timedelta
 from typing import Dict, List, Union, Optional
 
 from flask import current_app
 from pydantic import BaseModel, Field
+from sqlalchemy import or_
 from tldextract import extract
 
+from app.extensions import db
 from app.models.base import Group, Pool
 from app.models.mirrors import Proxy
 
@@ -17,6 +20,9 @@ class MMMirror(BaseModel):
     origin_domain_root: str = Field(description="The registered domain name of the origin, excluding subdomains")
     valid_from: str = Field(description="The date on which the mirror was added to the system")
     valid_to: Optional[str] = Field(description="The date on which the mirror was decommissioned")
+    # countries: List[Tuple[str, int]] = Field(description="A list mapping of risk levels to country")
+    country: Optional[str] = Field(description="The country code of the country in which the origin is targeted")
+    risk: int = Field(description="A risk score for the origin in the target country")
 
 
 class MirrorMapping(BaseModel):
@@ -35,17 +41,40 @@ class MirrorMapping(BaseModel):
 
 
 def mirror_mapping(_: Optional[Pool]) -> Dict[str, Union[str, Dict[str, str]]]:
+    one_week_ago = datetime.utcnow() - timedelta(days=7)
+
+    proxies = (
+        db.session.query(Proxy)  # type: ignore[no-untyped-call]
+        .filter(or_(Proxy.destroyed.is_(None), Proxy.destroyed > one_week_ago))
+        .filter(Proxy.url.is_not(None))
+        .all()
+    )
+
+    result = {}
+    for proxy in proxies:
+        if proxy.origin.countries:  # Check if there are any associated countries
+            risk_levels = proxy.origin.risk_level.items()
+            highest_risk_country = max(risk_levels, key=lambda x: x[1])
+            highest_risk_country_code = highest_risk_country[0]
+            highest_risk_level = highest_risk_country[1]
+        else:
+            highest_risk_country_code = "ZZ"
+            highest_risk_level = 0
+
+        result[proxy.url.lstrip("https://")] = MMMirror(
+            origin_domain=proxy.origin.domain_name,
+            origin_domain_normalized=proxy.origin.domain_name.replace("www.", ""),
+            origin_domain_root=extract(proxy.origin.domain_name).registered_domain,
+            valid_from=proxy.added.isoformat(),
+            valid_to=proxy.destroyed.isoformat() if proxy.destroyed is not None else None,
+            # countries=[],  # TODO: countries,
+            country=highest_risk_country_code,
+            risk=highest_risk_level
+        )
+
     return MirrorMapping(
-        version="1.1",
-        mappings={
-            d.url.lstrip("https://"): MMMirror(
-                origin_domain=d.origin.domain_name,
-                origin_domain_normalized=d.origin.domain_name.replace("www.", ""),
-                origin_domain_root=extract(d.origin.domain_name).registered_domain,
-                valid_from=d.added.isoformat(),
-                valid_to=d.destroyed.isoformat() if d.destroyed is not None else None
-            ) for d in Proxy.query.all() if d.url is not None
-        },
+        version="1.2",
+        mappings=result,
         s3_buckets=[
             f"{current_app.config['GLOBAL_NAMESPACE']}-{g.group_name.lower()}-logs-cloudfront"
             for g in Group.query.filter(Group.destroyed.is_(None)).all()

app/models/__init__.py

@@ -1,7 +1,7 @@
 import logging
 from abc import abstractmethod
 from datetime import datetime
-from typing import Union, List, Optional, Any
+from typing import Union, List, Optional, Any, Dict
 
 from app.brm.brn import BRN
 from app.extensions import db
@@ -37,6 +37,21 @@ class AbstractConfiguration(db.Model):  # type: ignore
         ]
 
 
+class Deprecation(db.Model):  # type: ignore[name-defined,misc]
+    id = db.Column(db.Integer, primary_key=True)
+    resource_type = db.Column(db.String(50))
+    resource_id = db.Column(db.Integer)
+    deprecated_at = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
+    meta = db.Column(db.JSON())
+    reason = db.Column(db.String(), nullable=False)
+
+    @property
+    def resource(self) -> "AbstractResource":
+        from app.models.mirrors import Proxy
+        model = {'Proxy': Proxy}[self.resource_type]
+        return model.query.get(self.resource_id)  # type: ignore[no-any-return]
+
+
 class AbstractResource(db.Model):  # type: ignore
     __abstract__ = True
 
@@ -72,12 +87,14 @@ class AbstractResource(db.Model):  # type: ignore
     def brn(self) -> BRN:
         raise NotImplementedError()
 
-    def deprecate(self, *, reason: str) -> bool:
+    def deprecate(self, *, reason: str, meta: Optional[Dict[str, Any]] = None) -> bool:
         """
         Marks the resource as deprecated. In the event that the resource was already
         deprecated, no change will be recorded and the function will return False.
 
         :param reason: an opaque string that records the deprecation reason
+        :param meta: metadata associated with the deprecation reason, such as the circumstances in which censorship was
+                     detected
         :return: if the proxy was deprecated
         """
         if self.deprecated is None:
@@ -85,10 +102,26 @@ class AbstractResource(db.Model):  # type: ignore
             self.deprecated = datetime.utcnow()
             self.deprecation_reason = reason
             self.updated = datetime.utcnow()
+        if reason not in [d.reason for d in self.deprecations]:
+            new_deprecation = Deprecation(
+                resource_type=type(self).__name__,
+                resource_id=self.id,
+                reason=reason,
+                meta=meta
+            )
+            db.session.add(new_deprecation)
             return True
-        logging.info("Not deprecating %s (reason=%s) because it's already deprecated", self.brn, reason)
+        logging.info("Not deprecating %s (reason=%s) because it's already deprecated with that reason.",
+                     self.brn, reason)
         return False
 
+    @property
+    def deprecations(self) -> List[Deprecation]:
+        return Deprecation.query.filter_by(  # type: ignore[no-any-return]
+            resource_type='Proxy',
+            resource_id=self.id
+        ).all()
+
     def destroy(self) -> None:
         """
         Marks the resource for destruction.

app/models/mirrors.py

@@ -1,5 +1,5 @@
 import json
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Optional, List, Union, Any, Dict
 
 from tldextract import extract
@@ -8,9 +8,17 @@ from werkzeug.datastructures import FileStorage
 from app.brm.brn import BRN
 from app.brm.utils import thumbnail_uploaded_image, create_data_uri, normalize_color
 from app.extensions import db
-from app.models import AbstractConfiguration, AbstractResource
+from app.models import AbstractConfiguration, AbstractResource, Deprecation
 from app.models.onions import Onion
 
+country_origin = db.Table(
+    'country_origin',
+    db.metadata,
+    db.Column('country_id', db.ForeignKey('country.id'), primary_key=True),
+    db.Column('origin_id', db.ForeignKey('origin.id'), primary_key=True),
+    extend_existing=True,
+)
+
 
 class Origin(AbstractConfiguration):
     group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
@@ -18,9 +26,11 @@ class Origin(AbstractConfiguration):
     auto_rotation = db.Column(db.Boolean, nullable=False)
     smart = db.Column(db.Boolean(), nullable=False)
     assets = db.Column(db.Boolean(), nullable=False)
+    risk_level_override = db.Column(db.Integer(), nullable=True)
 
     group = db.relationship("Group", back_populates="origins")
     proxies = db.relationship("Proxy", back_populates="origin")
+    countries = db.relationship("Country", secondary=country_origin, back_populates='origins')
 
     @property
     def brn(self) -> BRN:
@@ -35,7 +45,7 @@ class Origin(AbstractConfiguration):
     @classmethod
     def csv_header(cls) -> List[str]:
         return super().csv_header() + [
-            "group_id", "domain_name", "auto_rotation", "smart"
+            "group_id", "domain_name", "auto_rotation", "smart", "assets", "country"
         ]
 
     def destroy(self) -> None:
@@ -51,6 +61,75 @@ class Origin(AbstractConfiguration):
         domain_name: str = self.domain_name
         return f"https://{domain_name.replace(tld, onion.onion_name)}.onion"
 
+    @property
+    def risk_level(self) -> Dict[str, int]:
+        if self.risk_level_override:
+            return {country.country_code: self.risk_level_override for country in self.countries}
+        frequency_factor = 0
+        recency_factor = 0
+        recent_deprecations = (
+            db.session.query(Deprecation)  # type: ignore[no-untyped-call]
+            .join(Proxy,
+                  Deprecation.resource_id == Proxy.id)
+            .join(Origin, Origin.id == Proxy.origin_id)
+            .filter(
+                Origin.id == self.id,
+                Deprecation.resource_type == 'Proxy',
+                Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168)
+            )
+            .distinct(Proxy.id)
+            .all()
+        )
+        for deprecation in recent_deprecations:
+            recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1)
+            frequency_factor += 1
+        risk_levels: Dict[str, int] = {}
+        for country in self.countries:
+            risk_levels[country.country_code.upper()] = int(max(1, min(10, frequency_factor * recency_factor))) + country.risk_level
+        return risk_levels
+
+
+class Country(AbstractConfiguration):
+    @property
+    def brn(self) -> BRN:
+        return BRN(
+            group_id=0,
+            product="country",
+            provider="iso3166-1",
+            resource_type="alpha2",
+            resource_id=self.country_code
+        )
+
+    country_code = db.Column(db.String(2), nullable=False)
+    risk_level_override = db.Column(db.Integer(), nullable=True)
+
+    origins = db.relationship("Origin", secondary=country_origin, back_populates='countries')
+
+    @property
+    def risk_level(self) -> int:
+        if self.risk_level_override:
+            return int(self.risk_level_override // 2)
+        frequency_factor = 0
+        recency_factor = 0
+        recent_deprecations = (
+            db.session.query(Deprecation)  # type: ignore[no-untyped-call]
+            .join(Proxy,
+                  Deprecation.resource_id == Proxy.id)
+            .join(Origin, Origin.id == Proxy.origin_id)
+            .join(Origin.countries)
+            .filter(
+                Country.id == self.id,
+                Deprecation.resource_type == 'Proxy',
+                Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168)
+            )
+            .distinct(Proxy.id)
+            .all()
+        )
+        for deprecation in recent_deprecations:
+            recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1)
+            frequency_factor += 1
+        return int(max(1, min(10, frequency_factor * recency_factor)))
+
 
 class StaticOrigin(AbstractConfiguration):
     group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)

app/portal/__init__.py

@@ -14,6 +14,7 @@ from app.models.bridges import Bridge
 from app.models.mirrors import Origin, Proxy
 from app.models.base import Group
 from app.models.onions import Eotk
+from app.portal.country import bp as country
 from app.portal.cloud import bp as cloud
 from app.portal.automation import bp as automation
 from app.portal.bridgeconf import bp as bridgeconf
@@ -35,6 +36,7 @@ portal.register_blueprint(automation, url_prefix="/automation")
 portal.register_blueprint(bridgeconf, url_prefix="/bridgeconf")
 portal.register_blueprint(bridge, url_prefix="/bridge")
 portal.register_blueprint(cloud, url_prefix="/cloud")
+portal.register_blueprint(country, url_prefix="/country")
 portal.register_blueprint(eotk, url_prefix="/eotk")
 portal.register_blueprint(group, url_prefix="/group")
 portal.register_blueprint(list_, url_prefix="/list")

app/portal/country.py

@@ -0,0 +1,74 @@
+from datetime import datetime
+
+import sqlalchemy
+from flask import Blueprint, render_template, Response, flash
+from flask.typing import ResponseReturnValue
+from flask_wtf import FlaskForm
+from wtforms import IntegerField, BooleanField, SubmitField
+
+from app.extensions import db
+from app.models.mirrors import Country
+
+bp = Blueprint("country", __name__)
+
+_SECTION_TEMPLATE_VARS = {
+    "section": "country",
+    "help_url": "https://bypass.censorship.guide/user/countries.html"
+}
+
+
+@bp.app_template_filter("country_flag")
+def filter_country_flag(country_code: str) -> str:
+    country_code = country_code.upper()
+
+    # Calculate the regional indicator symbol for each letter in the country code
+    base = ord('\U0001F1E6') - ord('A')
+    flag = ''.join([chr(ord(char) + base) for char in country_code])
+
+    return flag
+
+
+@bp.route('/list')
+def country_list() -> ResponseReturnValue:
+    countries = Country.query.filter(Country.destroyed.is_(None)).all()
+    print(len(countries))
+    return render_template("list.html.j2",
+                           title="Countries",
+                           item="country",
+                           new_link=None,
+                           items=sorted(countries, key=lambda x: x.country_code),
+                           **_SECTION_TEMPLATE_VARS
+                           )
+
+
+class EditCountryForm(FlaskForm):  # type: ignore[misc]
+    risk_level_override = BooleanField("Force Risk Level Override?")
+    risk_level_override_number = IntegerField("Forced Risk Level", description="Number from 0 to 20", default=0)
+    submit = SubmitField('Save Changes')
+
+
+@bp.route('/edit/<country_id>', methods=['GET', 'POST'])
+def country_edit(country_id: int) -> ResponseReturnValue:
+    country = Country.query.filter(Country.id == country_id).first()
+    if country is None:
+        return Response(render_template("error.html.j2",
+                                        section="country",
+                                        header="404 Country Not Found",
+                                        message="The requested country could not be found."),
+                        status=404)
+    form = EditCountryForm(risk_level_override=country.risk_level_override is not None,
+                           risk_level_override_number=country.risk_level_override)
+    if form.validate_on_submit():
+        if form.risk_level_override.data:
+            country.risk_level_override = form.risk_level_override_number.data
+        else:
+            country.risk_level_override = None
+        country.updated = datetime.utcnow()
+        try:
+            db.session.commit()
+            flash("Saved changes to country.", "success")
+        except sqlalchemy.exc.SQLAlchemyError:
+            flash("An error occurred saving the changes to the country.", "danger")
+    return render_template("country.html.j2",
+                           section="country",
+                           country=country, form=form)

app/portal/origin.py

@@ -3,17 +3,18 @@ from datetime import datetime
 from typing import Optional, List
 
 import requests
+import sqlalchemy
 from flask import flash, redirect, url_for, render_template, Response, Blueprint
 from flask.typing import ResponseReturnValue
 from flask_wtf import FlaskForm
 from sqlalchemy import exc
-from wtforms import StringField, SelectField, SubmitField, BooleanField
+from wtforms import StringField, SelectField, SubmitField, BooleanField, IntegerField
 from wtforms.validators import DataRequired
 
 from app.extensions import db
 from app.models.base import Group
-from app.models.mirrors import Origin
-from app.portal.util import response_404, view_lifecycle
+from app.models.mirrors import Origin, Country
+from app.portal.util import response_404, view_lifecycle, LifecycleForm
 
 bp = Blueprint("origin", __name__)
 
@@ -28,15 +29,22 @@ class NewOriginForm(FlaskForm):  # type: ignore
     submit = SubmitField('Save Changes')
 
 
-class EditOriginForm(FlaskForm):  # type: ignore
+class EditOriginForm(FlaskForm):  # type: ignore[misc]
     description = StringField('Description', validators=[DataRequired()])
     group = SelectField('Group', validators=[DataRequired()])
     auto_rotate = BooleanField("Enable auto-rotation?")
     smart_proxy = BooleanField("Requires smart proxy?")
     asset_domain = BooleanField("Used to host assets for other domains?", default=False)
+    risk_level_override = BooleanField("Force Risk Level Override?")
+    risk_level_override_number = IntegerField("Forced Risk Level", description="Number from 0 to 20", default=0)
     submit = SubmitField('Save Changes')
 
 
+class CountrySelectForm(FlaskForm):  # type: ignore[misc]
+    country = SelectField("Country", validators=[DataRequired()])
+    submit = SubmitField('Save Changes', render_kw={"class": "btn btn-success"})
+
+
 def final_domain_name(domain_name: str) -> str:
     session = requests.Session()
     r = session.get(f"https://{domain_name}/", allow_redirects=True, timeout=10)
@@ -84,7 +92,9 @@ def origin_edit(origin_id: int) -> ResponseReturnValue:
                           description=origin.description,
                           auto_rotate=origin.auto_rotation,
                           smart_proxy=origin.smart,
-                          asset_domain=origin.assets)
+                          asset_domain=origin.assets,
+                          risk_level_override=origin.risk_level_override is not None,
+                          risk_level_override_number=origin.risk_level_override)
     form.group.choices = [(x.id, x.group_name) for x in Group.query.all()]
     if form.validate_on_submit():
         origin.group_id = form.group.data
@@ -92,12 +102,16 @@ def origin_edit(origin_id: int) -> ResponseReturnValue:
         origin.auto_rotation = form.auto_rotate.data
         origin.smart = form.smart_proxy.data
         origin.assets = form.asset_domain.data
+        if form.risk_level_override.data:
+            origin.risk_level_override = form.risk_level_override_number.data
+        else:
+            origin.risk_level_override = None
         origin.updated = datetime.utcnow()
         try:
             db.session.commit()
-            flash("Saved changes to group.", "success")
+            flash(f"Saved changes for origin {origin.domain_name}.", "success")
         except exc.SQLAlchemyError:
-            flash("An error occurred saving the changes to the group.", "danger")
+            flash("An error occurred saving the changes to the origin.", "danger")
     return render_template("origin.html.j2",
                            section="origin",
                            origin=origin, form=form)
@@ -144,3 +158,74 @@ def origin_destroy(origin_id: int) -> ResponseReturnValue:
         resource=origin,
         action="destroy"
     )
+
+
+@bp.route('/country_remove/<origin_id>/<country_id>', methods=['GET', 'POST'])
+def origin_country_remove(origin_id: int, country_id: int) -> ResponseReturnValue:
+    origin = Origin.query.filter(Origin.id == origin_id).first()
+    if origin is None:
+        return Response(render_template("error.html.j2",
+                                        section="origin",
+                                        header="404 Pool Not Found",
+                                        message="The requested origin could not be found."),
+                        status=404)
+    country = Country.query.filter(Country.id == country_id).first()
+    if country is None:
+        return Response(render_template("error.html.j2",
+                                        section="origin",
+                                        header="404 Country Not Found",
+                                        message="The requested country could not be found."),
+                        status=404)
+    if country not in origin.countries:
+        return Response(render_template("error.html.j2",
+                                        section="origin",
+                                        header="404 Country Not In Pool",
+                                        message="The requested country could not be found in the specified origin."),
+                        status=404)
+    form = LifecycleForm()
+    if form.validate_on_submit():
+        origin.countries.remove(country)
+        try:
+            db.session.commit()
+            flash("Saved changes to origin.", "success")
+            return redirect(url_for("portal.origin.origin_edit", origin_id=origin.id))
+        except sqlalchemy.exc.SQLAlchemyError:
+            flash("An error occurred saving the changes to the origin.", "danger")
+    return render_template("lifecycle.html.j2",
+                           header=f"Remove {country.country_name} from the {origin.domain_name} origin?",
+                           message="Stop monitoring in this country.",
+                           section="origin",
+                           origin=origin, form=form)
+
+
+@bp.route('/country_add/<origin_id>', methods=['GET', 'POST'])
+def origin_country_add(origin_id: int) -> ResponseReturnValue:
+    origin = Origin.query.filter(Origin.id == origin_id).first()
+    if origin is None:
+        return Response(render_template("error.html.j2",
+                                        section="origin",
+                                        header="404 Origin Not Found",
+                                        message="The requested origin could not be found."),
+                        status=404)
+    form = CountrySelectForm()
+    form.country.choices = [(x.id, f"{x.country_code} - {x.description}") for x in Country.query.all()]
+    if form.validate_on_submit():
+        country = Country.query.filter(Country.id == form.country.data).first()
+        if country is None:
+            return Response(render_template("error.html.j2",
+                                            section="origin",
+                                            header="404 Country Not Found",
+                                            message="The requested country could not be found."),
+                            status=404)
+        origin.countries.append(country)
+        try:
+            db.session.commit()
+            flash("Saved changes to origin.", "success")
+            return redirect(url_for("portal.origin.origin_edit", origin_id=origin.id))
+        except sqlalchemy.exc.SQLAlchemyError:
+            flash("An error occurred saving the changes to the origin.", "danger")
+    return render_template("lifecycle.html.j2",
+                           header=f"Add a country to {origin.domain_name}",
+                           message="Enable monitoring from this country:",
+                           section="origin",
+                           origin=origin, form=form)

app/portal/templates/base.html.j2

@@ -87,6 +87,12 @@
                             {{ icon("cloud") }} Cloud Accounts
                         </a>
                     </li>
+                    <li class="nav-item">
+                        <a class="nav-link{% if section == "country" %} active{% endif %}"
+                           href="{{ url_for("portal.country.country_list") }}">
+                            {{ icon("geo") }} Countries
+                        </a>
+                    </li>
                     <li class="nav-item">
                         <a class="nav-link{% if section == "pool" %} active{% endif %}"
                            href="{{ url_for("portal.pool.pool_list") }}">

app/portal/templates/country.html.j2

@@ -0,0 +1,17 @@
+{% extends "base.html.j2" %}
+{% from 'bootstrap5/form.html' import render_form %}
+{% from "tables.html.j2" import origins_table %}
+
+{% block content %}
+    <h1 class="h2 mt-3">Country</h1>
+    <h2 class="h3">{{ country.description }} {{ country.country_code | country_flag }}</h2>
+
+    <div style="border: 1px solid #666;" class="p-3">
+        {{ render_form(form) }}
+    </div>
+
+    <h3 class="mt-3">Origins</h3>
+    {% if country.origins %}
+        {{ origins_table(country.origins) }}
+    {% endif %}
+{% endblock %}

app/portal/templates/icons.html.j2

@@ -69,6 +69,10 @@
              viewBox="0 0 16 16">
             <path d="M8.354 1.146a.5.5 0 0 0-.708 0l-6 6A.5.5 0 0 0 1.5 7.5v7a.5.5 0 0 0 .5.5h4.5a.5.5 0 0 0 .5-.5v-4h2v4a.5.5 0 0 0 .5.5H14a.5.5 0 0 0 .5-.5v-7a.5.5 0 0 0-.146-.354L13 5.793V2.5a.5.5 0 0 0-.5-.5h-1a.5.5 0 0 0-.5.5v1.293L8.354 1.146zM2.5 14V7.707l5.5-5.5 5.5 5.5V14H10v-4a.5.5 0 0 0-.5-.5h-3a.5.5 0 0 0-.5.5v4H2.5z"/>
         </svg>
+    {% elif i == "geo" %}
+        <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-geo" viewBox="0 0 16 16">
+            <path fill-rule="evenodd" d="M8 1a3 3 0 1 0 0 6 3 3 0 0 0 0-6zM4 4a4 4 0 1 1 4.5 3.969V13.5a.5.5 0 0 1-1 0V7.97A4 4 0 0 1 4 3.999zm2.493 8.574a.5.5 0 0 1-.411.575c-.712.118-1.28.295-1.655.493a1.319 1.319 0 0 0-.37.265.301.301 0 0 0-.057.09V14l.002.008a.147.147 0 0 0 .016.033.617.617 0 0 0 .145.15c.165.13.435.27.813.395.751.25 1.82.414 3.024.414s2.273-.163 3.024-.414c.378-.126.648-.265.813-.395a.619.619 0 0 0 .146-.15.148.148 0 0 0 .015-.033L12 14v-.004a.301.301 0 0 0-.057-.09 1.318 1.318 0 0 0-.37-.264c-.376-.198-.943-.375-1.655-.493a.5.5 0 1 1 .164-.986c.77.127 1.452.328 1.957.594C12.5 13 13 13.4 13 14c0 .426-.26.752-.544.977-.29.228-.68.413-1.116.558-.878.293-2.059.465-3.34.465-1.281 0-2.462-.172-3.34-.465-.436-.145-.826-.33-1.116-.558C3.26 14.752 3 14.426 3 14c0-.599.5-1 .961-1.243.505-.266 1.187-.467 1.957-.594a.5.5 0 0 1 .575.411z"/>
+        </svg>
     {% elif i == "life-preserver" %}
         <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-life-preserver"
              viewBox="0 0 16 16">

app/portal/templates/list.html.j2

@@ -1,5 +1,6 @@
 {% extends "base.html.j2" %}
 {% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, cloud_accounts_table,
+        countries_table,
         eotk_table, groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table,
         pools_table, proxies_table, static_table, webhook_table %}
 
@@ -21,6 +22,8 @@
         {{ bridges_table(items) }}
     {% elif item == "cloud account" %}
         {{ cloud_accounts_table(items) }}
+    {% elif item == "country" %}
+        {{ countries_table(items) }}
     {% elif item == "eotk" %}
         {{ eotk_table(items) }}
     {% elif item == "group" %}

app/portal/templates/origin.html.j2

@@ -1,6 +1,6 @@
 {% extends "base.html.j2" %}
 {% from 'bootstrap5/form.html' import render_form %}
-{% from "tables.html.j2" import alarms_table, bridges_table, proxies_table %}
+{% from "tables.html.j2" import alarms_table, bridges_table, countries_table, proxies_table %}
 
 {% block content %}
     <h1 class="h2 mt-3">Origins</h1>
@@ -14,6 +14,10 @@
         {{ render_form(form) }}
     </div>
 
+    <h3>Countries</h3>
+    <p><a href="{{ url_for("portal.origin.origin_country_add", origin_id=origin.id) }}" class="btn btn-sm btn-success">Add Country</a></p>
+    {{ countries_table(origin.countries, origin) }}
+
     {% if origin.alarms %}
     <h3>Alarms</h3>
     {{ alarms_table(origin.alarms) }}

app/portal/templates/tables.html.j2

@@ -51,6 +51,39 @@
     </div>
 {% endmacro %}
 
+{% macro countries_table(countries, origin=None) %}
+    <div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Country Code</th>
+                <th scope="col">Country</th>
+                <th scope="col">Risk Level</th>
+                <th scope="col">Origins</th>
+                <th scope="col">Actions</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for country in countries %}
+                    <tr class="align-middle">
+                        <td>{{ country.country_code }}</td>
+                        <td>{{ country.description }} {{ country.country_code | country_flag }}</td>
+                        <td>{{ country.risk_level | string }}</td>
+                        <td>{{ country.origins | length }}</td>
+                        <td>
+                            <a href="{{ url_for("portal.country.country_edit", country_id=country.id) }}" class="btn btn-sm btn-primary">View/Edit</a>
+                            {% if origin %}
+                                <a href="{{ url_for("portal.origin.origin_country_remove", origin_id=origin.id, country_id=country.id) }}" class="btn btn-danger btn-sm">Remove</a>
+                            {% endif %}
+                        </td>
+                    </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endmacro %}
+
+
 {% macro instances_table(application, instances) %}
     <div class="table-responsive">
         <table class="table table-striped table-sm">
@@ -110,6 +143,8 @@
     </div>
 {% endmacro %}
 
+
+
 {% macro eotk_table(instances) %}
 {{ instances_table("eotk", instances) }}
 {% endmacro %}
@@ -252,6 +287,7 @@
             <tr>
                 <th scope="col">Name</th>
                 <th scope="col">Description</th>
+                <th scope="col">Risk Level</th>
                 <th scope="col">Auto-Rotation</th>
                 <th scope="col">Smart Proxy</th>
                 <th scope="col">Assets Origin</th>
@@ -270,6 +306,7 @@
                             {{ origin.domain_name }}
                         </td>
                         <td>{{ origin.description }}</td>
+                        <td>{{ origin.risk_level.values() | max | default("n/a") }}</td>
                         <td>{% if origin.auto_rotation %}✅{% else %}❌{% endif %}</td>
                         <td>{% if origin.smart %}✅{% else %}❌{% endif %}</td>
                         <td>{% if origin.assets %}✅{% else %}❌{% endif %}</td>

app/terraform/block_external.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 
 import requests
 
@@ -35,17 +36,26 @@ class BlockExternalAutomation(BlockMirrorAutomation):
 
     def fetch(self) -> None:
         user_agent = {'User-agent': 'BypassCensorship/1.0'}
-        if isinstance(app.config.get('EXTERNAL_CHECK_URL', []), list):
-            check_urls = app.config.get('EXTERNAL_CHECK_URL', [])
-        elif isinstance(app.config.get('EXTERNAL_CHECK_URL'), str):
-            check_urls = [app.config['EXTERNAL_CHECK_URL']]
+        check_urls_config = app.config.get('EXTERNAL_CHECK_URL', [])
+
+        if isinstance(check_urls_config, dict):
+            # Config is already a dictionary, use as is.
+            check_urls = check_urls_config
+        elif isinstance(check_urls_config, list):
+            # Convert list of strings to a dictionary with "external_N" keys.
+            check_urls = {f"external_{i}": url for i, url in enumerate(check_urls_config)}
+        elif isinstance(check_urls_config, str):
+            # Single string, convert to a dictionary with key "external".
+            check_urls = {"external": check_urls_config}
         else:
-            check_urls = []
-        for check_url in check_urls:
+            # Fallback if the config item is neither dict, list, nor string.
+            check_urls = {}
+        for source, check_url in check_urls.items():
             if self._data is None:
-                self._data = []
-            self._data.extend(requests.get(check_url, headers=user_agent, timeout=30).json())
+                self._data = defaultdict(list)
+            self._data[source].extend(requests.get(check_url, headers=user_agent, timeout=30).json())
 
     def parse(self) -> None:
-        self.patterns.extend(["https://" + trim_http_https(pattern) for pattern in self._data])
+        for source, patterns in self._data.items():
+            self.patterns[source].extend(["https://" + trim_http_https(pattern) for pattern in patterns])
         logging.debug("Found URLs: %s", self.patterns)

app/terraform/block_mirror.py

@@ -1,8 +1,9 @@
+from collections import defaultdict
 from datetime import datetime, timedelta
 import logging
 from abc import abstractmethod
 import fnmatch
-from typing import Tuple, List, Any, Optional
+from typing import Tuple, List, Any, Optional, Dict
 
 from app.extensions import db
 from app.models.activity import Activity
@@ -11,14 +12,14 @@ from app.terraform import BaseAutomation
 
 
 class BlockMirrorAutomation(BaseAutomation):
-    patterns: List[str]
+    patterns: Dict[str, List[str]]
     _data: Any
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         """
         Constructor method.
         """
-        self.patterns = []
+        self.patterns = defaultdict(list)
         self._data = None
         super().__init__(*args, **kwargs)
 
@@ -29,7 +30,9 @@ class BlockMirrorAutomation(BaseAutomation):
         logging.debug("Parse complete")
         rotated = []
         proxy_urls = list(filter(lambda u: u is not None, active_proxy_urls()))
-        for pattern in self.patterns:
+        for source, patterns in self.patterns.items():
+            logging.debug("Processing blocked URLs from %s", source)
+            for pattern in patterns:
                 blocked_urls = fnmatch.filter(proxy_urls, pattern)
                 for blocked_url in blocked_urls:
                     if not (proxy := proxy_by_url(blocked_url)):
@@ -41,7 +44,7 @@ class BlockMirrorAutomation(BaseAutomation):
                     if proxy.added > datetime.utcnow() - timedelta(hours=3):
                         logging.debug("Not rotating a proxy less than 3 hours old")
                         continue
-                if proxy.deprecate(reason=self.short_name):
+                    if proxy.deprecate(reason=f"block_{source}"):
                         logging.info("Rotated %s", proxy.url)
                         rotated.append((proxy.url, proxy.origin.domain_name))
                     else:

app/terraform/block_roskomsvoboda.py

@@ -93,7 +93,7 @@ class BlockRoskomsvobodaAutomation(BlockMirrorAutomation):
             for _event, element in lxml.etree.iterparse(BytesIO(self._data),
                                                         resolve_entities=False):
                 if element.tag == "domain":
-                    self.patterns.append("https://" + element.text.strip())
+                    self.patterns["roskomsvoboda"].append("https://" + element.text.strip())
         except XMLSyntaxError:
             activity = Activity(
                 activity_type="automation",

migrations/versions/bbec86de37c4_adds_geo_monitoring.py

@@ -0,0 +1,117 @@
+import json
+from datetime import datetime
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import table, column
+from sqlalchemy.orm import Session
+
+revision = 'bbec86de37c4'
+down_revision = '278bcfb487d3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('country',
+                    sa.Column('country_code', sa.String(length=2), nullable=False),
+                    sa.Column('risk_level_override', sa.Integer(), nullable=True),
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('description', sa.String(length=255), nullable=False),
+                    sa.Column('added', sa.DateTime(), nullable=False),
+                    sa.Column('updated', sa.DateTime(), nullable=False),
+                    sa.Column('destroyed', sa.DateTime(), nullable=True),
+                    sa.PrimaryKeyConstraint('id', name=op.f('pk_country'))
+                    )
+    op.create_table('deprecation',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('resource_type', sa.String(length=50), nullable=True),
+                    sa.Column('resource_id', sa.Integer(), nullable=True),
+                    sa.Column('deprecated_at', sa.DateTime(), nullable=False),
+                    sa.Column('reason', sa.String(), nullable=False),
+                    sa.Column('meta', sa.JSON()),
+                    sa.PrimaryKeyConstraint('id', name=op.f('pk_deprecation'))
+                    )
+    op.create_table('country_origin',
+                    sa.Column('country_id', sa.Integer(), nullable=False),
+                    sa.Column('origin_id', sa.Integer(), nullable=False),
+                    sa.ForeignKeyConstraint(['country_id'], ['country.id'],
+                                            name=op.f('fk_country_origin_country_id_country')),
+                    sa.ForeignKeyConstraint(['origin_id'], ['origin.id'],
+                                            name=op.f('fk_country_origin_origin_id_origin')),
+                    sa.PrimaryKeyConstraint('country_id', 'origin_id', name=op.f('pk_country_origin'))
+                    )
+    with op.batch_alter_table('origin', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('risk_level_override', sa.Integer(), nullable=True))
+
+    countries = json.load(open("migrations/countries.json"))
+
+    country_table = table(
+        'country',
+        column('id', sa.Integer),
+        column('country_code', sa.String),
+        column('description', sa.String),
+        column('risk_level_override', sa.Integer),
+        column('added', sa.DateTime),
+        column('updated', sa.DateTime),
+    )
+
+    # Iterate through each country and insert it using the 'country_table' definition
+    for country in countries:
+        op.execute(
+            country_table.insert().values(
+                country_code=country['Code'],
+                description=country['Name'],
+                risk_level_override=None,  # Assuming risk level override is initially None
+                added=datetime.utcnow(),
+                updated=datetime.utcnow()
+            )
+        )
+
+    deprecation_table = table('deprecation',
+                              column('id', sa.Integer),
+                              column('resource_type', sa.String),
+                              column('resource_id', sa.Integer),
+                              column('deprecated_at', sa.DateTime),
+                              column('reason', sa.String)
+                              )
+
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    resource_tables = ['proxy', 'bridge']
+
+    for table_name in resource_tables:
+        # Query the existing deprecations
+        results = session.execute(
+            sa.select(
+                column('id'),
+                column('deprecated'),
+                column('deprecation_reason')
+            ).select_from(table(table_name))
+        )
+
+        # Iterate over each row and create a corresponding entry in the Deprecation table
+        for id_, deprecated, reason in results:
+            if deprecated is not None:  # Only migrate if there's a deprecation date
+                op.execute(
+                    deprecation_table.insert().values(
+                        resource_type=table_name.title(),  # The class name is used, not the table name
+                        resource_id=id_,
+                        deprecated_at=deprecated,
+                        reason=reason
+                    )
+                )
+
+    session.commit()
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('origin', schema=None) as batch_op:
+        batch_op.drop_column('risk_level_override')
+
+    op.drop_table('country_origin')
+    op.drop_table('deprecation')
+    op.drop_table('country')
+    # ### end Alembic commands ###