diff --git a/app/lists/mirror_mapping.py b/app/lists/mirror_mapping.py index 803822e..248a599 100644 --- a/app/lists/mirror_mapping.py +++ b/app/lists/mirror_mapping.py @@ -1,12 +1,15 @@ # pylint: disable=too-few-public-methods import builtins +from datetime import datetime, timedelta from typing import Dict, List, Union, Optional from flask import current_app from pydantic import BaseModel, Field +from sqlalchemy import or_ from tldextract import extract +from app.extensions import db from app.models.base import Group, Pool from app.models.mirrors import Proxy @@ -17,6 +20,9 @@ class MMMirror(BaseModel): origin_domain_root: str = Field(description="The registered domain name of the origin, excluding subdomains") valid_from: str = Field(description="The date on which the mirror was added to the system") valid_to: Optional[str] = Field(description="The date on which the mirror was decommissioned") + # countries: List[Tuple[str, int]] = Field(description="A list mapping of risk levels to country") + country: Optional[str] = Field(description="The country code of the country in which the origin is targeted") + risk: int = Field(description="A risk score for the origin in the target country") class MirrorMapping(BaseModel): @@ -35,17 +41,40 @@ class MirrorMapping(BaseModel): def mirror_mapping(_: Optional[Pool]) -> Dict[str, Union[str, Dict[str, str]]]: + one_week_ago = datetime.utcnow() - timedelta(days=7) + + proxies = ( + db.session.query(Proxy) # type: ignore[no-untyped-call] + .filter(or_(Proxy.destroyed.is_(None), Proxy.destroyed > one_week_ago)) + .filter(Proxy.url.is_not(None)) + .all() + ) + + result = {} + for proxy in proxies: + if proxy.origin.countries: # Check if there are any associated countries + risk_levels = proxy.origin.risk_level.items() + highest_risk_country = max(risk_levels, key=lambda x: x[1]) + highest_risk_country_code = highest_risk_country[0] + highest_risk_level = highest_risk_country[1] + else: + highest_risk_country_code = "ZZ" + highest_risk_level = 0 + + result[proxy.url.lstrip("https://")] = MMMirror( + origin_domain=proxy.origin.domain_name, + origin_domain_normalized=proxy.origin.domain_name.replace("www.", ""), + origin_domain_root=extract(proxy.origin.domain_name).registered_domain, + valid_from=proxy.added.isoformat(), + valid_to=proxy.destroyed.isoformat() if proxy.destroyed is not None else None, + # countries=[], # TODO: countries, + country=highest_risk_country_code, + risk=highest_risk_level + ) + return MirrorMapping( - version="1.1", - mappings={ - d.url.lstrip("https://"): MMMirror( - origin_domain=d.origin.domain_name, - origin_domain_normalized=d.origin.domain_name.replace("www.", ""), - origin_domain_root=extract(d.origin.domain_name).registered_domain, - valid_from=d.added.isoformat(), - valid_to=d.destroyed.isoformat() if d.destroyed is not None else None - ) for d in Proxy.query.all() if d.url is not None - }, + version="1.2", + mappings=result, s3_buckets=[ f"{current_app.config['GLOBAL_NAMESPACE']}-{g.group_name.lower()}-logs-cloudfront" for g in Group.query.filter(Group.destroyed.is_(None)).all() diff --git a/app/models/__init__.py b/app/models/__init__.py index 6f4507a..b7b4b9e 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,7 +1,7 @@ import logging from abc import abstractmethod from datetime import datetime -from typing import Union, List, Optional, Any +from typing import Union, List, Optional, Any, Dict from app.brm.brn import BRN from app.extensions import db @@ -37,6 +37,21 @@ class AbstractConfiguration(db.Model): # type: ignore ] +class Deprecation(db.Model): # type: ignore[name-defined,misc] + id = db.Column(db.Integer, primary_key=True) + resource_type = db.Column(db.String(50)) + resource_id = db.Column(db.Integer) + deprecated_at = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False) + meta = db.Column(db.JSON()) + reason = db.Column(db.String(), nullable=False) + + @property + def resource(self) -> "AbstractResource": + from app.models.mirrors import Proxy + model = {'Proxy': Proxy}[self.resource_type] + return model.query.get(self.resource_id) # type: ignore[no-any-return] + + class AbstractResource(db.Model): # type: ignore __abstract__ = True @@ -72,12 +87,14 @@ class AbstractResource(db.Model): # type: ignore def brn(self) -> BRN: raise NotImplementedError() - def deprecate(self, *, reason: str) -> bool: + def deprecate(self, *, reason: str, meta: Optional[Dict[str, Any]] = None) -> bool: """ Marks the resource as deprecated. In the event that the resource was already deprecated, no change will be recorded and the function will return False. :param reason: an opaque string that records the deprecation reason + :param meta: metadata associated with the deprecation reason, such as the circumstances in which censorship was + detected :return: if the proxy was deprecated """ if self.deprecated is None: @@ -85,10 +102,26 @@ class AbstractResource(db.Model): # type: ignore self.deprecated = datetime.utcnow() self.deprecation_reason = reason self.updated = datetime.utcnow() + if reason not in [d.reason for d in self.deprecations]: + new_deprecation = Deprecation( + resource_type=type(self).__name__, + resource_id=self.id, + reason=reason, + meta=meta + ) + db.session.add(new_deprecation) return True - logging.info("Not deprecating %s (reason=%s) because it's already deprecated", self.brn, reason) + logging.info("Not deprecating %s (reason=%s) because it's already deprecated with that reason.", + self.brn, reason) return False + @property + def deprecations(self) -> List[Deprecation]: + return Deprecation.query.filter_by( # type: ignore[no-any-return] + resource_type='Proxy', + resource_id=self.id + ).all() + def destroy(self) -> None: """ Marks the resource for destruction. diff --git a/app/models/mirrors.py b/app/models/mirrors.py index c35e58a..46a80ea 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -1,5 +1,5 @@ import json -from datetime import datetime +from datetime import datetime, timedelta from typing import Optional, List, Union, Any, Dict from tldextract import extract @@ -8,9 +8,17 @@ from werkzeug.datastructures import FileStorage from app.brm.brn import BRN from app.brm.utils import thumbnail_uploaded_image, create_data_uri, normalize_color from app.extensions import db -from app.models import AbstractConfiguration, AbstractResource +from app.models import AbstractConfiguration, AbstractResource, Deprecation from app.models.onions import Onion +country_origin = db.Table( + 'country_origin', + db.metadata, + db.Column('country_id', db.ForeignKey('country.id'), primary_key=True), + db.Column('origin_id', db.ForeignKey('origin.id'), primary_key=True), + extend_existing=True, +) + class Origin(AbstractConfiguration): group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False) @@ -18,9 +26,11 @@ class Origin(AbstractConfiguration): auto_rotation = db.Column(db.Boolean, nullable=False) smart = db.Column(db.Boolean(), nullable=False) assets = db.Column(db.Boolean(), nullable=False) + risk_level_override = db.Column(db.Integer(), nullable=True) group = db.relationship("Group", back_populates="origins") proxies = db.relationship("Proxy", back_populates="origin") + countries = db.relationship("Country", secondary=country_origin, back_populates='origins') @property def brn(self) -> BRN: @@ -35,7 +45,7 @@ class Origin(AbstractConfiguration): @classmethod def csv_header(cls) -> List[str]: return super().csv_header() + [ - "group_id", "domain_name", "auto_rotation", "smart" + "group_id", "domain_name", "auto_rotation", "smart", "assets", "country" ] def destroy(self) -> None: @@ -51,6 +61,75 @@ class Origin(AbstractConfiguration): domain_name: str = self.domain_name return f"https://{domain_name.replace(tld, onion.onion_name)}.onion" + @property + def risk_level(self) -> Dict[str, int]: + if self.risk_level_override: + return {country.country_code: self.risk_level_override for country in self.countries} + frequency_factor = 0 + recency_factor = 0 + recent_deprecations = ( + db.session.query(Deprecation) # type: ignore[no-untyped-call] + .join(Proxy, + Deprecation.resource_id == Proxy.id) + .join(Origin, Origin.id == Proxy.origin_id) + .filter( + Origin.id == self.id, + Deprecation.resource_type == 'Proxy', + Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168) + ) + .distinct(Proxy.id) + .all() + ) + for deprecation in recent_deprecations: + recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1) + frequency_factor += 1 + risk_levels: Dict[str, int] = {} + for country in self.countries: + risk_levels[country.country_code.upper()] = int(max(1, min(10, frequency_factor * recency_factor))) + country.risk_level + return risk_levels + + +class Country(AbstractConfiguration): + @property + def brn(self) -> BRN: + return BRN( + group_id=0, + product="country", + provider="iso3166-1", + resource_type="alpha2", + resource_id=self.country_code + ) + + country_code = db.Column(db.String(2), nullable=False) + risk_level_override = db.Column(db.Integer(), nullable=True) + + origins = db.relationship("Origin", secondary=country_origin, back_populates='countries') + + @property + def risk_level(self) -> int: + if self.risk_level_override: + return int(self.risk_level_override // 2) + frequency_factor = 0 + recency_factor = 0 + recent_deprecations = ( + db.session.query(Deprecation) # type: ignore[no-untyped-call] + .join(Proxy, + Deprecation.resource_id == Proxy.id) + .join(Origin, Origin.id == Proxy.origin_id) + .join(Origin.countries) + .filter( + Country.id == self.id, + Deprecation.resource_type == 'Proxy', + Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168) + ) + .distinct(Proxy.id) + .all() + ) + for deprecation in recent_deprecations: + recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1) + frequency_factor += 1 + return int(max(1, min(10, frequency_factor * recency_factor))) + class StaticOrigin(AbstractConfiguration): group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False) diff --git a/app/portal/__init__.py b/app/portal/__init__.py index 23e8dff..eb1271f 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -14,6 +14,7 @@ from app.models.bridges import Bridge from app.models.mirrors import Origin, Proxy from app.models.base import Group from app.models.onions import Eotk +from app.portal.country import bp as country from app.portal.cloud import bp as cloud from app.portal.automation import bp as automation from app.portal.bridgeconf import bp as bridgeconf @@ -35,6 +36,7 @@ portal.register_blueprint(automation, url_prefix="/automation") portal.register_blueprint(bridgeconf, url_prefix="/bridgeconf") portal.register_blueprint(bridge, url_prefix="/bridge") portal.register_blueprint(cloud, url_prefix="/cloud") +portal.register_blueprint(country, url_prefix="/country") portal.register_blueprint(eotk, url_prefix="/eotk") portal.register_blueprint(group, url_prefix="/group") portal.register_blueprint(list_, url_prefix="/list") diff --git a/app/portal/country.py b/app/portal/country.py new file mode 100644 index 0000000..1a5b700 --- /dev/null +++ b/app/portal/country.py @@ -0,0 +1,74 @@ +from datetime import datetime + +import sqlalchemy +from flask import Blueprint, render_template, Response, flash +from flask.typing import ResponseReturnValue +from flask_wtf import FlaskForm +from wtforms import IntegerField, BooleanField, SubmitField + +from app.extensions import db +from app.models.mirrors import Country + +bp = Blueprint("country", __name__) + +_SECTION_TEMPLATE_VARS = { + "section": "country", + "help_url": "https://bypass.censorship.guide/user/countries.html" +} + + +@bp.app_template_filter("country_flag") +def filter_country_flag(country_code: str) -> str: + country_code = country_code.upper() + + # Calculate the regional indicator symbol for each letter in the country code + base = ord('\U0001F1E6') - ord('A') + flag = ''.join([chr(ord(char) + base) for char in country_code]) + + return flag + + +@bp.route('/list') +def country_list() -> ResponseReturnValue: + countries = Country.query.filter(Country.destroyed.is_(None)).all() + print(len(countries)) + return render_template("list.html.j2", + title="Countries", + item="country", + new_link=None, + items=sorted(countries, key=lambda x: x.country_code), + **_SECTION_TEMPLATE_VARS + ) + + +class EditCountryForm(FlaskForm): # type: ignore[misc] + risk_level_override = BooleanField("Force Risk Level Override?") + risk_level_override_number = IntegerField("Forced Risk Level", description="Number from 0 to 20", default=0) + submit = SubmitField('Save Changes') + + +@bp.route('/edit/', methods=['GET', 'POST']) +def country_edit(country_id: int) -> ResponseReturnValue: + country = Country.query.filter(Country.id == country_id).first() + if country is None: + return Response(render_template("error.html.j2", + section="country", + header="404 Country Not Found", + message="The requested country could not be found."), + status=404) + form = EditCountryForm(risk_level_override=country.risk_level_override is not None, + risk_level_override_number=country.risk_level_override) + if form.validate_on_submit(): + if form.risk_level_override.data: + country.risk_level_override = form.risk_level_override_number.data + else: + country.risk_level_override = None + country.updated = datetime.utcnow() + try: + db.session.commit() + flash("Saved changes to country.", "success") + except sqlalchemy.exc.SQLAlchemyError: + flash("An error occurred saving the changes to the country.", "danger") + return render_template("country.html.j2", + section="country", + country=country, form=form) diff --git a/app/portal/origin.py b/app/portal/origin.py index ab48b9c..73457df 100644 --- a/app/portal/origin.py +++ b/app/portal/origin.py @@ -3,17 +3,18 @@ from datetime import datetime from typing import Optional, List import requests +import sqlalchemy from flask import flash, redirect, url_for, render_template, Response, Blueprint from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm from sqlalchemy import exc -from wtforms import StringField, SelectField, SubmitField, BooleanField +from wtforms import StringField, SelectField, SubmitField, BooleanField, IntegerField from wtforms.validators import DataRequired from app.extensions import db from app.models.base import Group -from app.models.mirrors import Origin -from app.portal.util import response_404, view_lifecycle +from app.models.mirrors import Origin, Country +from app.portal.util import response_404, view_lifecycle, LifecycleForm bp = Blueprint("origin", __name__) @@ -28,15 +29,22 @@ class NewOriginForm(FlaskForm): # type: ignore submit = SubmitField('Save Changes') -class EditOriginForm(FlaskForm): # type: ignore +class EditOriginForm(FlaskForm): # type: ignore[misc] description = StringField('Description', validators=[DataRequired()]) group = SelectField('Group', validators=[DataRequired()]) auto_rotate = BooleanField("Enable auto-rotation?") smart_proxy = BooleanField("Requires smart proxy?") asset_domain = BooleanField("Used to host assets for other domains?", default=False) + risk_level_override = BooleanField("Force Risk Level Override?") + risk_level_override_number = IntegerField("Forced Risk Level", description="Number from 0 to 20", default=0) submit = SubmitField('Save Changes') +class CountrySelectForm(FlaskForm): # type: ignore[misc] + country = SelectField("Country", validators=[DataRequired()]) + submit = SubmitField('Save Changes', render_kw={"class": "btn btn-success"}) + + def final_domain_name(domain_name: str) -> str: session = requests.Session() r = session.get(f"https://{domain_name}/", allow_redirects=True, timeout=10) @@ -84,7 +92,9 @@ def origin_edit(origin_id: int) -> ResponseReturnValue: description=origin.description, auto_rotate=origin.auto_rotation, smart_proxy=origin.smart, - asset_domain=origin.assets) + asset_domain=origin.assets, + risk_level_override=origin.risk_level_override is not None, + risk_level_override_number=origin.risk_level_override) form.group.choices = [(x.id, x.group_name) for x in Group.query.all()] if form.validate_on_submit(): origin.group_id = form.group.data @@ -92,12 +102,16 @@ def origin_edit(origin_id: int) -> ResponseReturnValue: origin.auto_rotation = form.auto_rotate.data origin.smart = form.smart_proxy.data origin.assets = form.asset_domain.data + if form.risk_level_override.data: + origin.risk_level_override = form.risk_level_override_number.data + else: + origin.risk_level_override = None origin.updated = datetime.utcnow() try: db.session.commit() - flash("Saved changes to group.", "success") + flash(f"Saved changes for origin {origin.domain_name}.", "success") except exc.SQLAlchemyError: - flash("An error occurred saving the changes to the group.", "danger") + flash("An error occurred saving the changes to the origin.", "danger") return render_template("origin.html.j2", section="origin", origin=origin, form=form) @@ -144,3 +158,74 @@ def origin_destroy(origin_id: int) -> ResponseReturnValue: resource=origin, action="destroy" ) + + +@bp.route('/country_remove//', methods=['GET', 'POST']) +def origin_country_remove(origin_id: int, country_id: int) -> ResponseReturnValue: + origin = Origin.query.filter(Origin.id == origin_id).first() + if origin is None: + return Response(render_template("error.html.j2", + section="origin", + header="404 Pool Not Found", + message="The requested origin could not be found."), + status=404) + country = Country.query.filter(Country.id == country_id).first() + if country is None: + return Response(render_template("error.html.j2", + section="origin", + header="404 Country Not Found", + message="The requested country could not be found."), + status=404) + if country not in origin.countries: + return Response(render_template("error.html.j2", + section="origin", + header="404 Country Not In Pool", + message="The requested country could not be found in the specified origin."), + status=404) + form = LifecycleForm() + if form.validate_on_submit(): + origin.countries.remove(country) + try: + db.session.commit() + flash("Saved changes to origin.", "success") + return redirect(url_for("portal.origin.origin_edit", origin_id=origin.id)) + except sqlalchemy.exc.SQLAlchemyError: + flash("An error occurred saving the changes to the origin.", "danger") + return render_template("lifecycle.html.j2", + header=f"Remove {country.country_name} from the {origin.domain_name} origin?", + message="Stop monitoring in this country.", + section="origin", + origin=origin, form=form) + + +@bp.route('/country_add/', methods=['GET', 'POST']) +def origin_country_add(origin_id: int) -> ResponseReturnValue: + origin = Origin.query.filter(Origin.id == origin_id).first() + if origin is None: + return Response(render_template("error.html.j2", + section="origin", + header="404 Origin Not Found", + message="The requested origin could not be found."), + status=404) + form = CountrySelectForm() + form.country.choices = [(x.id, f"{x.country_code} - {x.description}") for x in Country.query.all()] + if form.validate_on_submit(): + country = Country.query.filter(Country.id == form.country.data).first() + if country is None: + return Response(render_template("error.html.j2", + section="origin", + header="404 Country Not Found", + message="The requested country could not be found."), + status=404) + origin.countries.append(country) + try: + db.session.commit() + flash("Saved changes to origin.", "success") + return redirect(url_for("portal.origin.origin_edit", origin_id=origin.id)) + except sqlalchemy.exc.SQLAlchemyError: + flash("An error occurred saving the changes to the origin.", "danger") + return render_template("lifecycle.html.j2", + header=f"Add a country to {origin.domain_name}", + message="Enable monitoring from this country:", + section="origin", + origin=origin, form=form) diff --git a/app/portal/templates/base.html.j2 b/app/portal/templates/base.html.j2 index 3625ee0..d351f46 100644 --- a/app/portal/templates/base.html.j2 +++ b/app/portal/templates/base.html.j2 @@ -87,6 +87,12 @@ {{ icon("cloud") }} Cloud Accounts +
  • + + {{ icon("geo") }} Countries + +
  • diff --git a/app/portal/templates/country.html.j2 b/app/portal/templates/country.html.j2 new file mode 100644 index 0000000..a2c28c5 --- /dev/null +++ b/app/portal/templates/country.html.j2 @@ -0,0 +1,17 @@ +{% extends "base.html.j2" %} +{% from 'bootstrap5/form.html' import render_form %} +{% from "tables.html.j2" import origins_table %} + +{% block content %} +

    Country

    +

    {{ country.description }} {{ country.country_code | country_flag }}

    + +
    + {{ render_form(form) }} +
    + +

    Origins

    + {% if country.origins %} + {{ origins_table(country.origins) }} + {% endif %} +{% endblock %} diff --git a/app/portal/templates/icons.html.j2 b/app/portal/templates/icons.html.j2 index f11a356..e155a5a 100644 --- a/app/portal/templates/icons.html.j2 +++ b/app/portal/templates/icons.html.j2 @@ -69,6 +69,10 @@ viewBox="0 0 16 16"> + {% elif i == "geo" %} + + + {% elif i == "life-preserver" %} diff --git a/app/portal/templates/list.html.j2 b/app/portal/templates/list.html.j2 index 745722a..68bf882 100644 --- a/app/portal/templates/list.html.j2 +++ b/app/portal/templates/list.html.j2 @@ -1,5 +1,6 @@ {% extends "base.html.j2" %} {% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, cloud_accounts_table, + countries_table, eotk_table, groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, pools_table, proxies_table, static_table, webhook_table %} @@ -21,6 +22,8 @@ {{ bridges_table(items) }} {% elif item == "cloud account" %} {{ cloud_accounts_table(items) }} + {% elif item == "country" %} + {{ countries_table(items) }} {% elif item == "eotk" %} {{ eotk_table(items) }} {% elif item == "group" %} diff --git a/app/portal/templates/origin.html.j2 b/app/portal/templates/origin.html.j2 index 6891bfb..09c062e 100644 --- a/app/portal/templates/origin.html.j2 +++ b/app/portal/templates/origin.html.j2 @@ -1,6 +1,6 @@ {% extends "base.html.j2" %} {% from 'bootstrap5/form.html' import render_form %} -{% from "tables.html.j2" import alarms_table, bridges_table, proxies_table %} +{% from "tables.html.j2" import alarms_table, bridges_table, countries_table, proxies_table %} {% block content %}

    Origins

    @@ -14,6 +14,10 @@ {{ render_form(form) }} +

    Countries

    +

    Add Country

    + {{ countries_table(origin.countries, origin) }} + {% if origin.alarms %}

    Alarms

    {{ alarms_table(origin.alarms) }} diff --git a/app/portal/templates/tables.html.j2 b/app/portal/templates/tables.html.j2 index 6d2238a..ddf3d46 100644 --- a/app/portal/templates/tables.html.j2 +++ b/app/portal/templates/tables.html.j2 @@ -51,6 +51,39 @@ {% endmacro %} +{% macro countries_table(countries, origin=None) %} +
    + + + + + + + + + + + + {% for country in countries %} + + + + + + + + {% endfor %} + +
    Country CodeCountryRisk LevelOriginsActions
    {{ country.country_code }}{{ country.description }} {{ country.country_code | country_flag }}{{ country.risk_level | string }}{{ country.origins | length }} + View/Edit + {% if origin %} + Remove + {% endif %} +
    +
    +{% endmacro %} + + {% macro instances_table(application, instances) %}
    @@ -110,6 +143,8 @@ {% endmacro %} + + {% macro eotk_table(instances) %} {{ instances_table("eotk", instances) }} {% endmacro %} @@ -252,6 +287,7 @@ + @@ -270,6 +306,7 @@ {{ origin.domain_name }} + diff --git a/app/terraform/block_external.py b/app/terraform/block_external.py index 4b19b64..10a2bf2 100644 --- a/app/terraform/block_external.py +++ b/app/terraform/block_external.py @@ -1,4 +1,5 @@ import logging +from collections import defaultdict import requests @@ -35,17 +36,26 @@ class BlockExternalAutomation(BlockMirrorAutomation): def fetch(self) -> None: user_agent = {'User-agent': 'BypassCensorship/1.0'} - if isinstance(app.config.get('EXTERNAL_CHECK_URL', []), list): - check_urls = app.config.get('EXTERNAL_CHECK_URL', []) - elif isinstance(app.config.get('EXTERNAL_CHECK_URL'), str): - check_urls = [app.config['EXTERNAL_CHECK_URL']] + check_urls_config = app.config.get('EXTERNAL_CHECK_URL', []) + + if isinstance(check_urls_config, dict): + # Config is already a dictionary, use as is. + check_urls = check_urls_config + elif isinstance(check_urls_config, list): + # Convert list of strings to a dictionary with "external_N" keys. + check_urls = {f"external_{i}": url for i, url in enumerate(check_urls_config)} + elif isinstance(check_urls_config, str): + # Single string, convert to a dictionary with key "external". + check_urls = {"external": check_urls_config} else: - check_urls = [] - for check_url in check_urls: + # Fallback if the config item is neither dict, list, nor string. + check_urls = {} + for source, check_url in check_urls.items(): if self._data is None: - self._data = [] - self._data.extend(requests.get(check_url, headers=user_agent, timeout=30).json()) + self._data = defaultdict(list) + self._data[source].extend(requests.get(check_url, headers=user_agent, timeout=30).json()) def parse(self) -> None: - self.patterns.extend(["https://" + trim_http_https(pattern) for pattern in self._data]) + for source, patterns in self._data.items(): + self.patterns[source].extend(["https://" + trim_http_https(pattern) for pattern in patterns]) logging.debug("Found URLs: %s", self.patterns) diff --git a/app/terraform/block_mirror.py b/app/terraform/block_mirror.py index 58668e7..2e8158b 100644 --- a/app/terraform/block_mirror.py +++ b/app/terraform/block_mirror.py @@ -1,8 +1,9 @@ +from collections import defaultdict from datetime import datetime, timedelta import logging from abc import abstractmethod import fnmatch -from typing import Tuple, List, Any, Optional +from typing import Tuple, List, Any, Optional, Dict from app.extensions import db from app.models.activity import Activity @@ -11,14 +12,14 @@ from app.terraform import BaseAutomation class BlockMirrorAutomation(BaseAutomation): - patterns: List[str] + patterns: Dict[str, List[str]] _data: Any def __init__(self, *args: Any, **kwargs: Any) -> None: """ Constructor method. """ - self.patterns = [] + self.patterns = defaultdict(list) self._data = None super().__init__(*args, **kwargs) @@ -29,23 +30,25 @@ class BlockMirrorAutomation(BaseAutomation): logging.debug("Parse complete") rotated = [] proxy_urls = list(filter(lambda u: u is not None, active_proxy_urls())) - for pattern in self.patterns: - blocked_urls = fnmatch.filter(proxy_urls, pattern) - for blocked_url in blocked_urls: - if not (proxy := proxy_by_url(blocked_url)): - continue - logging.debug("Found %s blocked", proxy.url) - if not proxy.origin.auto_rotation: - logging.debug("Proxy auto-rotation forbidden for origin") - continue - if proxy.added > datetime.utcnow() - timedelta(hours=3): - logging.debug("Not rotating a proxy less than 3 hours old") - continue - if proxy.deprecate(reason=self.short_name): - logging.info("Rotated %s", proxy.url) - rotated.append((proxy.url, proxy.origin.domain_name)) - else: - logging.debug("Not rotating a proxy that is already deprecated") + for source, patterns in self.patterns.items(): + logging.debug("Processing blocked URLs from %s", source) + for pattern in patterns: + blocked_urls = fnmatch.filter(proxy_urls, pattern) + for blocked_url in blocked_urls: + if not (proxy := proxy_by_url(blocked_url)): + continue + logging.debug("Found %s blocked", proxy.url) + if not proxy.origin.auto_rotation: + logging.debug("Proxy auto-rotation forbidden for origin") + continue + if proxy.added > datetime.utcnow() - timedelta(hours=3): + logging.debug("Not rotating a proxy less than 3 hours old") + continue + if proxy.deprecate(reason=f"block_{source}"): + logging.info("Rotated %s", proxy.url) + rotated.append((proxy.url, proxy.origin.domain_name)) + else: + logging.debug("Not rotating a proxy that is already deprecated") if rotated: activity = Activity( activity_type="block", diff --git a/app/terraform/block_roskomsvoboda.py b/app/terraform/block_roskomsvoboda.py index 0347ae8..5ef41aa 100644 --- a/app/terraform/block_roskomsvoboda.py +++ b/app/terraform/block_roskomsvoboda.py @@ -93,7 +93,7 @@ class BlockRoskomsvobodaAutomation(BlockMirrorAutomation): for _event, element in lxml.etree.iterparse(BytesIO(self._data), resolve_entities=False): if element.tag == "domain": - self.patterns.append("https://" + element.text.strip()) + self.patterns["roskomsvoboda"].append("https://" + element.text.strip()) except XMLSyntaxError: activity = Activity( activity_type="automation", diff --git a/migrations/countries.json b/migrations/countries.json new file mode 100644 index 0000000..fe16d23 --- /dev/null +++ b/migrations/countries.json @@ -0,0 +1 @@ +[{"Code": "AF", "Name": "Afghanistan"},{"Code": "AX", "Name": "\u00c5land Islands"},{"Code": "AL", "Name": "Albania"},{"Code": "DZ", "Name": "Algeria"},{"Code": "AS", "Name": "American Samoa"},{"Code": "AD", "Name": "Andorra"},{"Code": "AO", "Name": "Angola"},{"Code": "AI", "Name": "Anguilla"},{"Code": "AQ", "Name": "Antarctica"},{"Code": "AG", "Name": "Antigua and Barbuda"},{"Code": "AR", "Name": "Argentina"},{"Code": "AM", "Name": "Armenia"},{"Code": "AW", "Name": "Aruba"},{"Code": "AU", "Name": "Australia"},{"Code": "AT", "Name": "Austria"},{"Code": "AZ", "Name": "Azerbaijan"},{"Code": "BS", "Name": "Bahamas"},{"Code": "BH", "Name": "Bahrain"},{"Code": "BD", "Name": "Bangladesh"},{"Code": "BB", "Name": "Barbados"},{"Code": "BY", "Name": "Belarus"},{"Code": "BE", "Name": "Belgium"},{"Code": "BZ", "Name": "Belize"},{"Code": "BJ", "Name": "Benin"},{"Code": "BM", "Name": "Bermuda"},{"Code": "BT", "Name": "Bhutan"},{"Code": "BO", "Name": "Bolivia, Plurinational State of"},{"Code": "BQ", "Name": "Bonaire, Sint Eustatius and Saba"},{"Code": "BA", "Name": "Bosnia and Herzegovina"},{"Code": "BW", "Name": "Botswana"},{"Code": "BV", "Name": "Bouvet Island"},{"Code": "BR", "Name": "Brazil"},{"Code": "IO", "Name": "British Indian Ocean Territory"},{"Code": "BN", "Name": "Brunei Darussalam"},{"Code": "BG", "Name": "Bulgaria"},{"Code": "BF", "Name": "Burkina Faso"},{"Code": "BI", "Name": "Burundi"},{"Code": "KH", "Name": "Cambodia"},{"Code": "CM", "Name": "Cameroon"},{"Code": "CA", "Name": "Canada"},{"Code": "CV", "Name": "Cape Verde"},{"Code": "KY", "Name": "Cayman Islands"},{"Code": "CF", "Name": "Central African Republic"},{"Code": "TD", "Name": "Chad"},{"Code": "CL", "Name": "Chile"},{"Code": "CN", "Name": "China"},{"Code": "CX", "Name": "Christmas Island"},{"Code": "CC", "Name": "Cocos (Keeling) Islands"},{"Code": "CO", "Name": "Colombia"},{"Code": "KM", "Name": "Comoros"},{"Code": "CG", "Name": "Congo"},{"Code": "CD", "Name": "Congo, the Democratic Republic of the"},{"Code": "CK", "Name": "Cook Islands"},{"Code": "CR", "Name": "Costa Rica"},{"Code": "CI", "Name": "C\u00f4te d'Ivoire"},{"Code": "HR", "Name": "Croatia"},{"Code": "CU", "Name": "Cuba"},{"Code": "CW", "Name": "Cura\u00e7ao"},{"Code": "CY", "Name": "Cyprus"},{"Code": "CZ", "Name": "Czech Republic"},{"Code": "DK", "Name": "Denmark"},{"Code": "DJ", "Name": "Djibouti"},{"Code": "DM", "Name": "Dominica"},{"Code": "DO", "Name": "Dominican Republic"},{"Code": "EC", "Name": "Ecuador"},{"Code": "EG", "Name": "Egypt"},{"Code": "SV", "Name": "El Salvador"},{"Code": "GQ", "Name": "Equatorial Guinea"},{"Code": "ER", "Name": "Eritrea"},{"Code": "EE", "Name": "Estonia"},{"Code": "ET", "Name": "Ethiopia"},{"Code": "FK", "Name": "Falkland Islands (Malvinas)"},{"Code": "FO", "Name": "Faroe Islands"},{"Code": "FJ", "Name": "Fiji"},{"Code": "FI", "Name": "Finland"},{"Code": "FR", "Name": "France"},{"Code": "GF", "Name": "French Guiana"},{"Code": "PF", "Name": "French Polynesia"},{"Code": "TF", "Name": "French Southern Territories"},{"Code": "GA", "Name": "Gabon"},{"Code": "GM", "Name": "Gambia"},{"Code": "GE", "Name": "Georgia"},{"Code": "DE", "Name": "Germany"},{"Code": "GH", "Name": "Ghana"},{"Code": "GI", "Name": "Gibraltar"},{"Code": "GR", "Name": "Greece"},{"Code": "GL", "Name": "Greenland"},{"Code": "GD", "Name": "Grenada"},{"Code": "GP", "Name": "Guadeloupe"},{"Code": "GU", "Name": "Guam"},{"Code": "GT", "Name": "Guatemala"},{"Code": "GG", "Name": "Guernsey"},{"Code": "GN", "Name": "Guinea"},{"Code": "GW", "Name": "Guinea-Bissau"},{"Code": "GY", "Name": "Guyana"},{"Code": "HT", "Name": "Haiti"},{"Code": "HM", "Name": "Heard Island and McDonald Islands"},{"Code": "VA", "Name": "Holy See (Vatican City State)"},{"Code": "HN", "Name": "Honduras"},{"Code": "HK", "Name": "Hong Kong"},{"Code": "HU", "Name": "Hungary"},{"Code": "IS", "Name": "Iceland"},{"Code": "IN", "Name": "India"},{"Code": "ID", "Name": "Indonesia"},{"Code": "IR", "Name": "Iran, Islamic Republic of"},{"Code": "IQ", "Name": "Iraq"},{"Code": "IE", "Name": "Ireland"},{"Code": "IM", "Name": "Isle of Man"},{"Code": "IL", "Name": "Israel"},{"Code": "IT", "Name": "Italy"},{"Code": "JM", "Name": "Jamaica"},{"Code": "JP", "Name": "Japan"},{"Code": "JE", "Name": "Jersey"},{"Code": "JO", "Name": "Jordan"},{"Code": "KZ", "Name": "Kazakhstan"},{"Code": "KE", "Name": "Kenya"},{"Code": "KI", "Name": "Kiribati"},{"Code": "KP", "Name": "Korea, Democratic People's Republic of"},{"Code": "KR", "Name": "Korea, Republic of"},{"Code": "KW", "Name": "Kuwait"},{"Code": "KG", "Name": "Kyrgyzstan"},{"Code": "LA", "Name": "Lao People's Democratic Republic"},{"Code": "LV", "Name": "Latvia"},{"Code": "LB", "Name": "Lebanon"},{"Code": "LS", "Name": "Lesotho"},{"Code": "LR", "Name": "Liberia"},{"Code": "LY", "Name": "Libya"},{"Code": "LI", "Name": "Liechtenstein"},{"Code": "LT", "Name": "Lithuania"},{"Code": "LU", "Name": "Luxembourg"},{"Code": "MO", "Name": "Macao"},{"Code": "MK", "Name": "Macedonia, the Former Yugoslav Republic of"},{"Code": "MG", "Name": "Madagascar"},{"Code": "MW", "Name": "Malawi"},{"Code": "MY", "Name": "Malaysia"},{"Code": "MV", "Name": "Maldives"},{"Code": "ML", "Name": "Mali"},{"Code": "MT", "Name": "Malta"},{"Code": "MH", "Name": "Marshall Islands"},{"Code": "MQ", "Name": "Martinique"},{"Code": "MR", "Name": "Mauritania"},{"Code": "MU", "Name": "Mauritius"},{"Code": "YT", "Name": "Mayotte"},{"Code": "MX", "Name": "Mexico"},{"Code": "FM", "Name": "Micronesia, Federated States of"},{"Code": "MD", "Name": "Moldova, Republic of"},{"Code": "MC", "Name": "Monaco"},{"Code": "MN", "Name": "Mongolia"},{"Code": "ME", "Name": "Montenegro"},{"Code": "MS", "Name": "Montserrat"},{"Code": "MA", "Name": "Morocco"},{"Code": "MZ", "Name": "Mozambique"},{"Code": "MM", "Name": "Myanmar"},{"Code": "NA", "Name": "Namibia"},{"Code": "NR", "Name": "Nauru"},{"Code": "NP", "Name": "Nepal"},{"Code": "NL", "Name": "Netherlands"},{"Code": "NC", "Name": "New Caledonia"},{"Code": "NZ", "Name": "New Zealand"},{"Code": "NI", "Name": "Nicaragua"},{"Code": "NE", "Name": "Niger"},{"Code": "NG", "Name": "Nigeria"},{"Code": "NU", "Name": "Niue"},{"Code": "NF", "Name": "Norfolk Island"},{"Code": "MP", "Name": "Northern Mariana Islands"},{"Code": "NO", "Name": "Norway"},{"Code": "OM", "Name": "Oman"},{"Code": "PK", "Name": "Pakistan"},{"Code": "PW", "Name": "Palau"},{"Code": "PS", "Name": "Palestine, State of"},{"Code": "PA", "Name": "Panama"},{"Code": "PG", "Name": "Papua New Guinea"},{"Code": "PY", "Name": "Paraguay"},{"Code": "PE", "Name": "Peru"},{"Code": "PH", "Name": "Philippines"},{"Code": "PN", "Name": "Pitcairn"},{"Code": "PL", "Name": "Poland"},{"Code": "PT", "Name": "Portugal"},{"Code": "PR", "Name": "Puerto Rico"},{"Code": "QA", "Name": "Qatar"},{"Code": "RE", "Name": "R\u00e9union"},{"Code": "RO", "Name": "Romania"},{"Code": "RU", "Name": "Russian Federation"},{"Code": "RW", "Name": "Rwanda"},{"Code": "BL", "Name": "Saint Barth\u00e9lemy"},{"Code": "SH", "Name": "Saint Helena, Ascension and Tristan da Cunha"},{"Code": "KN", "Name": "Saint Kitts and Nevis"},{"Code": "LC", "Name": "Saint Lucia"},{"Code": "MF", "Name": "Saint Martin (French part)"},{"Code": "PM", "Name": "Saint Pierre and Miquelon"},{"Code": "VC", "Name": "Saint Vincent and the Grenadines"},{"Code": "WS", "Name": "Samoa"},{"Code": "SM", "Name": "San Marino"},{"Code": "ST", "Name": "Sao Tome and Principe"},{"Code": "SA", "Name": "Saudi Arabia"},{"Code": "SN", "Name": "Senegal"},{"Code": "RS", "Name": "Serbia"},{"Code": "SC", "Name": "Seychelles"},{"Code": "SL", "Name": "Sierra Leone"},{"Code": "SG", "Name": "Singapore"},{"Code": "SX", "Name": "Sint Maarten (Dutch part)"},{"Code": "SK", "Name": "Slovakia"},{"Code": "SI", "Name": "Slovenia"},{"Code": "SB", "Name": "Solomon Islands"},{"Code": "SO", "Name": "Somalia"},{"Code": "ZA", "Name": "South Africa"},{"Code": "GS", "Name": "South Georgia and the South Sandwich Islands"},{"Code": "SS", "Name": "South Sudan"},{"Code": "ES", "Name": "Spain"},{"Code": "LK", "Name": "Sri Lanka"},{"Code": "SD", "Name": "Sudan"},{"Code": "SR", "Name": "Suriname"},{"Code": "SJ", "Name": "Svalbard and Jan Mayen"},{"Code": "SZ", "Name": "Swaziland"},{"Code": "SE", "Name": "Sweden"},{"Code": "CH", "Name": "Switzerland"},{"Code": "SY", "Name": "Syrian Arab Republic"},{"Code": "TW", "Name": "Taiwan, Province of China"},{"Code": "TJ", "Name": "Tajikistan"},{"Code": "TZ", "Name": "Tanzania, United Republic of"},{"Code": "TH", "Name": "Thailand"},{"Code": "TL", "Name": "Timor-Leste"},{"Code": "TG", "Name": "Togo"},{"Code": "TK", "Name": "Tokelau"},{"Code": "TO", "Name": "Tonga"},{"Code": "TT", "Name": "Trinidad and Tobago"},{"Code": "TN", "Name": "Tunisia"},{"Code": "TR", "Name": "Turkey"},{"Code": "TM", "Name": "Turkmenistan"},{"Code": "TC", "Name": "Turks and Caicos Islands"},{"Code": "TV", "Name": "Tuvalu"},{"Code": "UG", "Name": "Uganda"},{"Code": "UA", "Name": "Ukraine"},{"Code": "AE", "Name": "United Arab Emirates"},{"Code": "GB", "Name": "United Kingdom"},{"Code": "US", "Name": "United States"},{"Code": "UM", "Name": "United States Minor Outlying Islands"},{"Code": "UY", "Name": "Uruguay"},{"Code": "UZ", "Name": "Uzbekistan"},{"Code": "VU", "Name": "Vanuatu"},{"Code": "VE", "Name": "Venezuela, Bolivarian Republic of"},{"Code": "VN", "Name": "Viet Nam"},{"Code": "VG", "Name": "Virgin Islands, British"},{"Code": "VI", "Name": "Virgin Islands, U.S."},{"Code": "WF", "Name": "Wallis and Futuna"},{"Code": "EH", "Name": "Western Sahara"},{"Code": "YE", "Name": "Yemen"},{"Code": "ZM", "Name": "Zambia"},{"Code": "ZW", "Name": "Zimbabwe"}] \ No newline at end of file diff --git a/migrations/versions/bbec86de37c4_adds_geo_monitoring.py b/migrations/versions/bbec86de37c4_adds_geo_monitoring.py new file mode 100644 index 0000000..2dc031e --- /dev/null +++ b/migrations/versions/bbec86de37c4_adds_geo_monitoring.py @@ -0,0 +1,117 @@ +import json +from datetime import datetime + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import table, column +from sqlalchemy.orm import Session + +revision = 'bbec86de37c4' +down_revision = '278bcfb487d3' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('country', + sa.Column('country_code', sa.String(length=2), nullable=False), + sa.Column('risk_level_override', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.String(length=255), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('destroyed', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_country')) + ) + op.create_table('deprecation', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=50), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('deprecated_at', sa.DateTime(), nullable=False), + sa.Column('reason', sa.String(), nullable=False), + sa.Column('meta', sa.JSON()), + sa.PrimaryKeyConstraint('id', name=op.f('pk_deprecation')) + ) + op.create_table('country_origin', + sa.Column('country_id', sa.Integer(), nullable=False), + sa.Column('origin_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['country_id'], ['country.id'], + name=op.f('fk_country_origin_country_id_country')), + sa.ForeignKeyConstraint(['origin_id'], ['origin.id'], + name=op.f('fk_country_origin_origin_id_origin')), + sa.PrimaryKeyConstraint('country_id', 'origin_id', name=op.f('pk_country_origin')) + ) + with op.batch_alter_table('origin', schema=None) as batch_op: + batch_op.add_column(sa.Column('risk_level_override', sa.Integer(), nullable=True)) + + countries = json.load(open("migrations/countries.json")) + + country_table = table( + 'country', + column('id', sa.Integer), + column('country_code', sa.String), + column('description', sa.String), + column('risk_level_override', sa.Integer), + column('added', sa.DateTime), + column('updated', sa.DateTime), + ) + + # Iterate through each country and insert it using the 'country_table' definition + for country in countries: + op.execute( + country_table.insert().values( + country_code=country['Code'], + description=country['Name'], + risk_level_override=None, # Assuming risk level override is initially None + added=datetime.utcnow(), + updated=datetime.utcnow() + ) + ) + + deprecation_table = table('deprecation', + column('id', sa.Integer), + column('resource_type', sa.String), + column('resource_id', sa.Integer), + column('deprecated_at', sa.DateTime), + column('reason', sa.String) + ) + + bind = op.get_bind() + session = Session(bind=bind) + + resource_tables = ['proxy', 'bridge'] + + for table_name in resource_tables: + # Query the existing deprecations + results = session.execute( + sa.select( + column('id'), + column('deprecated'), + column('deprecation_reason') + ).select_from(table(table_name)) + ) + + # Iterate over each row and create a corresponding entry in the Deprecation table + for id_, deprecated, reason in results: + if deprecated is not None: # Only migrate if there's a deprecation date + op.execute( + deprecation_table.insert().values( + resource_type=table_name.title(), # The class name is used, not the table name + resource_id=id_, + deprecated_at=deprecated, + reason=reason + ) + ) + + session.commit() + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('origin', schema=None) as batch_op: + batch_op.drop_column('risk_level_override') + + op.drop_table('country_origin') + op.drop_table('deprecation') + op.drop_table('country') + # ### end Alembic commands ###
    Name DescriptionRisk Level Auto-Rotation Smart Proxy Assets Origin {{ origin.description }}{{ origin.risk_level.values() | max | default("n/a") }} {% if origin.auto_rotation %}✅{% else %}❌{% endif %} {% if origin.smart %}✅{% else %}❌{% endif %} {% if origin.assets %}✅{% else %}❌{% endif %}