From ca3fc844b7a262f3491c11dbe2ffe220f3a98de3 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Tue, 12 Jul 2022 11:09:23 +0100 Subject: [PATCH] block/bridge: refactor into general bridge block subsystem --- app/cli/automate.py | 2 +- app/models/bridges.py | 1 - app/terraform/block/__init__.py | 0 app/terraform/block/bridge.py | 101 +++++++++++++++++++++ app/terraform/block/bridge_github.py | 25 +++++ app/terraform/block/bridge_reachability.py | 21 +++++ app/terraform/block_bridge_github.py | 44 --------- 7 files changed, 148 insertions(+), 46 deletions(-) create mode 100644 app/terraform/block/__init__.py create mode 100644 app/terraform/block/bridge.py create mode 100644 app/terraform/block/bridge_github.py create mode 100644 app/terraform/block/bridge_reachability.py delete mode 100644 app/terraform/block_bridge_github.py diff --git a/app/cli/automate.py b/app/cli/automate.py index 41cb0ed..5364770 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -9,7 +9,7 @@ from app.extensions import db from app.models.activity import Activity from app.models.automation import Automation, AutomationState, AutomationLogs from app.terraform import BaseAutomation -from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation +from app.terraform.block.bridge_github import BlockBridgeGitHubAutomation from app.terraform.block_external import BlockExternalAutomation from app.terraform.block_ooni import BlockOONIAutomation from app.terraform.block_roskomsvoboda import BlockRoskomsvobodaAutomation diff --git a/app/models/bridges.py b/app/models/bridges.py index 105cce6..99717d6 100644 --- a/app/models/bridges.py +++ b/app/models/bridges.py @@ -10,7 +10,6 @@ class BridgeConf(AbstractConfiguration): group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False) provider = db.Column(db.String(20), nullable=False) method = db.Column(db.String(20), nullable=False) - description = db.Column(db.String(255)) number = db.Column(db.Integer()) group = db.relationship("Group", back_populates="bridgeconfs") diff --git a/app/terraform/block/__init__.py b/app/terraform/block/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/terraform/block/bridge.py b/app/terraform/block/bridge.py new file mode 100644 index 0000000..f47ac69 --- /dev/null +++ b/app/terraform/block/bridge.py @@ -0,0 +1,101 @@ +from datetime import datetime, timedelta +import logging +from abc import abstractmethod +import fnmatch +from typing import Tuple, List, Callable + +from app.extensions import db +from app.models.activity import Activity +from app.models.bridges import Bridge +from app.models.mirrors import Proxy +from app.terraform import BaseAutomation + + +class BlockBridgeAutomation(BaseAutomation): + patterns: List[str] + + def __init__(self) -> None: + """ + Constructor method. + """ + self.ips = [] + self.fingerprints = [] + self.hashed_fingerprints = [] + super().__init__() + + def perform_deprecations(self, ids: List[str], bridge_select_func: Callable[[str], Bridge] + ) -> List[Tuple[str, str]]: + rotated = [] + for id_ in ids: + bridge = bridge_select_func(id_) + logging.debug("Found %s blocked", bridge.fingerprint) + if bridge.added > datetime.utcnow() - timedelta(hours=3): + logging.debug("Not rotating a bridge less than 3 hours old") + continue + if bridge.deprecate(reason=self.short_name): + logging.info("Rotated %s", bridge.fingerprint) + rotated.append((bridge.fingerprint, bridge.conf.provider)) + else: + logging.debug("Not rotating a bridge that is already deprecated") + return rotated + + def automate(self, full: bool = False) -> Tuple[bool, str]: + self.fetch() + logging.debug("Fetch complete") + self.parse() + logging.debug("Parse complete") + rotated = [] + rotated.extend(self.perform_deprecations(self.ips, get_bridge_by_ip)) + rotated.extend(self.perform_deprecations(self.fingerprints, get_bridge_by_fingerprint)) + rotated.extend(self.perform_deprecations(self.fingerprints, get_bridge_by_hashed_fingerprint)) + if rotated: + activity = Activity( + activity_type="block", + text=(f"[{self.short_name}] ♻ Rotated {len(rotated)} bridges: \n" + + "\n".join([f"* {fingerprint} ({provider})" for fingerprint, provider in rotated])) + ) + db.session.add(activity) + activity.notify() + db.session.commit() + return True, "" + + @abstractmethod + def fetch(self) -> None: + """ + Fetch the blocklist data. It is the responsibility of the automation task + to persist this within the object for the parse step. + + :return: None + """ + + @abstractmethod + def parse(self) -> None: + """ + Parse the blocklist data. + + :return: None + """ + + +def get_bridge_by_ip(ip: str) -> Bridge: + return Bridge.query.filter( # type: ignore[no-any-return] + Bridge.deprecated.is_(None), + Bridge.destroyed.is_(None), + Bridge.bridgeline.contains(f" {ip} ") + ).first() + + +def get_bridge_by_fingerprint(fingerprint: str) -> Bridge: + return Bridge.query.filter( # type: ignore[no-any-return] + Bridge.deprecated.is_(None), + Bridge.destroyed.is_(None), + Bridge.fingerprint == fingerprint + ).first() + + +def get_bridge_by_hashed_fingerprint(hashed_fingerprint: str) -> Bridge: + return Bridge.query.filter( # type: ignore[no-any-return] + Bridge.deprecated.is_(None), + Bridge.destroyed.is_(None), + Bridge.hashed_fingerprint == hashed_fingerprint + ).first() diff --git a/app/terraform/block/bridge_github.py b/app/terraform/block/bridge_github.py new file mode 100644 index 0000000..d79da4f --- /dev/null +++ b/app/terraform/block/bridge_github.py @@ -0,0 +1,25 @@ +from flask import current_app +from github import Github + +from app.terraform.block.bridge_reachability import BlockBridgeReachabilityAutomation + + +class BlockBridgeGitHubAutomation(BlockBridgeReachabilityAutomation): + """ + Automation task to import bridge reachability results from GitHub. + """ + + short_name = "block_bridge_github" + description = "Import bridge reachability results from GitHub" + frequency = 30 + + def fetch(self) -> None: + github = Github(current_app.config['GITHUB_API_KEY']) + repo = github.get_repo(current_app.config['GITHUB_BRIDGE_REPO']) + for vantage_point in current_app.config['GITHUB_BRIDGE_VANTAGE_POINTS']: + contents = repo.get_contents(f"recentResult_{vantage_point}") + if isinstance(contents, list): + raise RuntimeError( + f"Expected a file at recentResult_{vantage_point}" + " but got a directory.") + self._lines = contents.decoded_content.decode('utf-8').splitlines() diff --git a/app/terraform/block/bridge_reachability.py b/app/terraform/block/bridge_reachability.py new file mode 100644 index 0000000..a12b602 --- /dev/null +++ b/app/terraform/block/bridge_reachability.py @@ -0,0 +1,21 @@ +import datetime +from typing import List + +from dateutil.parser import isoparse + +from app.terraform.block.bridge import BlockBridgeAutomation + + +class BlockBridgeReachabilityAutomation(BlockBridgeAutomation): + + _lines: List[str] + + def parse(self): + for line in self._lines: + parts = line.split("\t") + if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(days=3)): + # Skip results older than 3 days + continue + if int(parts[1]) < 40: + self.hashed_fingerprints.append(parts[0]) diff --git a/app/terraform/block_bridge_github.py b/app/terraform/block_bridge_github.py deleted file mode 100644 index 2131514..0000000 --- a/app/terraform/block_bridge_github.py +++ /dev/null @@ -1,44 +0,0 @@ -import datetime -from typing import Tuple - -from dateutil.parser import isoparse -from github import Github - -from app import app -from app.extensions import db -from app.models.bridges import Bridge -from app.terraform import BaseAutomation - - -class BlockBridgeGitHubAutomation(BaseAutomation): - """ - Automation task to import bridge reachability results from GitHub. - """ - - short_name = "block_bridge_github" - description = "Import bridge reachability results from GitHub" - frequency = 30 - - def automate(self, full: bool = False) -> Tuple[bool, str]: - github = Github(app.config['GITHUB_API_KEY']) - repo = github.get_repo(app.config['GITHUB_BRIDGE_REPO']) - for vantage_point in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']: - contents = repo.get_contents(f"recentResult_{vantage_point}") - if isinstance(contents, list): - return (False, - f"Expected a file at recentResult_{vantage_point}" - " but got a directory.") - results = contents.decoded_content.decode('utf-8').splitlines() - for result in results: - parts = result.split("\t") - if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(days=3)): - continue - if int(parts[1]) < 40: - bridge: Bridge = Bridge.query.filter( - Bridge.hashed_fingerprint == parts[0] - ).first() - if bridge is not None: - bridge.deprecate(reason="github") - db.session.commit() - return True, ""