block/bridge: refactor into general bridge block subsystem

app/cli/automate.py

@@ -9,7 +9,7 @@ from app.extensions import db
 from app.models.activity import Activity
 from app.models.automation import Automation, AutomationState, AutomationLogs
 from app.terraform import BaseAutomation
-from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation
+from app.terraform.block.bridge_github import BlockBridgeGitHubAutomation
 from app.terraform.block_external import BlockExternalAutomation
 from app.terraform.block_ooni import BlockOONIAutomation
 from app.terraform.block_roskomsvoboda import BlockRoskomsvobodaAutomation

app/models/bridges.py

@@ -10,7 +10,6 @@ class BridgeConf(AbstractConfiguration):
     group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
     provider = db.Column(db.String(20), nullable=False)
     method = db.Column(db.String(20), nullable=False)
-    description = db.Column(db.String(255))
     number = db.Column(db.Integer())
 
     group = db.relationship("Group", back_populates="bridgeconfs")

app/terraform/block/bridge.py

@@ -0,0 +1,101 @@
+from datetime import datetime, timedelta
+import logging
+from abc import abstractmethod
+import fnmatch
+from typing import Tuple, List, Callable
+
+from app.extensions import db
+from app.models.activity import Activity
+from app.models.bridges import Bridge
+from app.models.mirrors import Proxy
+from app.terraform import BaseAutomation
+
+
+class BlockBridgeAutomation(BaseAutomation):
+    patterns: List[str]
+
+    def __init__(self) -> None:
+        """
+        Constructor method.
+        """
+        self.ips = []
+        self.fingerprints = []
+        self.hashed_fingerprints = []
+        super().__init__()
+
+    def perform_deprecations(self, ids: List[str], bridge_select_func: Callable[[str], Bridge]
+                             ) -> List[Tuple[str, str]]:
+        rotated = []
+        for id_ in ids:
+            bridge = bridge_select_func(id_)
+            logging.debug("Found %s blocked", bridge.fingerprint)
+            if bridge.added > datetime.utcnow() - timedelta(hours=3):
+                logging.debug("Not rotating a bridge less than 3 hours old")
+                continue
+            if bridge.deprecate(reason=self.short_name):
+                logging.info("Rotated %s", bridge.fingerprint)
+                rotated.append((bridge.fingerprint, bridge.conf.provider))
+            else:
+                logging.debug("Not rotating a bridge that is already deprecated")
+        return rotated
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
+        self.fetch()
+        logging.debug("Fetch complete")
+        self.parse()
+        logging.debug("Parse complete")
+        rotated = []
+        rotated.extend(self.perform_deprecations(self.ips, get_bridge_by_ip))
+        rotated.extend(self.perform_deprecations(self.fingerprints, get_bridge_by_fingerprint))
+        rotated.extend(self.perform_deprecations(self.fingerprints, get_bridge_by_hashed_fingerprint))
+        if rotated:
+            activity = Activity(
+                activity_type="block",
+                text=(f"[{self.short_name}] ♻ Rotated {len(rotated)} bridges: \n"
+                      + "\n".join([f"* {fingerprint} ({provider})" for fingerprint, provider in rotated]))
+            )
+            db.session.add(activity)
+            activity.notify()
+            db.session.commit()
+        return True, ""
+
+    @abstractmethod
+    def fetch(self) -> None:
+        """
+        Fetch the blocklist data. It is the responsibility of the automation task
+        to persist this within the object for the parse step.
+
+        :return: None
+        """
+
+    @abstractmethod
+    def parse(self) -> None:
+        """
+        Parse the blocklist data.
+
+        :return: None
+        """
+
+
+def get_bridge_by_ip(ip: str) -> Bridge:
+    return Bridge.query.filter(  # type: ignore[no-any-return]
+        Bridge.deprecated.is_(None),
+        Bridge.destroyed.is_(None),
+        Bridge.bridgeline.contains(f" {ip} ")
+    ).first()
+
+
+def get_bridge_by_fingerprint(fingerprint: str) -> Bridge:
+    return Bridge.query.filter(  # type: ignore[no-any-return]
+        Bridge.deprecated.is_(None),
+        Bridge.destroyed.is_(None),
+        Bridge.fingerprint == fingerprint
+    ).first()
+
+
+def get_bridge_by_hashed_fingerprint(hashed_fingerprint: str) -> Bridge:
+    return Bridge.query.filter(  # type: ignore[no-any-return]
+        Bridge.deprecated.is_(None),
+        Bridge.destroyed.is_(None),
+        Bridge.hashed_fingerprint == hashed_fingerprint
+    ).first()

app/terraform/block/bridge_github.py

@@ -0,0 +1,25 @@
+from flask import current_app
+from github import Github
+
+from app.terraform.block.bridge_reachability import BlockBridgeReachabilityAutomation
+
+
+class BlockBridgeGitHubAutomation(BlockBridgeReachabilityAutomation):
+    """
+    Automation task to import bridge reachability results from GitHub.
+    """
+
+    short_name = "block_bridge_github"
+    description = "Import bridge reachability results from GitHub"
+    frequency = 30
+
+    def fetch(self) -> None:
+        github = Github(current_app.config['GITHUB_API_KEY'])
+        repo = github.get_repo(current_app.config['GITHUB_BRIDGE_REPO'])
+        for vantage_point in current_app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
+            contents = repo.get_contents(f"recentResult_{vantage_point}")
+            if isinstance(contents, list):
+                raise RuntimeError(
+                        f"Expected a file at recentResult_{vantage_point}"
+                        " but got a directory.")
+            self._lines = contents.decoded_content.decode('utf-8').splitlines()

app/terraform/block/bridge_reachability.py

@@ -0,0 +1,21 @@
+import datetime
+from typing import List
+
+from dateutil.parser import isoparse
+
+from app.terraform.block.bridge import BlockBridgeAutomation
+
+
+class BlockBridgeReachabilityAutomation(BlockBridgeAutomation):
+
+    _lines: List[str]
+
+    def parse(self):
+        for line in self._lines:
+            parts = line.split("\t")
+            if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc)
+                                     - datetime.timedelta(days=3)):
+                # Skip results older than 3 days
+                continue
+            if int(parts[1]) < 40:
+                self.hashed_fingerprints.append(parts[0])

app/terraform/block_bridge_github.py

@@ -1,44 +0,0 @@
-import datetime
-from typing import Tuple
-
-from dateutil.parser import isoparse
-from github import Github
-
-from app import app
-from app.extensions import db
-from app.models.bridges import Bridge
-from app.terraform import BaseAutomation
-
-
-class BlockBridgeGitHubAutomation(BaseAutomation):
-    """
-    Automation task to import bridge reachability results from GitHub.
-    """
-
-    short_name = "block_bridge_github"
-    description = "Import bridge reachability results from GitHub"
-    frequency = 30
-
-    def automate(self, full: bool = False) -> Tuple[bool, str]:
-        github = Github(app.config['GITHUB_API_KEY'])
-        repo = github.get_repo(app.config['GITHUB_BRIDGE_REPO'])
-        for vantage_point in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
-            contents = repo.get_contents(f"recentResult_{vantage_point}")
-            if isinstance(contents, list):
-                return (False,
-                        f"Expected a file at recentResult_{vantage_point}"
-                        " but got a directory.")
-            results = contents.decoded_content.decode('utf-8').splitlines()
-            for result in results:
-                parts = result.split("\t")
-                if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc)
-                                         - datetime.timedelta(days=3)):
-                    continue
-                if int(parts[1]) < 40:
-                    bridge: Bridge = Bridge.query.filter(
-                        Bridge.hashed_fingerprint == parts[0]
-                    ).first()
-                    if bridge is not None:
-                        bridge.deprecate(reason="github")
-        db.session.commit()
-        return True, ""