automation: herd blocking automations into framework

see #1

app/cli/automate.py

@@ -7,6 +7,9 @@ from app import app
 from app.extensions import db
 from app.models.automation import Automation, AutomationState, AutomationLogs
 from app.terraform import BaseAutomation
+from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation
+from app.terraform.block_external import BlockExternalAutomation
+from app.terraform.block_ooni import BlockOONIAutomation
 from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation
 from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation
 from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation
@@ -27,6 +30,9 @@ jobs = {
         AlarmProxyAzureCdnAutomation,
         AlarmProxyCloudfrontAutomation,
         AlarmProxyHTTPStatusAutomation,
+        BlockBridgeGitHubAutomation,
+        BlockExternalAutomation,
+        BlockOONIAutomation,
         BridgeAWSAutomation,
         BridgeGandiAutomation,
         BridgeHcloudAutomation,

app/terraform/block_bridge_github.py

@@ -1,4 +1,5 @@
 import datetime
+from typing import Tuple
 
 from dateutil.parser import isoparse
 from github import Github
@@ -6,26 +7,27 @@ from github import Github
 from app import app
 from app.extensions import db
 from app.models.bridges import Bridge
+from app.terraform import BaseAutomation
 
 
-def check_blocks():
+class BlockBridgeGitHubAutomation(BaseAutomation):
-    g = Github(app.config['GITHUB_API_KEY'])
+    short_name = "block_bridge_github"
-    repo = g.get_repo(app.config['GITHUB_BRIDGE_REPO'])
+    description = "Import bridge reachability results from GitHub"
-    for vp in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
-        results = repo.get_contents(f"recentResult_{vp}").decoded_content.decode('utf-8').splitlines()
-        for result in results:
-            parts = result.split("\t")
-            if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)):
-                continue
-            if int(parts[1]) < 40:
-                bridge: Bridge = Bridge.query.filter(
-                    Bridge.hashed_fingerprint == parts[0]
-                ).first()
-                if bridge is not None:
-                    bridge.deprecate(reason="github")
-                    db.session.commit()
 
-
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
-if __name__ == "__main__":
+        g = Github(app.config['GITHUB_API_KEY'])
-    with app.app_context():
+        repo = g.get_repo(app.config['GITHUB_BRIDGE_REPO'])
-        check_blocks()
+        for vp in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
+            results = repo.get_contents(f"recentResult_{vp}").decoded_content.decode('utf-8').splitlines()
+            for result in results:
+                parts = result.split("\t")
+                if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)):
+                    continue
+                if int(parts[1]) < 40:
+                    bridge: Bridge = Bridge.query.filter(
+                        Bridge.hashed_fingerprint == parts[0]
+                    ).first()
+                    if bridge is not None:
+                        bridge.deprecate(reason="github")
+        db.session.commit()
+        return True, ""

app/terraform/block_external.py

@@ -1,57 +1,60 @@
+from typing import Tuple
+
 from bs4 import BeautifulSoup
 import requests
 
 from app import app
 from app.extensions import db
 from app.models.mirrors import Proxy
+from app.terraform import BaseAutomation
 
 
-def check_blocks():
+class BlockExternalAutomation(BaseAutomation):
-    user_agent = {'User-agent': 'BypassCensorship/1.0'}
+    short_name = "block_external"
-    page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent)
+    description = "Import proxy reachability results from external source"
-    soup = BeautifulSoup(page.content, 'html.parser')
-    h2 = soup.find_all('h2')
-    div = soup.find_all('div', class_="overflow-auto mb-5")
 
-    results = {}
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
+        user_agent = {'User-agent': 'BypassCensorship/1.0'}
+        page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent)
+        soup = BeautifulSoup(page.content, 'html.parser')
+        h2 = soup.find_all('h2')
+        div = soup.find_all('div', class_="overflow-auto mb-5")
 
-    i = 0
+        results = {}
-    while i < len(h2):
-        if not div[i].div:
-            urls = []
-            a = div[i].find_all('a')
-            j = 0
-            while j < len(a):
-                urls.append(a[j].text)
-                j += 1
-            results[h2[i].text] = urls
-        else:
-            results[h2[i].text] = []
-        i += 1
 
-    for vp in results:
+        i = 0
-        if vp not in app.config['EXTERNAL_VANTAGE_POINTS']:
+        while i < len(h2):
-            continue
+            if not div[i].div:
-        for url in results[vp]:
+                urls = []
-            print(f"Found {url} blocked")
+                a = div[i].find_all('a')
-            proxy = Proxy.query.filter(
+                j = 0
-                Proxy.provider == "cloudfront",
+                while j < len(a):
-                Proxy.url == f"https://{url}"
+                    urls.append(a[j].text)
-            ).first()
+                    j += 1
-            if not proxy:
+                results[h2[i].text] = urls
-                print("Proxy not found")
+            else:
+                results[h2[i].text] = []
+            i += 1
+
+        for vp in results:
+            if vp not in app.config['EXTERNAL_VANTAGE_POINTS']:
                 continue
-            if not proxy.origin.auto_rotation:
+            for url in results[vp]:
-                print("Proxy auto-rotation forbidden for origin")
+                print(f"Found {url} blocked")
-                continue
+                proxy = Proxy.query.filter(
-            if proxy.deprecated:
+                    Proxy.provider == "cloudfront",
-                print("Proxy already marked blocked")
+                    Proxy.url == f"https://{url}"
-                continue
+                ).first()
-            proxy.deprecate(reason="external")
+                if not proxy:
+                    print("Proxy not found")
+                    continue
+                if not proxy.origin.auto_rotation:
+                    print("Proxy auto-rotation forbidden for origin")
+                    continue
+                if proxy.deprecated:
+                    print("Proxy already marked blocked")
+                    continue
+                proxy.deprecate(reason="external")
+        db.session.commit()
 
-    db.session.commit()
+        return True, ""
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        check_blocks()

app/terraform/block_ooni.py

@@ -1,18 +1,18 @@
 from collections import defaultdict
 from datetime import datetime
 from datetime import timedelta
-from typing import Dict
+from typing import Dict, Tuple
 
 import requests
 
-from app import app
 from app.extensions import db
 from app.models.alarms import Alarm, AlarmState
 from app.models.mirrors import Origin
+from app.terraform import BaseAutomation
 
 
 def check_origin(domain_name: str):
-    start_date = (datetime.utcnow() - timedelta(days=2)).strftime("%Y-%m-%dT%H%%3A%M")
+    start_date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M")
     end_date = datetime.utcnow().strftime("%Y-%m-%dT%H%%3A%M")
     api_url = f"https://api.ooni.io/api/v1/measurements?domain={domain_name}&since={start_date}&until={end_date}"
     result = defaultdict(lambda: {"anomaly": 0, "confirmed": 0, "failure": 0, "ok": 0})
@@ -22,7 +22,7 @@ def check_origin(domain_name: str):
 def _check_origin(api_url: str, result: Dict):
     print(f"Processing {api_url}")
     req = requests.get(api_url).json()
-    if not req['results']:
+    if 'results' not in req or not req['results']:
         return result
     for r in req['results']:
         not_ok = False
@@ -72,9 +72,14 @@ def set_ooni_alarm(origin_id: int, country: str, state: AlarmState, text: str):
     alarm.update_state(state, text)
 
 
-with app.app_context():
+class BlockOONIAutomation(BaseAutomation):
-    origins = Origin.query.filter(Origin.destroyed == None).all()
+    short_name = "block_ooni"
-    for origin in origins:
+    description = "Import origin and/or proxy reachability results from OONI"
-        ooni = threshold_origin(origin.domain_name)
+
-        for country in ooni:
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
-            set_ooni_alarm(origin.id, country.lower(), ooni[country]["state"], ooni[country]["message"])
+        origins = Origin.query.filter(Origin.destroyed == None).all()
+        for origin in origins:
+            ooni = threshold_origin(origin.domain_name)
+            for country in ooni:
+                set_ooni_alarm(origin.id, country.lower(), ooni[country]["state"], ooni[country]["message"])
+        return True, ""