automation: herd blocking automations into framework

see #1

app/cli/automate.py

@@ -7,6 +7,9 @@ from app import app
 from app.extensions import db
 from app.models.automation import Automation, AutomationState, AutomationLogs
 from app.terraform import BaseAutomation
+from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation
+from app.terraform.block_external import BlockExternalAutomation
+from app.terraform.block_ooni import BlockOONIAutomation
 from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation
 from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation
 from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation
@@ -27,6 +30,9 @@ jobs = {
         AlarmProxyAzureCdnAutomation,
         AlarmProxyCloudfrontAutomation,
         AlarmProxyHTTPStatusAutomation,
+        BlockBridgeGitHubAutomation,
+        BlockExternalAutomation,
+        BlockOONIAutomation,
         BridgeAWSAutomation,
         BridgeGandiAutomation,
         BridgeHcloudAutomation,

app/terraform/block_bridge_github.py

@@ -1,4 +1,5 @@
 import datetime
+from typing import Tuple
 
 from dateutil.parser import isoparse
 from github import Github
@@ -6,9 +7,14 @@ from github import Github
 from app import app
 from app.extensions import db
 from app.models.bridges import Bridge
+from app.terraform import BaseAutomation
 
 
-def check_blocks():
+class BlockBridgeGitHubAutomation(BaseAutomation):
+    short_name = "block_bridge_github"
+    description = "Import bridge reachability results from GitHub"
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
         g = Github(app.config['GITHUB_API_KEY'])
         repo = g.get_repo(app.config['GITHUB_BRIDGE_REPO'])
         for vp in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
@@ -24,8 +30,4 @@ def check_blocks():
                     if bridge is not None:
                         bridge.deprecate(reason="github")
         db.session.commit()
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        check_blocks()
+        return True, ""

app/terraform/block_external.py

@@ -1,12 +1,19 @@
+from typing import Tuple
+
 from bs4 import BeautifulSoup
 import requests
 
 from app import app
 from app.extensions import db
 from app.models.mirrors import Proxy
+from app.terraform import BaseAutomation
 
 
-def check_blocks():
+class BlockExternalAutomation(BaseAutomation):
+    short_name = "block_external"
+    description = "Import proxy reachability results from external source"
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
         user_agent = {'User-agent': 'BypassCensorship/1.0'}
         page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent)
         soup = BeautifulSoup(page.content, 'html.parser')
@@ -48,10 +55,6 @@ def check_blocks():
                     print("Proxy already marked blocked")
                     continue
                 proxy.deprecate(reason="external")
-
         db.session.commit()
 
-
-if __name__ == "__main__":
-    with app.app_context():
-        check_blocks()
+        return True, ""

app/terraform/block_ooni.py

@@ -1,18 +1,18 @@
 from collections import defaultdict
 from datetime import datetime
 from datetime import timedelta
-from typing import Dict
+from typing import Dict, Tuple
 
 import requests
 
-from app import app
 from app.extensions import db
 from app.models.alarms import Alarm, AlarmState
 from app.models.mirrors import Origin
+from app.terraform import BaseAutomation
 
 
 def check_origin(domain_name: str):
-    start_date = (datetime.utcnow() - timedelta(days=2)).strftime("%Y-%m-%dT%H%%3A%M")
+    start_date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M")
     end_date = datetime.utcnow().strftime("%Y-%m-%dT%H%%3A%M")
     api_url = f"https://api.ooni.io/api/v1/measurements?domain={domain_name}&since={start_date}&until={end_date}"
     result = defaultdict(lambda: {"anomaly": 0, "confirmed": 0, "failure": 0, "ok": 0})
@@ -22,7 +22,7 @@ def check_origin(domain_name: str):
 def _check_origin(api_url: str, result: Dict):
     print(f"Processing {api_url}")
     req = requests.get(api_url).json()
-    if not req['results']:
+    if 'results' not in req or not req['results']:
         return result
     for r in req['results']:
         not_ok = False
@@ -72,9 +72,14 @@ def set_ooni_alarm(origin_id: int, country: str, state: AlarmState, text: str):
     alarm.update_state(state, text)
 
 
-with app.app_context():
+class BlockOONIAutomation(BaseAutomation):
+    short_name = "block_ooni"
+    description = "Import origin and/or proxy reachability results from OONI"
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
         origins = Origin.query.filter(Origin.destroyed == None).all()
         for origin in origins:
             ooni = threshold_origin(origin.domain_name)
             for country in ooni:
                 set_ooni_alarm(origin.id, country.lower(), ooni[country]["state"], ooni[country]["message"])
+        return True, ""