diff --git a/app/cli/automate.py b/app/cli/automate.py index ffd7335..6b81e0f 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -7,6 +7,9 @@ from app import app from app.extensions import db from app.models.automation import Automation, AutomationState, AutomationLogs from app.terraform import BaseAutomation +from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation +from app.terraform.block_external import BlockExternalAutomation +from app.terraform.block_ooni import BlockOONIAutomation from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation @@ -27,6 +30,9 @@ jobs = { AlarmProxyAzureCdnAutomation, AlarmProxyCloudfrontAutomation, AlarmProxyHTTPStatusAutomation, + BlockBridgeGitHubAutomation, + BlockExternalAutomation, + BlockOONIAutomation, BridgeAWSAutomation, BridgeGandiAutomation, BridgeHcloudAutomation, diff --git a/app/terraform/block_bridge_github.py b/app/terraform/block_bridge_github.py index dc61fd9..92c976c 100644 --- a/app/terraform/block_bridge_github.py +++ b/app/terraform/block_bridge_github.py @@ -1,4 +1,5 @@ import datetime +from typing import Tuple from dateutil.parser import isoparse from github import Github @@ -6,26 +7,27 @@ from github import Github from app import app from app.extensions import db from app.models.bridges import Bridge +from app.terraform import BaseAutomation -def check_blocks(): - g = Github(app.config['GITHUB_API_KEY']) - repo = g.get_repo(app.config['GITHUB_BRIDGE_REPO']) - for vp in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']: - results = repo.get_contents(f"recentResult_{vp}").decoded_content.decode('utf-8').splitlines() - for result in results: - parts = result.split("\t") - if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)): - continue - if int(parts[1]) < 40: - bridge: Bridge = Bridge.query.filter( - Bridge.hashed_fingerprint == parts[0] - ).first() - if bridge is not None: - bridge.deprecate(reason="github") - db.session.commit() +class BlockBridgeGitHubAutomation(BaseAutomation): + short_name = "block_bridge_github" + description = "Import bridge reachability results from GitHub" - -if __name__ == "__main__": - with app.app_context(): - check_blocks() + def automate(self, full: bool = False) -> Tuple[bool, str]: + g = Github(app.config['GITHUB_API_KEY']) + repo = g.get_repo(app.config['GITHUB_BRIDGE_REPO']) + for vp in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']: + results = repo.get_contents(f"recentResult_{vp}").decoded_content.decode('utf-8').splitlines() + for result in results: + parts = result.split("\t") + if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)): + continue + if int(parts[1]) < 40: + bridge: Bridge = Bridge.query.filter( + Bridge.hashed_fingerprint == parts[0] + ).first() + if bridge is not None: + bridge.deprecate(reason="github") + db.session.commit() + return True, "" diff --git a/app/terraform/block_external.py b/app/terraform/block_external.py index c2d380b..ea1b5b7 100644 --- a/app/terraform/block_external.py +++ b/app/terraform/block_external.py @@ -1,57 +1,60 @@ +from typing import Tuple + from bs4 import BeautifulSoup import requests from app import app from app.extensions import db from app.models.mirrors import Proxy +from app.terraform import BaseAutomation -def check_blocks(): - user_agent = {'User-agent': 'BypassCensorship/1.0'} - page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent) - soup = BeautifulSoup(page.content, 'html.parser') - h2 = soup.find_all('h2') - div = soup.find_all('div', class_="overflow-auto mb-5") +class BlockExternalAutomation(BaseAutomation): + short_name = "block_external" + description = "Import proxy reachability results from external source" - results = {} + def automate(self, full: bool = False) -> Tuple[bool, str]: + user_agent = {'User-agent': 'BypassCensorship/1.0'} + page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent) + soup = BeautifulSoup(page.content, 'html.parser') + h2 = soup.find_all('h2') + div = soup.find_all('div', class_="overflow-auto mb-5") - i = 0 - while i < len(h2): - if not div[i].div: - urls = [] - a = div[i].find_all('a') - j = 0 - while j < len(a): - urls.append(a[j].text) - j += 1 - results[h2[i].text] = urls - else: - results[h2[i].text] = [] - i += 1 + results = {} - for vp in results: - if vp not in app.config['EXTERNAL_VANTAGE_POINTS']: - continue - for url in results[vp]: - print(f"Found {url} blocked") - proxy = Proxy.query.filter( - Proxy.provider == "cloudfront", - Proxy.url == f"https://{url}" - ).first() - if not proxy: - print("Proxy not found") + i = 0 + while i < len(h2): + if not div[i].div: + urls = [] + a = div[i].find_all('a') + j = 0 + while j < len(a): + urls.append(a[j].text) + j += 1 + results[h2[i].text] = urls + else: + results[h2[i].text] = [] + i += 1 + + for vp in results: + if vp not in app.config['EXTERNAL_VANTAGE_POINTS']: continue - if not proxy.origin.auto_rotation: - print("Proxy auto-rotation forbidden for origin") - continue - if proxy.deprecated: - print("Proxy already marked blocked") - continue - proxy.deprecate(reason="external") + for url in results[vp]: + print(f"Found {url} blocked") + proxy = Proxy.query.filter( + Proxy.provider == "cloudfront", + Proxy.url == f"https://{url}" + ).first() + if not proxy: + print("Proxy not found") + continue + if not proxy.origin.auto_rotation: + print("Proxy auto-rotation forbidden for origin") + continue + if proxy.deprecated: + print("Proxy already marked blocked") + continue + proxy.deprecate(reason="external") + db.session.commit() - db.session.commit() - - -if __name__ == "__main__": - with app.app_context(): - check_blocks() + return True, "" diff --git a/app/terraform/block_ooni.py b/app/terraform/block_ooni.py index e471e7b..3903fd6 100644 --- a/app/terraform/block_ooni.py +++ b/app/terraform/block_ooni.py @@ -1,18 +1,18 @@ from collections import defaultdict from datetime import datetime from datetime import timedelta -from typing import Dict +from typing import Dict, Tuple import requests -from app import app from app.extensions import db from app.models.alarms import Alarm, AlarmState from app.models.mirrors import Origin +from app.terraform import BaseAutomation def check_origin(domain_name: str): - start_date = (datetime.utcnow() - timedelta(days=2)).strftime("%Y-%m-%dT%H%%3A%M") + start_date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M") end_date = datetime.utcnow().strftime("%Y-%m-%dT%H%%3A%M") api_url = f"https://api.ooni.io/api/v1/measurements?domain={domain_name}&since={start_date}&until={end_date}" result = defaultdict(lambda: {"anomaly": 0, "confirmed": 0, "failure": 0, "ok": 0}) @@ -22,7 +22,7 @@ def check_origin(domain_name: str): def _check_origin(api_url: str, result: Dict): print(f"Processing {api_url}") req = requests.get(api_url).json() - if not req['results']: + if 'results' not in req or not req['results']: return result for r in req['results']: not_ok = False @@ -72,9 +72,14 @@ def set_ooni_alarm(origin_id: int, country: str, state: AlarmState, text: str): alarm.update_state(state, text) -with app.app_context(): - origins = Origin.query.filter(Origin.destroyed == None).all() - for origin in origins: - ooni = threshold_origin(origin.domain_name) - for country in ooni: - set_ooni_alarm(origin.id, country.lower(), ooni[country]["state"], ooni[country]["message"]) +class BlockOONIAutomation(BaseAutomation): + short_name = "block_ooni" + description = "Import origin and/or proxy reachability results from OONI" + + def automate(self, full: bool = False) -> Tuple[bool, str]: + origins = Origin.query.filter(Origin.destroyed == None).all() + for origin in origins: + ooni = threshold_origin(origin.domain_name) + for country in ooni: + set_ooni_alarm(origin.id, country.lower(), ooni[country]["state"], ooni[country]["message"]) + return True, ""