parent
8abe5d60fa
commit
10b60b0206
4 changed files with 90 additions and 74 deletions
|
|
@ -7,6 +7,9 @@ from app import app
|
|||
from app.extensions import db
|
||||
from app.models.automation import Automation, AutomationState, AutomationLogs
|
||||
from app.terraform import BaseAutomation
|
||||
from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation
|
||||
from app.terraform.block_external import BlockExternalAutomation
|
||||
from app.terraform.block_ooni import BlockOONIAutomation
|
||||
from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation
|
||||
from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation
|
||||
from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation
|
||||
|
|
@ -27,6 +30,9 @@ jobs = {
|
|||
AlarmProxyAzureCdnAutomation,
|
||||
AlarmProxyCloudfrontAutomation,
|
||||
AlarmProxyHTTPStatusAutomation,
|
||||
BlockBridgeGitHubAutomation,
|
||||
BlockExternalAutomation,
|
||||
BlockOONIAutomation,
|
||||
BridgeAWSAutomation,
|
||||
BridgeGandiAutomation,
|
||||
BridgeHcloudAutomation,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import datetime
|
||||
from typing import Tuple
|
||||
|
||||
from dateutil.parser import isoparse
|
||||
from github import Github
|
||||
|
|
@ -6,26 +7,27 @@ from github import Github
|
|||
from app import app
|
||||
from app.extensions import db
|
||||
from app.models.bridges import Bridge
|
||||
from app.terraform import BaseAutomation
|
||||
|
||||
|
||||
def check_blocks():
|
||||
g = Github(app.config['GITHUB_API_KEY'])
|
||||
repo = g.get_repo(app.config['GITHUB_BRIDGE_REPO'])
|
||||
for vp in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
|
||||
results = repo.get_contents(f"recentResult_{vp}").decoded_content.decode('utf-8').splitlines()
|
||||
for result in results:
|
||||
parts = result.split("\t")
|
||||
if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)):
|
||||
continue
|
||||
if int(parts[1]) < 40:
|
||||
bridge: Bridge = Bridge.query.filter(
|
||||
Bridge.hashed_fingerprint == parts[0]
|
||||
).first()
|
||||
if bridge is not None:
|
||||
bridge.deprecate(reason="github")
|
||||
db.session.commit()
|
||||
class BlockBridgeGitHubAutomation(BaseAutomation):
|
||||
short_name = "block_bridge_github"
|
||||
description = "Import bridge reachability results from GitHub"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with app.app_context():
|
||||
check_blocks()
|
||||
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
||||
g = Github(app.config['GITHUB_API_KEY'])
|
||||
repo = g.get_repo(app.config['GITHUB_BRIDGE_REPO'])
|
||||
for vp in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
|
||||
results = repo.get_contents(f"recentResult_{vp}").decoded_content.decode('utf-8').splitlines()
|
||||
for result in results:
|
||||
parts = result.split("\t")
|
||||
if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)):
|
||||
continue
|
||||
if int(parts[1]) < 40:
|
||||
bridge: Bridge = Bridge.query.filter(
|
||||
Bridge.hashed_fingerprint == parts[0]
|
||||
).first()
|
||||
if bridge is not None:
|
||||
bridge.deprecate(reason="github")
|
||||
db.session.commit()
|
||||
return True, ""
|
||||
|
|
|
|||
|
|
@ -1,57 +1,60 @@
|
|||
from typing import Tuple
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
|
||||
from app import app
|
||||
from app.extensions import db
|
||||
from app.models.mirrors import Proxy
|
||||
from app.terraform import BaseAutomation
|
||||
|
||||
|
||||
def check_blocks():
|
||||
user_agent = {'User-agent': 'BypassCensorship/1.0'}
|
||||
page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent)
|
||||
soup = BeautifulSoup(page.content, 'html.parser')
|
||||
h2 = soup.find_all('h2')
|
||||
div = soup.find_all('div', class_="overflow-auto mb-5")
|
||||
class BlockExternalAutomation(BaseAutomation):
|
||||
short_name = "block_external"
|
||||
description = "Import proxy reachability results from external source"
|
||||
|
||||
results = {}
|
||||
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
||||
user_agent = {'User-agent': 'BypassCensorship/1.0'}
|
||||
page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent)
|
||||
soup = BeautifulSoup(page.content, 'html.parser')
|
||||
h2 = soup.find_all('h2')
|
||||
div = soup.find_all('div', class_="overflow-auto mb-5")
|
||||
|
||||
i = 0
|
||||
while i < len(h2):
|
||||
if not div[i].div:
|
||||
urls = []
|
||||
a = div[i].find_all('a')
|
||||
j = 0
|
||||
while j < len(a):
|
||||
urls.append(a[j].text)
|
||||
j += 1
|
||||
results[h2[i].text] = urls
|
||||
else:
|
||||
results[h2[i].text] = []
|
||||
i += 1
|
||||
results = {}
|
||||
|
||||
for vp in results:
|
||||
if vp not in app.config['EXTERNAL_VANTAGE_POINTS']:
|
||||
continue
|
||||
for url in results[vp]:
|
||||
print(f"Found {url} blocked")
|
||||
proxy = Proxy.query.filter(
|
||||
Proxy.provider == "cloudfront",
|
||||
Proxy.url == f"https://{url}"
|
||||
).first()
|
||||
if not proxy:
|
||||
print("Proxy not found")
|
||||
i = 0
|
||||
while i < len(h2):
|
||||
if not div[i].div:
|
||||
urls = []
|
||||
a = div[i].find_all('a')
|
||||
j = 0
|
||||
while j < len(a):
|
||||
urls.append(a[j].text)
|
||||
j += 1
|
||||
results[h2[i].text] = urls
|
||||
else:
|
||||
results[h2[i].text] = []
|
||||
i += 1
|
||||
|
||||
for vp in results:
|
||||
if vp not in app.config['EXTERNAL_VANTAGE_POINTS']:
|
||||
continue
|
||||
if not proxy.origin.auto_rotation:
|
||||
print("Proxy auto-rotation forbidden for origin")
|
||||
continue
|
||||
if proxy.deprecated:
|
||||
print("Proxy already marked blocked")
|
||||
continue
|
||||
proxy.deprecate(reason="external")
|
||||
for url in results[vp]:
|
||||
print(f"Found {url} blocked")
|
||||
proxy = Proxy.query.filter(
|
||||
Proxy.provider == "cloudfront",
|
||||
Proxy.url == f"https://{url}"
|
||||
).first()
|
||||
if not proxy:
|
||||
print("Proxy not found")
|
||||
continue
|
||||
if not proxy.origin.auto_rotation:
|
||||
print("Proxy auto-rotation forbidden for origin")
|
||||
continue
|
||||
if proxy.deprecated:
|
||||
print("Proxy already marked blocked")
|
||||
continue
|
||||
proxy.deprecate(reason="external")
|
||||
db.session.commit()
|
||||
|
||||
db.session.commit()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with app.app_context():
|
||||
check_blocks()
|
||||
return True, ""
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from typing import Dict
|
||||
from typing import Dict, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from app import app
|
||||
from app.extensions import db
|
||||
from app.models.alarms import Alarm, AlarmState
|
||||
from app.models.mirrors import Origin
|
||||
from app.terraform import BaseAutomation
|
||||
|
||||
|
||||
def check_origin(domain_name: str):
|
||||
start_date = (datetime.utcnow() - timedelta(days=2)).strftime("%Y-%m-%dT%H%%3A%M")
|
||||
start_date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M")
|
||||
end_date = datetime.utcnow().strftime("%Y-%m-%dT%H%%3A%M")
|
||||
api_url = f"https://api.ooni.io/api/v1/measurements?domain={domain_name}&since={start_date}&until={end_date}"
|
||||
result = defaultdict(lambda: {"anomaly": 0, "confirmed": 0, "failure": 0, "ok": 0})
|
||||
|
|
@ -22,7 +22,7 @@ def check_origin(domain_name: str):
|
|||
def _check_origin(api_url: str, result: Dict):
|
||||
print(f"Processing {api_url}")
|
||||
req = requests.get(api_url).json()
|
||||
if not req['results']:
|
||||
if 'results' not in req or not req['results']:
|
||||
return result
|
||||
for r in req['results']:
|
||||
not_ok = False
|
||||
|
|
@ -72,9 +72,14 @@ def set_ooni_alarm(origin_id: int, country: str, state: AlarmState, text: str):
|
|||
alarm.update_state(state, text)
|
||||
|
||||
|
||||
with app.app_context():
|
||||
origins = Origin.query.filter(Origin.destroyed == None).all()
|
||||
for origin in origins:
|
||||
ooni = threshold_origin(origin.domain_name)
|
||||
for country in ooni:
|
||||
set_ooni_alarm(origin.id, country.lower(), ooni[country]["state"], ooni[country]["message"])
|
||||
class BlockOONIAutomation(BaseAutomation):
|
||||
short_name = "block_ooni"
|
||||
description = "Import origin and/or proxy reachability results from OONI"
|
||||
|
||||
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
||||
origins = Origin.query.filter(Origin.destroyed == None).all()
|
||||
for origin in origins:
|
||||
ooni = threshold_origin(origin.domain_name)
|
||||
for country in ooni:
|
||||
set_ooni_alarm(origin.id, country.lower(), ooni[country]["state"], ooni[country]["message"])
|
||||
return True, ""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue