66 lines
2.2 KiB
Python
66 lines
2.2 KiB
Python
import logging
|
|
from collections import defaultdict
|
|
|
|
import requests
|
|
|
|
from app import app
|
|
from app.terraform.block_mirror import BlockMirrorAutomation
|
|
|
|
|
|
def _trim_prefix(s: str, prefix: str) -> str:
|
|
if s.startswith(prefix):
|
|
return s[len(prefix) :]
|
|
return s
|
|
|
|
|
|
def trim_http_https(s: str) -> str:
|
|
"""
|
|
Return the string with "http://" or "https://" removed from the start of the string if present.
|
|
|
|
:param s: String to modify.
|
|
:return: Modified string.
|
|
"""
|
|
return _trim_prefix(_trim_prefix(s, "https://"), "http://")
|
|
|
|
|
|
class BlockExternalAutomation(BlockMirrorAutomation):
|
|
"""
|
|
Automation task to import proxy reachability results from external source.
|
|
"""
|
|
|
|
short_name = "block_external"
|
|
description = "Import proxy reachability results from external source"
|
|
|
|
_content: bytes
|
|
|
|
def fetch(self) -> None:
|
|
user_agent = {"User-agent": "BypassCensorship/1.0"}
|
|
check_urls_config = app.config.get("EXTERNAL_CHECK_URL", [])
|
|
|
|
if isinstance(check_urls_config, dict):
|
|
# Config is already a dictionary, use as is.
|
|
check_urls = check_urls_config
|
|
elif isinstance(check_urls_config, list):
|
|
# Convert list of strings to a dictionary with "external_N" keys.
|
|
check_urls = {
|
|
f"external_{i}": url for i, url in enumerate(check_urls_config)
|
|
}
|
|
elif isinstance(check_urls_config, str):
|
|
# Single string, convert to a dictionary with key "external".
|
|
check_urls = {"external": check_urls_config}
|
|
else:
|
|
# Fallback if the config item is neither dict, list, nor string.
|
|
check_urls = {}
|
|
for source, check_url in check_urls.items():
|
|
if self._data is None:
|
|
self._data = defaultdict(list)
|
|
self._data[source].extend(
|
|
requests.get(check_url, headers=user_agent, timeout=30).json()
|
|
)
|
|
|
|
def parse(self) -> None:
|
|
for source, patterns in self._data.items():
|
|
self.patterns[source].extend(
|
|
["https://" + trim_http_https(pattern) for pattern in patterns]
|
|
)
|
|
logging.debug("Found URLs: %s", self.patterns)
|