jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lint: reformat python code with black

Browse source
This commit is contained in:
Iain Learmonth 2024-12-06 18:15:47 +00:00
parent 331beb01b4
commit a406a7974b
88 changed files with 2579 additions and 1608 deletions
Download patch file Download diff file Expand all files Collapse all files

31
app/terraform/block/block_blocky.py
View file

@ -16,7 +16,7 @@ def clean_json_response(raw_response: str) -> Dict[str, Any]:
"""
end_index = raw_response.rfind("}")
if end_index != -1:
raw_response = raw_response[:end_index + 1]
raw_response = raw_response[: end_index + 1]
response: Dict[str, Any] = json.loads(raw_response)
return response
@ -27,20 +27,21 @@ def request_test_now(test_url: str) -> str:
"User-Agent": "bypasscensorship.org",
"Content-Type": "application/json;charset=utf-8",
"Pragma": "no-cache",
"Cache-Control": "no-cache"
"Cache-Control": "no-cache",
}
request_count = 0
while request_count < 180:
params = {
"url": test_url,
"timestamp": str(int(time.time())) # unix timestamp
}
response = requests.post(api_url, params=params, headers=headers, json={}, timeout=30)
params = {"url": test_url, "timestamp": str(int(time.time()))} # unix timestamp
response = requests.post(
api_url, params=params, headers=headers, json={}, timeout=30
)
response_data = clean_json_response(response.text)
print(f"Response: {response_data}")
if "url_test_id" in response_data.get("d", {}):
url_test_id: str = response_data["d"]["url_test_id"]
logging.debug("Test result for %s has test result ID %s", test_url, url_test_id)
logging.debug(
"Test result for %s has test result ID %s", test_url, url_test_id
)
return url_test_id
request_count += 1
time.sleep(2)
@ -52,13 +53,19 @@ def request_test_result(url_test_id: str) -> int:
headers = {
"User-Agent": "bypasscensorship.org",
"Pragma": "no-cache",
"Cache-Control": "no-cache"
"Cache-Control": "no-cache",
}
response = requests.get(url, headers=headers, timeout=30)
response_data = response.json()
tests = response_data.get("d", [])
non_zero_curl_exit_count: int = sum(1 for test in tests if test.get("curl_exit_value") != "0")
logging.debug("Test result for %s has %s non-zero exit values", url_test_id, non_zero_curl_exit_count)
non_zero_curl_exit_count: int = sum(
1 for test in tests if test.get("curl_exit_value") != "0"
)
logging.debug(
"Test result for %s has %s non-zero exit values",
url_test_id,
non_zero_curl_exit_count,
)
return non_zero_curl_exit_count
@ -81,7 +88,7 @@ class BlockBlockyAutomation(BlockMirrorAutomation):
Proxy.url.is_not(None),
Proxy.deprecated.is_(None),
Proxy.destroyed.is_(None),
Proxy.pool_id != -1
Proxy.pool_id != -1,
)
.all()
)

3
app/terraform/block/block_scriptzteam.py
View file

@ -15,7 +15,8 @@ class BlockBridgeScriptzteamAutomation(BlockBridgelinesAutomation):
def fetch(self) -> None:
r = requests.get(
"https://raw.githubusercontent.com/scriptzteam/Tor-Bridges-Collector/main/bridges-obfs4",
timeout=60)
timeout=60,
)
r.encoding = "utf-8"
contents = r.text
self._lines = contents.splitlines()

40
app/terraform/block/bridge.py
View file

@ -24,8 +24,9 @@ class BlockBridgeAutomation(BaseAutomation):
self.hashed_fingerprints = []
super().__init__(*args, **kwargs)
def perform_deprecations(self, ids: List[str], bridge_select_func: Callable[[str], Optional[Bridge]]
) -> List[Tuple[Optional[str], Any, Any]]:
def perform_deprecations(
self, ids: List[str], bridge_select_func: Callable[[str], Optional[Bridge]]
) -> List[Tuple[Optional[str], Any, Any]]:
rotated = []
for id_ in ids:
bridge = bridge_select_func(id_)
@ -37,7 +38,13 @@ class BlockBridgeAutomation(BaseAutomation):
continue
if bridge.deprecate(reason=self.short_name):
logging.info("Rotated %s", bridge.hashed_fingerprint)
rotated.append((bridge.fingerprint, bridge.cloud_account.provider, bridge.cloud_account.description))
rotated.append(
(
bridge.fingerprint,
bridge.cloud_account.provider,
bridge.cloud_account.description,
)
)
else:
logging.debug("Not rotating a bridge that is already deprecated")
return rotated
@ -50,15 +57,28 @@ class BlockBridgeAutomation(BaseAutomation):
rotated = []
rotated.extend(self.perform_deprecations(self.ips, get_bridge_by_ip))
logging.debug("Blocked by IP")
rotated.extend(self.perform_deprecations(self.fingerprints, get_bridge_by_fingerprint))
rotated.extend(
self.perform_deprecations(self.fingerprints, get_bridge_by_fingerprint)
)
logging.debug("Blocked by fingerprint")
rotated.extend(self.perform_deprecations(self.hashed_fingerprints, get_bridge_by_hashed_fingerprint))
rotated.extend(
self.perform_deprecations(
self.hashed_fingerprints, get_bridge_by_hashed_fingerprint
)
)
logging.debug("Blocked by hashed fingerprint")
if rotated:
activity = Activity(
activity_type="block",
text=(f"[{self.short_name}] ♻ Rotated {len(rotated)} bridges: \n"
+ "\n".join([f"* {fingerprint} ({provider}: {provider_description})" for fingerprint, provider, provider_description in rotated]))
text=(
f"[{self.short_name}] ♻ Rotated {len(rotated)} bridges: \n"
+ "\n".join(
[
f"* {fingerprint} ({provider}: {provider_description})"
for fingerprint, provider, provider_description in rotated
]
)
),
)
db.session.add(activity)
activity.notify()
@ -87,7 +107,7 @@ def get_bridge_by_ip(ip: str) -> Optional[Bridge]:
return Bridge.query.filter( # type: ignore[no-any-return]
Bridge.deprecated.is_(None),
Bridge.destroyed.is_(None),
Bridge.bridgeline.contains(f" {ip} ")
Bridge.bridgeline.contains(f" {ip} "),
).first()
@ -95,7 +115,7 @@ def get_bridge_by_fingerprint(fingerprint: str) -> Optional[Bridge]:
return Bridge.query.filter( # type: ignore[no-any-return]
Bridge.deprecated.is_(None),
Bridge.destroyed.is_(None),
Bridge.fingerprint == fingerprint
Bridge.fingerprint == fingerprint,
).first()
@ -103,5 +123,5 @@ def get_bridge_by_hashed_fingerprint(hashed_fingerprint: str) -> Optional[Bridge
return Bridge.query.filter( # type: ignore[no-any-return]
Bridge.deprecated.is_(None),
Bridge.destroyed.is_(None),
Bridge.hashed_fingerprint == hashed_fingerprint
Bridge.hashed_fingerprint == hashed_fingerprint,
).first()

4
app/terraform/block/bridge_bridgelines.py
View file

@ -17,6 +17,8 @@ class BlockBridgelinesAutomation(BlockBridgeAutomation, ABC):
fingerprint = parts[2]
self.ips.append(ip_address)
self.fingerprints.append(fingerprint)
logging.debug(f"Added blocked bridge with IP {ip_address} and fingerprint {fingerprint}")
logging.debug(
f"Added blocked bridge with IP {ip_address} and fingerprint {fingerprint}"
)
except IndexError:
logging.warning("A parsing error occured.")

14
app/terraform/block/bridge_github.py
View file

@ -1,8 +1,7 @@
from flask import current_app
from github import Github
from app.terraform.block.bridge_reachability import \
BlockBridgeReachabilityAutomation
from app.terraform.block.bridge_reachability import BlockBridgeReachabilityAutomation
class BlockBridgeGitHubAutomation(BlockBridgeReachabilityAutomation):
@ -15,12 +14,13 @@ class BlockBridgeGitHubAutomation(BlockBridgeReachabilityAutomation):
frequency = 30
def fetch(self) -> None:
github = Github(current_app.config['GITHUB_API_KEY'])
repo = github.get_repo(current_app.config['GITHUB_BRIDGE_REPO'])
for vantage_point in current_app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
github = Github(current_app.config["GITHUB_API_KEY"])
repo = github.get_repo(current_app.config["GITHUB_BRIDGE_REPO"])
for vantage_point in current_app.config["GITHUB_BRIDGE_VANTAGE_POINTS"]:
contents = repo.get_contents(f"recentResult_{vantage_point}")
if isinstance(contents, list):
raise RuntimeError(
f"Expected a file at recentResult_{vantage_point}"
" but got a directory.")
self._lines = contents.decoded_content.decode('utf-8').splitlines()
" but got a directory."
)
self._lines = contents.decoded_content.decode("utf-8").splitlines()

15
app/terraform/block/bridge_gitlab.py
View file

@ -1,8 +1,7 @@
from flask import current_app
from gitlab import Gitlab
from app.terraform.block.bridge_reachability import \
BlockBridgeReachabilityAutomation
from app.terraform.block.bridge_reachability import BlockBridgeReachabilityAutomation
class BlockBridgeGitlabAutomation(BlockBridgeReachabilityAutomation):
@ -16,15 +15,15 @@ class BlockBridgeGitlabAutomation(BlockBridgeReachabilityAutomation):
def fetch(self) -> None:
self._lines = list()
credentials = {"private_token": current_app.config['GITLAB_TOKEN']}
credentials = {"private_token": current_app.config["GITLAB_TOKEN"]}
if "GITLAB_URL" in current_app.config:
credentials['url'] = current_app.config['GITLAB_URL']
credentials["url"] = current_app.config["GITLAB_URL"]
gitlab = Gitlab(**credentials)
project = gitlab.projects.get(current_app.config['GITLAB_BRIDGE_PROJECT'])
for vantage_point in current_app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
project = gitlab.projects.get(current_app.config["GITLAB_BRIDGE_PROJECT"])
for vantage_point in current_app.config["GITHUB_BRIDGE_VANTAGE_POINTS"]:
contents = project.files.get(
file_path=f"recentResult_{vantage_point}",
ref=current_app.config["GITLAB_BRIDGE_BRANCH"]
ref=current_app.config["GITLAB_BRIDGE_BRANCH"],
)
# Decode the base64 first, then decode the UTF-8 string
self._lines.extend(contents.decode().decode('utf-8').splitlines())
self._lines.extend(contents.decode().decode("utf-8").splitlines())

6
app/terraform/block/bridge_reachability.py
View file

@ -14,8 +14,10 @@ class BlockBridgeReachabilityAutomation(BlockBridgeAutomation, ABC):
def parse(self) -> None:
for line in self._lines:
parts = line.split("\t")
if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc)
- datetime.timedelta(days=3)):
if isoparse(parts[2]) < (
datetime.datetime.now(datetime.timezone.utc)
- datetime.timedelta(days=3)
):
# Skip results older than 3 days
continue
if int(parts[1]) < 40:

4
app/terraform/block/bridge_roskomsvoboda.py
View file

@ -13,7 +13,9 @@ class BlockBridgeRoskomsvobodaAutomation(BlockBridgeAutomation):
_data: Any
def fetch(self) -> None:
self._data = requests.get("https://reestr.rublacklist.net/api/v3/ips/", timeout=180).json()
self._data = requests.get(
"https://reestr.rublacklist.net/api/v3/ips/", timeout=180
).json()
def parse(self) -> None:
self.ips.extend(self._data)