feat: new block plugin for blocky

app/cli/automate.py

@@ -12,6 +12,7 @@ from app.extensions import db
 from app.models.activity import Activity
 from app.models.automation import Automation, AutomationState, AutomationLogs
 from app.terraform import BaseAutomation
+from app.terraform.block.block_blocky import BlockBlockyAutomation
 from app.terraform.block.bridge_dnsc import BlockBridgeDnscAutomation
 from app.terraform.block.bridge_github import BlockBridgeGitHubAutomation
 from app.terraform.block.bridge_gitlab import BlockBridgeGitlabAutomation
@@ -51,6 +52,7 @@ jobs = {
         BlockBridgeGitlabAutomation,
         BlockBridgeRoskomsvobodaAutomation,
         BlockBridgeScriptzteamAutomation,
+        BlockBlockyAutomation,
         BlockExternalAutomation,
         BlockOONIAutomation,
         BlockRoskomsvobodaAutomation,

app/models/__init__.py

@@ -58,11 +58,11 @@ class AbstractResource(db.Model):  # type: ignore
     __abstract__ = True
 
     id: Mapped[int] = mapped_column(db.Integer, primary_key=True)
-    added: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    updated: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    deprecated: Mapped[Optional[datetime]] = mapped_column(db.DateTime())
-    deprecation_reason: Mapped[Optional[str]] = mapped_column(db.String())
-    destroyed: Mapped[Optional[datetime]] = mapped_column(db.DateTime())
+    added: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow)
+    updated: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow)
+    deprecated: Mapped[Optional[datetime]]
+    deprecation_reason: Mapped[Optional[str]]
+    destroyed: Mapped[Optional[datetime]]
 
     def __init__(self, *,
                  id: Optional[int] = None,

app/models/mirrors.py

@@ -35,7 +35,7 @@ class OriginDict(TypedDict):
 
 class Origin(AbstractConfiguration):
     group_id: Mapped[int] = mapped_column(db.Integer, db.ForeignKey("group.id"))
-    domain_name: Mapped[str] = mapped_column(db.String(255), unique=True)
+    domain_name: Mapped[str] = mapped_column(unique=True)
     auto_rotation: Mapped[bool]
     smart: Mapped[bool]
     assets: Mapped[bool]
@@ -129,8 +129,8 @@ class Country(AbstractConfiguration):
             resource_id=self.country_code
         )
 
-    country_code = mapped_column(db.String(2), nullable=False)
-    risk_level_override = mapped_column(db.Integer(), nullable=True)
+    country_code: Mapped[str]
+    risk_level_override: Mapped[Optional[int]]
 
     origins = db.relationship("Origin", secondary=country_origin, back_populates='countries')
 

app/terraform/block/block_blocky.py

@@ -0,0 +1,90 @@
+import json
+import logging
+import time
+from typing import Any, Dict
+
+import requests
+
+from app.extensions import db
+from app.models.mirrors import Country, Origin, Proxy, country_origin
+from app.terraform.block_mirror import BlockMirrorAutomation
+
+
+def clean_json_response(raw_response: str) -> Dict[str, Any]:
+    """
+    Seems to be a bug in the API where a <script> tag is appended after the JSON.
+    """
+    end_index = raw_response.rfind("}")
+    if end_index != -1:
+        raw_response = raw_response[:end_index + 1]
+    response: Dict[str, Any] = json.loads(raw_response)
+    return response
+
+
+def request_test_now(test_url: str) -> str:
+    api_url = "https://blocky.greatfire.org/api/test_now"
+    headers = {
+        "User-Agent": "bypasscensorship.org",
+        "Content-Type": "application/json;charset=utf-8",
+        "Pragma": "no-cache",
+        "Cache-Control": "no-cache"
+    }
+    request_count = 0
+    while request_count < 20:
+        params = {
+            "url": test_url,
+            "timestamp": str(int(time.time()))  # unix timestamp
+        }
+        response = requests.post(api_url, params=params, headers=headers, json={}, timeout=30)
+        response_data = clean_json_response(response.text)
+        print(f"Response: {response_data}")
+        if "url_test_id" in response_data.get("d", {}):
+            url_test_id: str = response_data["d"]["url_test_id"]
+            logging.debug("Test result for %s has test result ID %s", test_url, url_test_id)
+            return url_test_id
+        request_count += 1
+        time.sleep(1)
+    raise RuntimeWarning("Exceeded timeout requesting result")
+
+
+def request_test_result(url_test_id: str) -> int:
+    url = f"https://blocky.greatfire.org/api/url_test_result/{url_test_id}"
+    headers = {
+        "User-Agent": "bypasscensorship.org",
+        "Pragma": "no-cache",
+        "Cache-Control": "no-cache"
+    }
+    response = requests.get(url, headers=headers, timeout=30)
+    response_data = response.json()
+    tests = response_data.get("d", [])
+    non_zero_curl_exit_count: int = sum(1 for test in tests if test.get("curl_exit_value") != "0")
+    logging.debug("Test result for %s has %s non-zero exit values", url_test_id, non_zero_curl_exit_count)
+    return non_zero_curl_exit_count
+
+
+class BlockBlockyAutomation(BlockMirrorAutomation):
+    short_name = "block_blocky"
+    description = "Use the Blocky API to test for blocked mirrors in China"
+    interval = 300
+
+    def fetch(self) -> None:
+        pass
+
+    def parse(self) -> None:
+        cn_proxies = (
+            db.session.query(Proxy.url)
+            .join(Origin, Proxy.origin_id == Origin.id)
+            .join(country_origin, Origin.id == country_origin.c.origin_id)
+            .join(Country, Country.id == country_origin.c.country_id)
+            .filter(
+                Country.country_code == "CN",
+                Proxy.deprecated.is_(None),
+                Proxy.destroyed.is_(None),
+                Proxy.pool_id != -1
+            )
+            .all()
+        )
+        for proxy_url in cn_proxies:
+            test_id = request_test_now(proxy_url)
+            if request_test_result(test_id) > 1:
+                self.patterns["blocky"].append(proxy_url)