block: import blocklist from roskomsvoboda

app/cli/automate.py

@@ -10,6 +10,7 @@ from app.terraform import BaseAutomation
 from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation
 from app.terraform.block_external import BlockExternalAutomation
 from app.terraform.block_ooni import BlockOONIAutomation
+from app.terraform.block_roskomsvoboda import BlockRoskomsvobodaAutomation
 from app.terraform.eotk import EotkAutomation
 from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation
 from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation
@@ -34,6 +35,7 @@ jobs = {
         BlockBridgeGitHubAutomation,
         BlockExternalAutomation,
         BlockOONIAutomation,
+        BlockRoskomsvobodaAutomation,
         BridgeAWSAutomation,
         BridgeGandiAutomation,
         BridgeHcloudAutomation,
@@ -86,7 +88,8 @@ def run_job(job: BaseAutomation, *, force: bool = False, ignore_schedule: bool =
         logs = repr(e)
     if success:
         automation.state = AutomationState.IDLE
-        automation.next_run = datetime.datetime.utcnow() + datetime.timedelta(minutes=7)
+        automation.next_run = datetime.datetime.utcnow() + datetime.timedelta(
+            minutes=getattr(job, "frequency", 7))
     else:
         automation.state = AutomationState.ERROR
         automation.enabled = False

app/terraform/block_roskomsvoboda.py

@@ -0,0 +1,34 @@
+from fnmatch import fnmatch
+from typing import Tuple, List
+
+import requests
+
+from app.extensions import db
+from app.models.mirrors import Proxy
+from app.terraform import BaseAutomation
+
+
+class BlockRoskomsvobodaAutomation(BaseAutomation):
+    short_name = "block_roskomsvoboda"
+    description = "Import Russian blocklist from RosKomSvoboda"
+    frequency = 120
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
+        proxies: List[Proxy] = Proxy.query.filter(
+            Proxy.deprecated == None,
+            Proxy.destroyed == None
+        ).all()
+        patterns = requests.get("https://reestr.rublacklist.net/api/v2/domains/json").json()
+        for pattern in patterns:
+            for p in proxies:
+                if fnmatch(p.url[len("https://"):], pattern):
+                    print(f"Found {p.url} blocked")
+                    if not p.origin.auto_rotation:
+                        print("Proxy auto-rotation forbidden for origin")
+                        continue
+                    if p.deprecated:
+                        print("Proxy already marked blocked")
+                        continue
+                    p.deprecate(reason="roskomsvoboda")
+        db.session.commit()
+        return True, ""