From bbf070339a82d3d8fb1d23bd13d9a55f7916b65d Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Sat, 18 Jun 2022 13:35:25 +0100 Subject: [PATCH] block: optimising for RKS large pattern list --- app/terraform/block_mirror.py | 13 ++++++++----- app/terraform/block_roskomsvoboda.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/terraform/block_mirror.py b/app/terraform/block_mirror.py index fe72241..2342a3b 100644 --- a/app/terraform/block_mirror.py +++ b/app/terraform/block_mirror.py @@ -23,13 +23,16 @@ class BlockMirrorAutomation(BaseAutomation): def automate(self, full: bool = False) -> Tuple[bool, str]: self.fetch() + logging.debug("Fetch complete") self.parse() + logging.debug("Parse complete") rotated = list() - for pattern in self.patterns: - for proxy in active_proxies(): - if proxy.url is None: - # Not ready yet - continue + for proxy in active_proxies(): + if proxy.url is None: + # Not ready yet + continue + logging.debug("Testing active proxy %s", proxy.url) + for pattern in self.patterns: if fnmatch(proxy.url, pattern): logging.debug("Found %s blocked", proxy.url) if not proxy.origin.auto_rotation: diff --git a/app/terraform/block_roskomsvoboda.py b/app/terraform/block_roskomsvoboda.py index 46e93ae..457772e 100644 --- a/app/terraform/block_roskomsvoboda.py +++ b/app/terraform/block_roskomsvoboda.py @@ -23,7 +23,7 @@ class BlockRoskomsvobodaAutomation(BlockMirrorAutomation): _data: Any def fetch(self) -> None: - self._data = requests.get("https://reestr.rublacklist.net/api/v2/domains/json").json() + self._data = requests.get("https://reestr.rublacklist.net/api/v2/domains/json/").json() def parse(self) -> None: self.patterns.extend(["https://" + pattern for pattern in self._data])