diff --git a/app/terraform/block_mirror.py b/app/terraform/block_mirror.py index 2342a3b..909f4ce 100644 --- a/app/terraform/block_mirror.py +++ b/app/terraform/block_mirror.py @@ -1,3 +1,6 @@ +import functools +import os +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta import logging from abc import abstractmethod @@ -11,7 +14,6 @@ from app.terraform import BaseAutomation class BlockMirrorAutomation(BaseAutomation): - patterns: List[str] def __init__(self) -> None: @@ -32,20 +34,19 @@ class BlockMirrorAutomation(BaseAutomation): # Not ready yet continue logging.debug("Testing active proxy %s", proxy.url) - for pattern in self.patterns: - if fnmatch(proxy.url, pattern): - logging.debug("Found %s blocked", proxy.url) - if not proxy.origin.auto_rotation: - logging.debug("Proxy auto-rotation forbidden for origin") - continue - if proxy.added > datetime.utcnow() - timedelta(hours=3): - logging.debug("Not rotating a proxy less than 3 hours old") - continue - if proxy.deprecate(reason=self.short_name): - logging.info("Rotated %s", proxy.url) - rotated.append((proxy.url, proxy.origin.domain_name)) - else: - logging.debug("Not rotating a proxy that is already deprecated") + if is_match(proxy.url, self.patterns): + logging.debug("Found %s blocked", proxy.url) + if not proxy.origin.auto_rotation: + logging.debug("Proxy auto-rotation forbidden for origin") + continue + if proxy.added > datetime.utcnow() - timedelta(hours=3): + logging.debug("Not rotating a proxy less than 3 hours old") + continue + if proxy.deprecate(reason=self.short_name): + logging.info("Rotated %s", proxy.url) + rotated.append((proxy.url, proxy.origin.domain_name)) + else: + logging.debug("Not rotating a proxy that is already deprecated") if rotated: activity = Activity( activity_type="block", @@ -80,3 +81,12 @@ def active_proxies() -> List[Proxy]: Proxy.deprecated.is_(None), Proxy.destroyed.is_(None) ).all() + + +def is_match(test_url: str, patterns: List[str]): + with ThreadPoolExecutor(os.cpu_count() - 1) as executor: + url_fnmatch = functools.partial(fnmatch, test_url) + for result in executor.map(fnmatch, patterns): + if result: + return True + return False