jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

block: introduce threading to speed it up

Browse source
This commit is contained in:
Iain Learmonth 2022-06-18 13:47:29 +01:00
parent bbf070339a
commit f78e4d67ad
1 changed files with 25 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

40
app/terraform/block_mirror.py
View file

@ -1,3 +1,6 @@
import functools
import os
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta from datetime import datetime, timedelta
import logging import logging
from abc import abstractmethod from abc import abstractmethod
@ -11,7 +14,6 @@ from app.terraform import BaseAutomation
class BlockMirrorAutomation(BaseAutomation): class BlockMirrorAutomation(BaseAutomation):
patterns: List[str] patterns: List[str]
def __init__(self) -> None: def __init__(self) -> None:
@ -32,20 +34,19 @@ class BlockMirrorAutomation(BaseAutomation):
# Not ready yet # Not ready yet
continue continue
logging.debug("Testing active proxy %s", proxy.url) logging.debug("Testing active proxy %s", proxy.url)
for pattern in self.patterns: if is_match(proxy.url, self.patterns):
if fnmatch(proxy.url, pattern): logging.debug("Found %s blocked", proxy.url)
logging.debug("Found %s blocked", proxy.url) if not proxy.origin.auto_rotation:
if not proxy.origin.auto_rotation: logging.debug("Proxy auto-rotation forbidden for origin")
logging.debug("Proxy auto-rotation forbidden for origin") continue
continue if proxy.added > datetime.utcnow() - timedelta(hours=3):
if proxy.added > datetime.utcnow() - timedelta(hours=3): logging.debug("Not rotating a proxy less than 3 hours old")
logging.debug("Not rotating a proxy less than 3 hours old") continue
continue if proxy.deprecate(reason=self.short_name):
if proxy.deprecate(reason=self.short_name): logging.info("Rotated %s", proxy.url)
logging.info("Rotated %s", proxy.url) rotated.append((proxy.url, proxy.origin.domain_name))
rotated.append((proxy.url, proxy.origin.domain_name)) else:
else: logging.debug("Not rotating a proxy that is already deprecated")
logging.debug("Not rotating a proxy that is already deprecated")
if rotated: if rotated:
activity = Activity( activity = Activity(
activity_type="block", activity_type="block",
@ -80,3 +81,12 @@ def active_proxies() -> List[Proxy]:
Proxy.deprecated.is_(None), Proxy.deprecated.is_(None),
Proxy.destroyed.is_(None) Proxy.destroyed.is_(None)
).all() ).all()
def is_match(test_url: str, patterns: List[str]):
with ThreadPoolExecutor(os.cpu_count() - 1) as executor:
url_fnmatch = functools.partial(fnmatch, test_url)
for result in executor.map(fnmatch, patterns):
if result:
return True
return False