jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

block: optimising for RKS large pattern list

Browse source
This commit is contained in:
Iain Learmonth 2022-06-18 13:35:25 +01:00
parent 293d83cc1c
commit bbf070339a
2 changed files with 9 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

13
app/terraform/block_mirror.py
View file

@ -23,13 +23,16 @@ class BlockMirrorAutomation(BaseAutomation):
def automate(self, full: bool = False) -> Tuple[bool, str]: def automate(self, full: bool = False) -> Tuple[bool, str]:
self.fetch() self.fetch()
logging.debug("Fetch complete")
self.parse() self.parse()
logging.debug("Parse complete")
rotated = list() rotated = list()
for pattern in self.patterns: for proxy in active_proxies():
for proxy in active_proxies(): if proxy.url is None:
if proxy.url is None: # Not ready yet
# Not ready yet continue
continue logging.debug("Testing active proxy %s", proxy.url)
for pattern in self.patterns:
if fnmatch(proxy.url, pattern): if fnmatch(proxy.url, pattern):
logging.debug("Found %s blocked", proxy.url) logging.debug("Found %s blocked", proxy.url)
if not proxy.origin.auto_rotation: if not proxy.origin.auto_rotation:

2
app/terraform/block_roskomsvoboda.py
View file

@ -23,7 +23,7 @@ class BlockRoskomsvobodaAutomation(BlockMirrorAutomation):
_data: Any _data: Any
def fetch(self) -> None: def fetch(self) -> None:
self._data = requests.get("https://reestr.rublacklist.net/api/v2/domains/json").json() self._data = requests.get("https://reestr.rublacklist.net/api/v2/domains/json/").json()
def parse(self) -> None: def parse(self) -> None:
self.patterns.extend(["https://" + pattern for pattern in self._data]) self.patterns.extend(["https://" + pattern for pattern in self._data])