jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: geo risk scores

Browse source
This commit is contained in:
Iain Learmonth 2023-10-29 15:45:10 +00:00
parent 315dae7f06
commit 0e0d499428
17 changed files with 558 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

43
app/terraform/block_mirror.py
View file

@ -1,8 +1,9 @@
from collections import defaultdict
from datetime import datetime, timedelta
import logging
from abc import abstractmethod
import fnmatch
from typing import Tuple, List, Any, Optional
from typing import Tuple, List, Any, Optional, Dict
from app.extensions import db
from app.models.activity import Activity
@ -11,14 +12,14 @@ from app.terraform import BaseAutomation
class BlockMirrorAutomation(BaseAutomation):
patterns: List[str]
patterns: Dict[str, List[str]]
_data: Any
def __init__(self, *args: Any, **kwargs: Any) -> None:
"""
Constructor method.
"""
self.patterns = []
self.patterns = defaultdict(list)
self._data = None
super().__init__(*args, **kwargs)
@ -29,23 +30,25 @@ class BlockMirrorAutomation(BaseAutomation):
logging.debug("Parse complete")
rotated = []
proxy_urls = list(filter(lambda u: u is not None, active_proxy_urls()))
for pattern in self.patterns:
blocked_urls = fnmatch.filter(proxy_urls, pattern)
for blocked_url in blocked_urls:
if not (proxy := proxy_by_url(blocked_url)):
continue
logging.debug("Found %s blocked", proxy.url)
if not proxy.origin.auto_rotation:
logging.debug("Proxy auto-rotation forbidden for origin")
continue
if proxy.added > datetime.utcnow() - timedelta(hours=3):
logging.debug("Not rotating a proxy less than 3 hours old")
continue
if proxy.deprecate(reason=self.short_name):
logging.info("Rotated %s", proxy.url)
rotated.append((proxy.url, proxy.origin.domain_name))
else:
logging.debug("Not rotating a proxy that is already deprecated")
for source, patterns in self.patterns.items():
logging.debug("Processing blocked URLs from %s", source)
for pattern in patterns:
blocked_urls = fnmatch.filter(proxy_urls, pattern)
for blocked_url in blocked_urls:
if not (proxy := proxy_by_url(blocked_url)):
continue
logging.debug("Found %s blocked", proxy.url)
if not proxy.origin.auto_rotation:
logging.debug("Proxy auto-rotation forbidden for origin")
continue
if proxy.added > datetime.utcnow() - timedelta(hours=3):
logging.debug("Not rotating a proxy less than 3 hours old")
continue
if proxy.deprecate(reason=f"block_{source}"):
logging.info("Rotated %s", proxy.url)
rotated.append((proxy.url, proxy.origin.domain_name))
else:
logging.debug("Not rotating a proxy that is already deprecated")
if rotated:
activity = Activity(
activity_type="block",