majuna/app/terraform/block_mirror.py

import fnmatch
import logging
from abc import abstractmethod
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Tuple

from app.extensions import db
from app.models.activity import Activity
from app.models.mirrors import Proxy
from app.terraform import BaseAutomation


class BlockMirrorAutomation(BaseAutomation):
    patterns: Dict[str, List[str]]
    _data: Any

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        """
        Constructor method.
        """
        self.patterns = defaultdict(list)
        self._data = None
        super().__init__(*args, **kwargs)

    def automate(self, full: bool = False) -> Tuple[bool, str]:
        self.fetch()
        logging.debug("Fetch complete")
        self.parse()
        logging.debug("Parse complete")
        rotated = []
        proxy_urls = list(filter(lambda u: u is not None, active_proxy_urls()))
        for source, patterns in self.patterns.items():
            logging.debug("Processing blocked URLs from %s", source)
            for pattern in patterns:
                blocked_urls = fnmatch.filter(proxy_urls, pattern)
                for blocked_url in blocked_urls:
                    if not (proxy := proxy_by_url(blocked_url)):
                        continue
                    logging.debug("Found %s blocked", proxy.url)
                    if not proxy.origin.auto_rotation:
                        logging.debug("Proxy auto-rotation forbidden for origin")
                        continue
                    if proxy.added > datetime.now(tz=timezone.utc) - timedelta(hours=3):
                        logging.debug("Not rotating a proxy less than 3 hours old")
                        continue
                    if proxy.deprecate(reason=f"block_{source}"):
                        logging.info("Rotated %s", proxy.url)
                        rotated.append((proxy.url, proxy.origin.domain_name))
                    else:
                        logging.debug("Not rotating a proxy that is already deprecated")
        if rotated:
            activity = Activity(
                activity_type="block",
                text=(f"[{self.short_name}] ♻ Rotated {len(rotated)} proxies️️: \n"
                      + "\n".join([f"* {proxy_domain} ({origin_domain})" for proxy_domain, origin_domain in rotated]))
            )
            db.session.add(activity)
            activity.notify()
            db.session.commit()
        return True, ""

    @abstractmethod
    def fetch(self) -> None:
        """
        Fetch the blocklist data. It is the responsibility of the automation task
        to persist this within the object for the parse step.

        :return: None
        """

    @abstractmethod
    def parse(self) -> None:
        """
        Parse the blocklist data.

        :return: None
        """


def active_proxy_urls() -> List[str]:
    return [proxy.url for proxy in Proxy.query.filter(
        Proxy.deprecated.is_(None),
        Proxy.destroyed.is_(None)
    ).all()]


def proxy_by_url(url: str) -> Optional[Proxy]:
    return Proxy.query.filter(  # type: ignore[no-any-return]
        Proxy.deprecated.is_(None),
        Proxy.destroyed.is_(None),
        Proxy.url == url
    ).first()