majuna/app/terraform/block_roskomsvoboda.py

from fnmatch import fnmatch
from typing import Tuple, List

import requests

from app.extensions import db
from app.models.activity import Activity
from app.models.mirrors import Proxy
from app.terraform import BaseAutomation


class BlockRoskomsvobodaAutomation(BaseAutomation):
    """
    Automation task to import Russian blocklist from RosKomSvoboda.

    This task will import the Russian state register of prohibited sites,
    which is part of the enforcement of federal laws of the Russian Federation
    No. 139-FZ, No. 187-FZ, No. 398-FZ and a number of others that regulate
    the dissemination of information on the Internet.

    Where proxies are found to be blocked they will be rotated.
    """
    short_name = "block_roskomsvoboda"
    description = "Import Russian blocklist from RosKomSvoboda"
    frequency = 90

    def automate(self, full: bool = False) -> Tuple[bool, str]:
        activities = []
        proxies: List[Proxy] = Proxy.query.filter(
            Proxy.deprecated.is_(None),
            Proxy.destroyed.is_(None)
        ).all()
        patterns = requests.get("https://reestr.rublacklist.net/api/v2/domains/json").json()
        for pattern in patterns:
            for proxy in proxies:
                if proxy.url is None:
                    # Not ready yet
                    continue
                if fnmatch(proxy.url[len("https://"):], pattern):
                    print(f"Found {proxy.url} blocked")
                    if not proxy.origin.auto_rotation:
                        print("Proxy auto-rotation forbidden for origin")
                        continue
                    proxy.deprecate(reason="roskomsvoboda")
                    activities.append(Activity(
                        activity_type="block",
                        text=(f"Proxy {proxy.url} for {proxy.origin.domain_name} detected blocked "
                              "according to RosKomSvoboda. Rotation scheduled.")
                    ))
        for activity in activities:
            db.session.add(activity)
        db.session.commit()
        for activity in activities:
            activity.notify()
        return True, ""