diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39c9d8b..ac5ccbd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,44 +2,51 @@ image: python:3.8-bullseye test:bandit: script: - - cp config.yaml.example config.yaml - - apt update && apt install build-essential - - pip install -r requirements.txt --quiet - - pip install bandit --quiet - - bandit -r app - -test:mypy: - script: - - cp config.yaml.example config.yaml - - apt update && apt install build-essential - - pip install -r requirements.txt --quiet - - pip install -r requirements-types.txt --quiet - - mypy app + - cp config.yaml.example config.yaml + - apt update && apt install build-essential + - pip install -r requirements.txt --quiet + - pip install bandit --quiet + - bandit -r app test:docs: stage: test script: - - cp config.yaml.example config.yaml - - apt update && apt install build-essential - - pip install -r requirements.txt - - pip install -U sphinx sphinx-press-theme sphinx-jsonschema - - pushd scripts && python update_schemas.py && popd - - pushd docs && sphinx-build -b html . ../public && popd + - cp config.yaml.example config.yaml + - apt update && apt install build-essential + - pip install -r requirements.txt + - pip install -U sphinx sphinx-press-theme sphinx-jsonschema + - pushd scripts && python update_schemas.py && popd + - pushd docs && sphinx-build -b html . ../public && popd rules: - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH - + +test:flake8: + script: + - cp config.yaml.example config.yaml + - apt update && apt install build-essential + - pip install -r requirements.txt --quiet + - pip install flake8 --quiet + - flake8 app + +test:mypy: + script: + - cp config.yaml.example config.yaml + - apt update && apt install build-essential + - pip install -r requirements.txt --quiet + - pip install -r requirements-types.txt --quiet + - mypy app + pages: stage: deploy script: - - cp config.yaml.example config.yaml - - apt update && apt install build-essential - - pip install -r requirements.txt - - pip install -U sphinx sphinx-press-theme sphinx-jsonschema - - pushd scripts && python update_schemas.py && popd - - pushd docs && sphinx-build -b html . ../public && popd + - cp config.yaml.example config.yaml + - apt update && apt install build-essential + - pip install -r requirements.txt + - pip install -U sphinx sphinx-press-theme sphinx-jsonschema + - pushd scripts && python update_schemas.py && popd + - pushd docs && sphinx-build -b html . ../public && popd artifacts: paths: - public rules: - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH - diff --git a/app/__init__.py b/app/__init__.py index c6932f1..ada98d7 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -5,9 +5,6 @@ import yaml from app.extensions import db from app.extensions import migrate from app.extensions import bootstrap -from app.lists.bc2 import mirror_sites -from app.models.mirrors import Origin, Proxy, Mirror -from app.models.base import Group from app.portal import portal app = Flask(__name__) diff --git a/app/lists/bc2.py b/app/lists/bc2.py index 2991135..bd36f98 100644 --- a/app/lists/bc2.py +++ b/app/lists/bc2.py @@ -4,7 +4,7 @@ from typing import List, Dict, Union from pydantic import BaseModel, Field -from app.models.mirrors import Origin +from app.models.mirrors import Origin, Proxy, Mirror class BC2Alternative(BaseModel): @@ -18,52 +18,48 @@ class BC2Alternative(BaseModel): class BC2Site(BaseModel): main_domain: str = Field( description="The main domain name of the website, excluding \"www.\" if present.", - examples=["bbc.co.uk", "bbc.com", "guardianproject.info"] - ) + examples=["bbc.co.uk", "bbc.com", "guardianproject.info"]) available_alternatives: List[BC2Alternative] class BypassCensorship2(BaseModel): - version: str = Field( - description="Version number of the Bypass Censorship Extension schema in use", - ) + version: str = Field(description="Version number of the Bypass Censorship Extension schema in use", ) sites: List[BC2Site] class Config: title = "Bypass Censorship Version 2" -def mirror_sites(provider: str = "cloudfront") -> Dict[ - str, Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]: +def mirror_alternative(mirror: Mirror): return { - "version": "2.0", - "sites": [{ - "main_domain": x.description[len("proxy:"):].replace("www.", "") if x.description.startswith( - "proxy:") else x.domain_name.replace("www.", ""), - "available_alternatives": [ - { - "proto": "tor" if ".onion" in a.url else "https", - "type": "eotk" if ".onion" in a.url else "mirror", - "created_at": str(a.added), - "updated_at": str(a.updated), - "url": a.url - } for a in x.mirrors if not a.deprecated and not a.destroyed - ] + [ - { - "proto": "https", - "type": "mirror", - "created_at": str(a.added), - "updated_at": str(a.updated), - "url": a.url - } for a in x.proxies if - a.url is not None - and not a.deprecated - and not a.destroyed - and a.provider == provider - ]} for x in Origin.query.order_by(Origin.domain_name).all() if x.destroyed is None - ] + "proto": "tor" if ".onion" in mirror.url else "https", + "type": "eotk" if ".onion" in mirror.url else "mirror", + "created_at": str(mirror.added), + "updated_at": str(mirror.updated), + "url": mirror.url } +def proxy_alternative(proxy: Proxy): + return { + "proto": "https", + "type": "mirror", + "created_at": str(proxy.added), + "updated_at": str(proxy.updated), + "url": proxy.url + } + + +def mirror_sites(provider: str = "cloudfront") -> Dict[str, + Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]: + return {"version": "2.0", "sites": [{ + "main_domain": x.description[len("proxy:"):].replace("www.", "") if x.description.startswith( + "proxy:") else x.domain_name.replace("www.", ""), + "available_alternatives": [mirror_alternative(a) for a in x.mirrors if not a.deprecated and not a.destroyed] + [ + proxy_alternative(a) for a in x.proxies if + a.url is not None and not a.deprecated and not a.destroyed and a.provider == provider]} for x in + Origin.query.order_by(Origin.domain_name).all() if x.destroyed is None]} + + if getattr(builtins, "__sphinx_build__", False): schema = BypassCensorship2.schema_json() diff --git a/app/lists/bridgelines.py b/app/lists/bridgelines.py index e4c0852..e4d071b 100644 --- a/app/lists/bridgelines.py +++ b/app/lists/bridgelines.py @@ -28,9 +28,9 @@ class Bridgelines(BaseModel): def bridgelines(*, distribution_method: Optional[str] = None) -> Dict[str, Any]: bridges: Iterable[Bridge] = Bridge.query.filter( - Bridge.destroyed == None, - Bridge.deprecated == None, - Bridge.bridgeline != None + Bridge.destroyed.is_(None), + Bridge.deprecated.is_(None), + Bridge.bridgeline.is_not(None) ).all() if distribution_method is not None: bridges = [b for b in bridges diff --git a/app/lists/mirror_mapping.py b/app/lists/mirror_mapping.py index c47da4a..845f79e 100644 --- a/app/lists/mirror_mapping.py +++ b/app/lists/mirror_mapping.py @@ -46,7 +46,7 @@ def mirror_mapping() -> Dict[str, Union[str, Dict[str, str]]]: }, s3_buckets=[ f"{app.config['GLOBAL_NAMESPACE']}-{g.group_name.lower()}-logs-cloudfront" - for g in Group.query.filter(Group.destroyed == None).all() + for g in Group.query.filter(Group.destroyed.is_(None)).all() ] ).dict() diff --git a/app/models/activity.py b/app/models/activity.py index 6e0bdcc..059d0bf 100644 --- a/app/models/activity.py +++ b/app/models/activity.py @@ -37,7 +37,7 @@ class Activity(db.Model): # type: ignore def notify(self) -> int: count = 0 hooks = Webhook.query.filter( - Webhook.destroyed == None + Webhook.destroyed.is_(None) ) for hook in hooks: hook.send(self.text) @@ -55,4 +55,4 @@ class Webhook(AbstractConfiguration): else: # Matrix as default data = {"body": text} - r = requests.post(self.url, json=data) + requests.post(self.url, json=data) diff --git a/app/portal/__init__.py b/app/portal/__init__.py index 883a5c2..f2537ea 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -10,7 +10,6 @@ from app.models.alarms import Alarm, AlarmState from app.models.bridges import Bridge from app.models.mirrors import Origin, Proxy from app.models.base import Group -from app.portal.list import NewMirrorListForm from app.portal.automation import bp as automation from app.portal.bridgeconf import bp as bridgeconf from app.portal.bridge import bp as bridge @@ -53,7 +52,7 @@ def format_datetime(s: Optional[datetime]) -> str: def total_origins_blocked() -> int: count = 0 - for o in Origin.query.filter(Origin.destroyed == None).all(): + for o in Origin.query.filter(Origin.destroyed.is_(None)).all(): for a in o.alarms: if a.alarm_type.startswith("origin-block-ooni-"): if a.alarm_state == AlarmState.WARNING: @@ -66,7 +65,7 @@ def total_origins_blocked() -> int: def portal_home() -> ResponseReturnValue: groups = Group.query.order_by(Group.group_name).all() now = datetime.now(timezone.utc) - proxies = Proxy.query.filter(Proxy.destroyed == None).all() + proxies = Proxy.query.filter(Proxy.destroyed.is_(None)).all() last24 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=1))).all()) last72 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=3))).all()) lastweek = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=7))).all()) @@ -74,15 +73,15 @@ def portal_home() -> ResponseReturnValue: s: len(Alarm.query.filter(Alarm.alarm_state == s.upper(), Alarm.last_updated > (now - timedelta(days=1))).all()) for s in ["critical", "warning", "ok", "unknown"] } - bridges = Bridge.query.filter(Bridge.destroyed == None).all() + bridges = Bridge.query.filter(Bridge.destroyed.is_(None)).all() br_last = { d: len(Bridge.query.filter(Bridge.deprecated > (now - timedelta(days=d))).all()) for d in [1, 3, 7] } activity = Activity.query.filter(Activity.added > (now - timedelta(days=2))).order_by(desc(Activity.added)).all() - onionified = len([o for o in Origin.query.filter(Origin.destroyed == None).all() if o.onion() != None]) + onionified = len([o for o in Origin.query.filter(Origin.destroyed.is_(None)).all() if o.onion() is not None]) ooni_blocked = total_origins_blocked() - total_origins = len(Origin.query.filter(Origin.destroyed == None).all()) + total_origins = len(Origin.query.filter(Origin.destroyed.is_(None)).all()) return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72, lastweek=lastweek, proxies=proxies, **alarms, activity=activity, total_origins=total_origins, onionified=onionified, br_last=br_last, ooni_blocked=ooni_blocked, bridges=bridges) @@ -91,7 +90,7 @@ def portal_home() -> ResponseReturnValue: @portal.route("/search") def search() -> ResponseReturnValue: query = request.args.get("query") - proxies = Proxy.query.filter(or_(Proxy.url.contains(query)), Proxy.destroyed == None).all() + proxies = Proxy.query.filter(or_(Proxy.url.contains(query)), Proxy.destroyed.is_(None)).all() origins = Origin.query.filter(or_(Origin.description.contains(query), Origin.domain_name.contains(query))).all() return render_template("search.html.j2", section="home", proxies=proxies, origins=origins) diff --git a/app/portal/automation.py b/app/portal/automation.py index 8cfca85..819c2fe 100644 --- a/app/portal/automation.py +++ b/app/portal/automation.py @@ -22,7 +22,7 @@ class EditAutomationForm(FlaskForm): # type: ignore @bp.route("/list") def automation_list() -> ResponseReturnValue: automations = Automation.query.filter( - Automation.destroyed == None).order_by(Automation.description).all() + Automation.destroyed.is_(None)).order_by(Automation.description).all() return render_template("list.html.j2", section="automation", title="Automation Jobs", @@ -57,11 +57,11 @@ def automation_edit(automation_id: int) -> ResponseReturnValue: def automation_kick(automation_id: int) -> ResponseReturnValue: automation = Automation.query.filter( Automation.id == automation_id, - Automation.destroyed == None).first() + Automation.destroyed.is_(None)).first() if automation is None: return response_404("The requested bridge configuration could not be found.") return view_lifecycle( - header=f"Kick automation timer?", + header="Kick automation timer?", message=automation.description, success_view="portal.automation.automation_list", success_message="This automation job will next run within 1 minute.", diff --git a/app/portal/bridge.py b/app/portal/bridge.py index 399ec3d..c6a20fe 100644 --- a/app/portal/bridge.py +++ b/app/portal/bridge.py @@ -12,7 +12,7 @@ bp = Blueprint("bridge", __name__) @bp.route("/list") def bridge_list() -> ResponseReturnValue: - bridges = Bridge.query.filter(Bridge.destroyed == None).all() + bridges = Bridge.query.filter(Bridge.destroyed.is_(None)).all() return render_template("list.html.j2", section="bridge", title="Tor Bridges", @@ -22,7 +22,7 @@ def bridge_list() -> ResponseReturnValue: @bp.route("/block/", methods=['GET', 'POST']) def bridge_blocked(bridge_id: int) -> ResponseReturnValue: - bridge: Optional[Bridge] = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first() + bridge: Optional[Bridge] = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed.is_(None)).first() if bridge is None: return Response(render_template("error.html.j2", header="404 Proxy Not Found", diff --git a/app/portal/bridgeconf.py b/app/portal/bridgeconf.py index 291a1ca..d424cf4 100644 --- a/app/portal/bridgeconf.py +++ b/app/portal/bridgeconf.py @@ -33,7 +33,7 @@ class EditBridgeConfForm(FlaskForm): # type: ignore @bp.route("/list") def bridgeconf_list() -> ResponseReturnValue: - bridgeconfs: List[BridgeConf] = BridgeConf.query.filter(BridgeConf.destroyed == None).all() + bridgeconfs: List[BridgeConf] = BridgeConf.query.filter(BridgeConf.destroyed.is_(None)).all() return render_template("list.html.j2", section="bridgeconf", title="Tor Bridge Configurations", @@ -110,11 +110,11 @@ def bridgeconf_edit(bridgeconf_id: int) -> ResponseReturnValue: @bp.route("/destroy/", methods=['GET', 'POST']) def bridgeconf_destroy(bridgeconf_id: int) -> ResponseReturnValue: - bridgeconf = BridgeConf.query.filter(BridgeConf.id == bridgeconf_id, BridgeConf.destroyed == None).first() + bridgeconf = BridgeConf.query.filter(BridgeConf.id == bridgeconf_id, BridgeConf.destroyed.is_(None)).first() if bridgeconf is None: return response_404("The requested bridge configuration could not be found.") return view_lifecycle( - header=f"Destroy bridge configuration?", + header="Destroy bridge configuration?", message=bridgeconf.description, success_view="portal.bridgeconf.bridgeconf_list", success_message="All bridges from the destroyed configuration will shortly be destroyed at their providers.", diff --git a/app/portal/eotk.py b/app/portal/eotk.py index 86f126c..2e20464 100644 --- a/app/portal/eotk.py +++ b/app/portal/eotk.py @@ -10,7 +10,7 @@ bp = Blueprint("eotk", __name__) @bp.route("/list") def eotk_list() -> ResponseReturnValue: - instances = Eotk.query.filter(Eotk.destroyed == None).order_by(desc(Eotk.added)).all() + instances = Eotk.query.filter(Eotk.destroyed.is_(None)).order_by(desc(Eotk.added)).all() return render_template("list.html.j2", section="eotk", title="EOTK Instances", diff --git a/app/portal/list.py b/app/portal/list.py index a2357f7..1f89f4a 100644 --- a/app/portal/list.py +++ b/app/portal/list.py @@ -31,7 +31,7 @@ def list_format_name(s: str) -> str: @bp.route('/list') def list_list() -> ResponseReturnValue: - lists = MirrorList.query.filter(MirrorList.destroyed == None).all() + lists = MirrorList.query.filter(MirrorList.destroyed.is_(None)).all() return render_template("list.html.j2", section="list", title="Mirror Lists", @@ -62,11 +62,11 @@ def list_preview(format_: str) -> ResponseReturnValue: @bp.route("/destroy/", methods=['GET', 'POST']) def list_destroy(list_id: int) -> ResponseReturnValue: - list_ = MirrorList.query.filter(MirrorList.id == list_id, MirrorList.destroyed == None).first() + list_ = MirrorList.query.filter(MirrorList.id == list_id, MirrorList.destroyed.is_(None)).first() if list_ is None: return response_404("The requested bridge configuration could not be found.") return view_lifecycle( - header=f"Destroy mirror list?", + header="Destroy mirror list?", message=list_.description, success_view="portal.list.list_list", success_message="This list will no longer be updated and may be deleted depending on the provider.", @@ -96,7 +96,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue: try: db.session.add(list_) db.session.commit() - flash(f"Created new mirror list.", "success") + flash("Created new mirror list.", "success") return redirect(url_for("portal.list.list_list")) except exc.SQLAlchemyError as e: print(e) diff --git a/app/portal/onion.py b/app/portal/onion.py index 66b3c56..77b9dc3 100644 --- a/app/portal/onion.py +++ b/app/portal/onion.py @@ -97,7 +97,7 @@ def onion_list() -> ResponseReturnValue: @bp.route("/destroy/", methods=['GET', 'POST']) def onion_destroy(onion_id: int) -> ResponseReturnValue: - onion = Onion.query.filter(Onion.id == onion_id, Onion.destroyed == None).first() + onion = Onion.query.filter(Onion.id == onion_id, Onion.destroyed.is_(None)).first() if onion is None: return response_404("The requested onion service could not be found.") return view_lifecycle( diff --git a/app/portal/origin.py b/app/portal/origin.py index 7a4b257..67182a1 100644 --- a/app/portal/origin.py +++ b/app/portal/origin.py @@ -115,7 +115,7 @@ def origin_onion() -> ResponseReturnValue: @bp.route("/destroy/", methods=['GET', 'POST']) def origin_destroy(origin_id: int) -> ResponseReturnValue: - origin = Origin.query.filter(Origin.id == origin_id, Origin.destroyed == None).first() + origin = Origin.query.filter(Origin.id == origin_id, Origin.destroyed.is_(None)).first() if origin is None: return response_404("The requested origin could not be found.") return view_lifecycle( diff --git a/app/portal/proxy.py b/app/portal/proxy.py index ab4c9fb..0b729b2 100644 --- a/app/portal/proxy.py +++ b/app/portal/proxy.py @@ -11,7 +11,7 @@ bp = Blueprint("proxy", __name__) @bp.route("/list") def proxy_list() -> ResponseReturnValue: - proxies = Proxy.query.filter(Proxy.destroyed == None).order_by(desc(Proxy.added)).all() + proxies = Proxy.query.filter(Proxy.destroyed.is_(None)).order_by(desc(Proxy.added)).all() return render_template("list.html.j2", section="proxy", title="Proxies", @@ -21,7 +21,7 @@ def proxy_list() -> ResponseReturnValue: @bp.route("/block/", methods=['GET', 'POST']) def proxy_block(proxy_id: int) -> ResponseReturnValue: - proxy = Proxy.query.filter(Proxy.id == proxy_id, Proxy.destroyed == None).first() + proxy = Proxy.query.filter(Proxy.id == proxy_id, Proxy.destroyed.is_(None)).first() if proxy is None: return Response(render_template("error.html.j2", header="404 Proxy Not Found", diff --git a/app/portal/webhook.py b/app/portal/webhook.py index 57f50b0..2386f9b 100644 --- a/app/portal/webhook.py +++ b/app/portal/webhook.py @@ -48,7 +48,7 @@ def webhook_new() -> ResponseReturnValue: db.session.commit() flash(f"Created new webhook {webhook.url}.", "success") return redirect(url_for("portal.webhook.webhook_edit", webhook_id=webhook.id)) - except exc.SQLAlchemyError as e: + except exc.SQLAlchemyError: flash("Failed to create new webhook.", "danger") return redirect(url_for("portal.webhook.webhook_list")) return render_template("new.html.j2", section="webhook", form=form) @@ -95,7 +95,7 @@ def webhook_list() -> ResponseReturnValue: @bp.route("/destroy/", methods=['GET', 'POST']) def webhook_destroy(webhook_id: int) -> ResponseReturnValue: - webhook: Optional[Webhook] = Webhook.query.filter(Webhook.id == webhook_id, Webhook.destroyed == None).first() + webhook: Optional[Webhook] = Webhook.query.filter(Webhook.id == webhook_id, Webhook.destroyed.is_(None)).first() if webhook is None: return response_404("The requested webhook could not be found.") return view_lifecycle( diff --git a/app/terraform/alarms/proxy_azure_cdn.py b/app/terraform/alarms/proxy_azure_cdn.py index 9f8325f..22e32ff 100644 --- a/app/terraform/alarms/proxy_azure_cdn.py +++ b/app/terraform/alarms/proxy_azure_cdn.py @@ -28,11 +28,11 @@ class AlarmProxyAzureCdnAutomation(BaseAutomation): if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"] for proxy in Proxy.query.filter( Proxy.provider == "azure_cdn", - Proxy.destroyed == None + Proxy.destroyed.is_(None) ): alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high") if proxy.origin.group.group_name.lower() not in firing: alarm.update_state(AlarmState.OK, "Azure monitor alert not firing") else: alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing") - return True, "" \ No newline at end of file + return True, "" diff --git a/app/terraform/alarms/proxy_cloudfront.py b/app/terraform/alarms/proxy_cloudfront.py index 05b0881..fe4e182 100644 --- a/app/terraform/alarms/proxy_cloudfront.py +++ b/app/terraform/alarms/proxy_cloudfront.py @@ -47,7 +47,7 @@ class AlarmProxyCloudfrontAutomation(BaseAutomation): db.session.add(alarm) alarm.last_updated = datetime.datetime.utcnow() deployed_count = len(Proxy.query.filter( - Proxy.destroyed == None).all()) + Proxy.destroyed.is_(None)).all()) old_state = alarm.alarm_state if deployed_count > 370: alarm.alarm_state = AlarmState.CRITICAL diff --git a/app/terraform/alarms/proxy_http_status.py b/app/terraform/alarms/proxy_http_status.py index a216456..3f61016 100644 --- a/app/terraform/alarms/proxy_http_status.py +++ b/app/terraform/alarms/proxy_http_status.py @@ -29,7 +29,7 @@ class AlarmProxyHTTPStatusAutomation(BaseAutomation): def automate(self, full: bool = False) -> Tuple[bool, str]: proxies = Proxy.query.filter( - Proxy.destroyed == None + Proxy.destroyed.is_(None) ) for proxy in proxies: try: diff --git a/app/terraform/block_ooni.py b/app/terraform/block_ooni.py index b815065..1fa20c9 100644 --- a/app/terraform/block_ooni.py +++ b/app/terraform/block_ooni.py @@ -1,7 +1,7 @@ from collections import defaultdict from datetime import datetime from datetime import timedelta -from typing import Dict, Tuple, Union, Any +from typing import Dict, Tuple, Any import requests @@ -78,7 +78,7 @@ class BlockOONIAutomation(BaseAutomation): frequency = 240 def automate(self, full: bool = False) -> Tuple[bool, str]: - origins = Origin.query.filter(Origin.destroyed == None).all() + origins = Origin.query.filter(Origin.destroyed.is_(None)).all() for origin in origins: ooni = threshold_origin(origin.domain_name) for country in ooni: diff --git a/app/terraform/block_roskomsvoboda.py b/app/terraform/block_roskomsvoboda.py index 8f8b65a..4576b99 100644 --- a/app/terraform/block_roskomsvoboda.py +++ b/app/terraform/block_roskomsvoboda.py @@ -17,8 +17,8 @@ class BlockRoskomsvobodaAutomation(BaseAutomation): def automate(self, full: bool = False) -> Tuple[bool, str]: activities = [] proxies: List[Proxy] = Proxy.query.filter( - Proxy.deprecated == None, - Proxy.destroyed == None + Proxy.deprecated.is_(None), + Proxy.destroyed.is_(None) ).all() patterns = requests.get("https://reestr.rublacklist.net/api/v2/domains/json").json() for pattern in patterns: diff --git a/app/terraform/bridge/__init__.py b/app/terraform/bridge/__init__.py index 6435237..4e36b4e 100644 --- a/app/terraform/bridge/__init__.py +++ b/app/terraform/bridge/__init__.py @@ -23,12 +23,12 @@ class BridgeAutomation(TerraformAutomation): def create_missing(self) -> None: bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter( BridgeConf.provider == self.provider, - BridgeConf.destroyed == None + BridgeConf.destroyed.is_(None) ).all() for bridgeconf in bridgeconfs: active_bridges = Bridge.query.filter( Bridge.conf_id == bridgeconf.id, - Bridge.deprecated == None + Bridge.deprecated.is_(None) ).all() if len(active_bridges) < bridgeconf.number: for i in range(bridgeconf.number - len(active_bridges)): @@ -49,7 +49,7 @@ class BridgeAutomation(TerraformAutomation): def destroy_expired(self) -> None: cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=0) bridges = [b for b in Bridge.query.filter( - Bridge.destroyed == None, + Bridge.destroyed.is_(None), Bridge.deprecated < cutoff ).all() if b.conf.provider == self.provider] for bridge in bridges: @@ -66,7 +66,7 @@ class BridgeAutomation(TerraformAutomation): self.template, groups=Group.query.all(), bridgeconfs=BridgeConf.query.filter( - BridgeConf.destroyed == None, + BridgeConf.destroyed.is_(None), BridgeConf.provider == self.provider ).all(), global_namespace=app.config['GLOBAL_NAMESPACE'], diff --git a/app/terraform/bridge/aws.py b/app/terraform/bridge/aws.py index e9509d5..65f16d8 100644 --- a/app/terraform/bridge/aws.py +++ b/app/terraform/bridge/aws.py @@ -20,17 +20,17 @@ class BridgeAWSAutomation(BridgeAutomation): } } } - + provider "aws" { access_key = "{{ aws_access_key }}" secret_key = "{{ aws_secret_key }}" region = "us-east-1" } - + locals { ssh_key = file("{{ ssh_public_key_path }}") } - + {% for group in groups %} module "label_{{ group.id }}" { source = "cloudposse/label/null" @@ -40,7 +40,7 @@ class BridgeAWSAutomation(BridgeAutomation): label_order = ["namespace", "tenant", "name", "attributes"] } {% endfor %} - + {% for bridgeconf in bridgeconfs %} {% for bridge in bridgeconf.bridges %} {% if not bridge.destroyed %} @@ -54,11 +54,11 @@ class BridgeAWSAutomation(BridgeAutomation): attributes = ["{{ bridge.id }}"] distribution_method = "{{ bridge.conf.method }}" } - + output "bridge_hashed_fingerprint_{{ bridge.id }}" { value = module.bridge_{{ bridge.id }}.hashed_fingerprint } - + output "bridge_bridgeline_{{ bridge.id }}" { value = module.bridge_{{ bridge.id }}.bridgeline sensitive = true diff --git a/app/terraform/bridge/ovh.py b/app/terraform/bridge/ovh.py index 8e7e07c..323f8e9 100644 --- a/app/terraform/bridge/ovh.py +++ b/app/terraform/bridge/ovh.py @@ -34,7 +34,7 @@ class BridgeOvhAutomation(BridgeAutomation): } } } - + provider "openstack" { auth_url = "https://auth.cloud.ovh.net/v3/" domain_name = "Default" # Domain name - Always at 'default' for OVHcloud @@ -42,24 +42,24 @@ class BridgeOvhAutomation(BridgeAutomation): password = "{{ ovh_openstack_password }}" tenant_id = "{{ ovh_openstack_tenant_id }}" } - + provider "ovh" { endpoint = "ovh-eu" application_key = "{{ ovh_cloud_application_key }}" application_secret = "{{ ovh_cloud_application_secret }}" consumer_key = "{{ ovh_cloud_consumer_key }}" } - + locals { public_ssh_key = file("{{ ssh_public_key_path }}") private_ssh_key = file("{{ ssh_private_key_path }}") } - + data "ovh_cloud_project_regions" "regions" { service_name = "{{ ovh_openstack_tenant_id }}" has_services_up = ["instance"] } - + {% for group in groups %} module "label_{{ group.id }}" { source = "cloudposse/label/null" @@ -69,19 +69,19 @@ class BridgeOvhAutomation(BridgeAutomation): label_order = ["namespace", "tenant", "name", "attributes"] } {% endfor %} - + {% for bridgeconf in bridgeconfs %} {% for bridge in bridgeconf.bridges %} {% if not bridge.destroyed %} resource "random_shuffle" "region_{{ bridge.id }}" { input = data.ovh_cloud_project_regions.regions.names result_count = 1 - + lifecycle { ignore_changes = [input] # don't replace all the bridges if a new region appears } } - + module "bridge_{{ bridge.id }}" { source = "sr2c/tor-bridge/openstack" version = "0.0.7" @@ -94,11 +94,11 @@ class BridgeOvhAutomation(BridgeAutomation): contact_info = "hi" distribution_method = "{{ bridge.conf.method }}" } - + output "bridge_hashed_fingerprint_{{ bridge.id }}" { value = module.bridge_{{ bridge.id }}.hashed_fingerprint } - + output "bridge_bridgeline_{{ bridge.id }}" { value = module.bridge_{{ bridge.id }}.bridgeline sensitive = true diff --git a/app/terraform/eotk/aws.py b/app/terraform/eotk/aws.py index 633189e..14c005a 100644 --- a/app/terraform/eotk/aws.py +++ b/app/terraform/eotk/aws.py @@ -15,7 +15,7 @@ def update_eotk_instance(group_id: int, Eotk.group_id == group_id, Eotk.region == region, Eotk.provider == "aws", - Eotk.destroyed == None + Eotk.destroyed.is_(None) ).first() if instance is None: instance = Eotk() @@ -45,20 +45,20 @@ class EotkAWSAutomation(TerraformAutomation): } } } - + provider "aws" { access_key = "{{ aws_access_key }}" secret_key = "{{ aws_secret_key }}" region = "us-east-2" } - + provider "aws" { access_key = "{{ aws_access_key }}" secret_key = "{{ aws_secret_key }}" region = "eu-central-1" alias = "second_region" } - + {% for group in groups %} module "eotk_{{ group.id }}" { providers = { @@ -80,8 +80,8 @@ class EotkAWSAutomation(TerraformAutomation): self.tf_write( self.template, groups=Group.query.filter( - Group.eotk == True, - Group.destroyed == None + Group.eotk.is_(True), + Group.destroyed.is_(None) ).all(), global_namespace=app.config['GLOBAL_NAMESPACE'], **{ diff --git a/app/terraform/list/__init__.py b/app/terraform/list/__init__.py index 7dc21a0..063c7bf 100644 --- a/app/terraform/list/__init__.py +++ b/app/terraform/list/__init__.py @@ -25,7 +25,7 @@ class ListAutomation(TerraformAutomation): self.tf_write( self.template, lists=MirrorList.query.filter( - MirrorList.destroyed == None, + MirrorList.destroyed.is_(None), MirrorList.provider == self.provider, ).all(), global_namespace=app.config['GLOBAL_NAMESPACE'], diff --git a/app/terraform/list/gitlab.py b/app/terraform/list/gitlab.py index 924cc22..ee3572c 100644 --- a/app/terraform/list/gitlab.py +++ b/app/terraform/list/gitlab.py @@ -22,16 +22,16 @@ class ListGitlabAutomation(ListAutomation): } } } - + provider "gitlab" { token = "{{ gitlab_token }}" } - + {% for list in lists %} data "gitlab_project" "project_{{ list.id }}" { id = "{{ list.container }}" } - + resource "gitlab_repository_file" "file_{{ list.id }}" { project = data.gitlab_project.project_{{ list.id }}.id file_path = "{{ list.filename }}" @@ -41,6 +41,6 @@ class ListGitlabAutomation(ListAutomation): author_name = "{{ gitlab_author_name }}" commit_message = "{{ gitlab_commit_message }}" } - + {% endfor %} """ diff --git a/app/terraform/proxy/__init__.py b/app/terraform/proxy/__init__.py index 9c653c8..63abc77 100644 --- a/app/terraform/proxy/__init__.py +++ b/app/terraform/proxy/__init__.py @@ -83,8 +83,8 @@ class ProxyAutomation(TerraformAutomation): def deprecate_orphaned_proxies(self) -> None: proxies = Proxy.query.filter( - Proxy.deprecated == None, - Proxy.destroyed == None, + Proxy.deprecated.is_(None), + Proxy.destroyed.is_(None), Proxy.provider == self.provider ).all() for proxy in proxies: @@ -95,7 +95,7 @@ class ProxyAutomation(TerraformAutomation): def destroy_expired_proxies(self) -> None: cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3) proxies = Proxy.query.filter( - Proxy.destroyed == None, + Proxy.destroyed.is_(None), Proxy.provider == self.provider, Proxy.deprecated < cutoff ).all() @@ -123,7 +123,7 @@ class ProxyAutomation(TerraformAutomation): groups=Group.query.all(), proxies=Proxy.query.filter( Proxy.provider == self.provider, - Proxy.destroyed == None + Proxy.destroyed.is_(None) ).all(), subgroups=self.get_subgroups(), global_namespace=app.config['GLOBAL_NAMESPACE'], diff --git a/app/terraform/proxy/azure_cdn.py b/app/terraform/proxy/azure_cdn.py index b43e7f9..f3fabed 100644 --- a/app/terraform/proxy/azure_cdn.py +++ b/app/terraform/proxy/azure_cdn.py @@ -31,21 +31,20 @@ class ProxyAzureCdnAutomation(ProxyAutomation): } } } - + provider "azurerm" { features {} - client_id = "{{ azure_client_id }}" client_secret = "{{ azure_client_secret }}" subscription_id = "{{ azure_subscription_id }}" tenant_id = "{{ azure_tenant_id }}" skip_provider_registration = true } - + data "azurerm_resource_group" "this" { name = "{{ azure_resource_group_name }}" } - + resource "azurerm_storage_account" "this" { name = "{{ azure_storage_account_name }}" resource_group_name = data.azurerm_resource_group.this.name @@ -53,7 +52,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation): account_tier = "Standard" account_replication_type = "RAGRS" } - + {% for group in groups %} module "label_{{ group.id }}" { source = "cloudposse/label/null" @@ -62,49 +61,48 @@ class ProxyAzureCdnAutomation(ProxyAutomation): tenant = "{{ group.group_name }}" label_order = ["namespace", "tenant", "name", "attributes"] } - + {% for subgroup in subgroups[group.id] %} resource "azurerm_cdn_profile" "profile_{{ group.id }}_{{ subgroup }}" { name = "${module.label_{{ group.id }}.id}-sub{{ subgroup }}" location = "{{ azure_location }}" resource_group_name = data.azurerm_resource_group.this.name sku = "Standard_Microsoft" - - tags = module.label_{{ group.id }}.tags + tags = module.label_{{ group.id }}.tags } - + resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}_{{ subgroup }}" { name = "cdn-diagnostics" target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id storage_account_id = azurerm_storage_account.this.id - + log { category = "AzureCDNAccessLog" enabled = true - + retention_policy { enabled = true days = 90 } } - + metric { category = "AllMetrics" enabled = true - + retention_policy { enabled = true days = 90 } } } - + resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}_{{ subgroup }}" { name = "bandwidth-out-high-${module.label_{{ group.id }}.id}-sub{{ subgroup }}" resource_group_name = data.azurerm_resource_group.this.name scopes = [azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id] description = "Action will be triggered when response size is too high." - + criteria { metric_namespace = "Microsoft.Cdn/profiles" metric_name = "ResponseSize" @@ -112,26 +110,26 @@ class ProxyAzureCdnAutomation(ProxyAutomation): operator = "GreaterThan" threshold = 21474836481 } - + window_size = "PT1H" } {% endfor %} {% endfor %} - + {% for proxy in proxies %} resource "azurerm_cdn_endpoint" "endpoint_{{ proxy.id }}" { name = "{{ proxy.slug }}" profile_name = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}_{{ proxy.psg }}.name location = "{{ azure_location }}" resource_group_name = data.azurerm_resource_group.this.name - + origin_host_header = "{{ proxy.origin.domain_name }}" - + origin { name = "upstream" host_name = "{{ proxy.origin.domain_name }}" } - + global_delivery_rule { modify_request_header_action { action = "Append" @@ -140,16 +138,16 @@ class ProxyAzureCdnAutomation(ProxyAutomation): } } } - + resource "azurerm_monitor_diagnostic_setting" "diagnostic_{{ proxy.id }}" { name = "cdn-diagnostics" target_resource_id = azurerm_cdn_endpoint.endpoint_{{ proxy.id }}.id storage_account_id = azurerm_storage_account.this.id - + log { category = "CoreAnalytics" enabled = true - + retention_policy { enabled = true days = 90 @@ -162,7 +160,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation): def import_state(self, state: Optional[Any]) -> None: proxies = Proxy.query.filter( Proxy.provider == self.provider, - Proxy.destroyed == None + Proxy.destroyed.is_(None) ).all() for proxy in proxies: proxy.url = f"https://{proxy.slug}.azureedge.net" diff --git a/app/terraform/proxy/cloudfront.py b/app/terraform/proxy/cloudfront.py index f88bcb8..bfc9405 100644 --- a/app/terraform/proxy/cloudfront.py +++ b/app/terraform/proxy/cloudfront.py @@ -24,13 +24,13 @@ class ProxyCloudfrontAutomation(ProxyAutomation): } } } - + provider "aws" { access_key = "{{ aws_access_key }}" secret_key = "{{ aws_secret_key }}" region = "us-east-2" } - + {% for group in groups %} module "label_{{ group.id }}" { source = "cloudposse/label/null" @@ -39,7 +39,7 @@ class ProxyCloudfrontAutomation(ProxyAutomation): tenant = "{{ group.group_name }}" label_order = ["namespace", "tenant", "name", "attributes"] } - + module "log_bucket_{{ group.id }}" { source = "cloudposse/s3-log-storage/aws" version = "0.28.0" @@ -51,12 +51,12 @@ class ProxyCloudfrontAutomation(ProxyAutomation): glacier_transition_days = 60 expiration_days = 90 } - + resource "aws_sns_topic" "alarms_{{ group.id }}" { name = "${module.label_{{ group.id }}.id}-cloudfront-alarms" } {% endfor %} - + {% for proxy in proxies %} module "cloudfront_{{ proxy.id }}" { source = "sr2c/bc-proxy/aws" diff --git a/app/terraform/proxy/fastly.py b/app/terraform/proxy/fastly.py index 7607ed2..75704ea 100644 --- a/app/terraform/proxy/fastly.py +++ b/app/terraform/proxy/fastly.py @@ -61,7 +61,7 @@ module "log_bucket_{{ group.id }}" { {% if group.id == 3 %} resource "fastly_service_vcl" "service_{{ group.id }}" { name = module.label_{{ group.id }}.id - + {% for origin in group.origins %} {% for proxy in origin.proxies %} {% if proxy.destroyed == None and proxy.provider == "fastly" %} @@ -71,7 +71,7 @@ resource "fastly_service_vcl" "service_{{ group.id }}" { } {% endif %} {% endfor %} - + backend { address = "{{ origin.domain_name }}" name = "{{ origin.description }}" @@ -110,7 +110,7 @@ def create_missing_proxies(): def destroy_expired_proxies(): cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3) proxies = Proxy.query.filter( - Proxy.destroyed == None, + Proxy.destroyed.is_(None), Proxy.provider == "fastly", Proxy.deprecated < cutoff ).all() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..e44b810 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +ignore = E501