ci: add flake8
This commit is contained in:
parent
014596d271
commit
dd501a6e4e
32 changed files with 170 additions and 171 deletions
|
@ -2,44 +2,51 @@ image: python:3.8-bullseye
|
||||||
|
|
||||||
test:bandit:
|
test:bandit:
|
||||||
script:
|
script:
|
||||||
- cp config.yaml.example config.yaml
|
- cp config.yaml.example config.yaml
|
||||||
- apt update && apt install build-essential
|
- apt update && apt install build-essential
|
||||||
- pip install -r requirements.txt --quiet
|
- pip install -r requirements.txt --quiet
|
||||||
- pip install bandit --quiet
|
- pip install bandit --quiet
|
||||||
- bandit -r app
|
- bandit -r app
|
||||||
|
|
||||||
test:mypy:
|
|
||||||
script:
|
|
||||||
- cp config.yaml.example config.yaml
|
|
||||||
- apt update && apt install build-essential
|
|
||||||
- pip install -r requirements.txt --quiet
|
|
||||||
- pip install -r requirements-types.txt --quiet
|
|
||||||
- mypy app
|
|
||||||
|
|
||||||
test:docs:
|
test:docs:
|
||||||
stage: test
|
stage: test
|
||||||
script:
|
script:
|
||||||
- cp config.yaml.example config.yaml
|
- cp config.yaml.example config.yaml
|
||||||
- apt update && apt install build-essential
|
- apt update && apt install build-essential
|
||||||
- pip install -r requirements.txt
|
- pip install -r requirements.txt
|
||||||
- pip install -U sphinx sphinx-press-theme sphinx-jsonschema
|
- pip install -U sphinx sphinx-press-theme sphinx-jsonschema
|
||||||
- pushd scripts && python update_schemas.py && popd
|
- pushd scripts && python update_schemas.py && popd
|
||||||
- pushd docs && sphinx-build -b html . ../public && popd
|
- pushd docs && sphinx-build -b html . ../public && popd
|
||||||
rules:
|
rules:
|
||||||
- if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH
|
- if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH
|
||||||
|
|
||||||
|
test:flake8:
|
||||||
|
script:
|
||||||
|
- cp config.yaml.example config.yaml
|
||||||
|
- apt update && apt install build-essential
|
||||||
|
- pip install -r requirements.txt --quiet
|
||||||
|
- pip install flake8 --quiet
|
||||||
|
- flake8 app
|
||||||
|
|
||||||
|
test:mypy:
|
||||||
|
script:
|
||||||
|
- cp config.yaml.example config.yaml
|
||||||
|
- apt update && apt install build-essential
|
||||||
|
- pip install -r requirements.txt --quiet
|
||||||
|
- pip install -r requirements-types.txt --quiet
|
||||||
|
- mypy app
|
||||||
|
|
||||||
pages:
|
pages:
|
||||||
stage: deploy
|
stage: deploy
|
||||||
script:
|
script:
|
||||||
- cp config.yaml.example config.yaml
|
- cp config.yaml.example config.yaml
|
||||||
- apt update && apt install build-essential
|
- apt update && apt install build-essential
|
||||||
- pip install -r requirements.txt
|
- pip install -r requirements.txt
|
||||||
- pip install -U sphinx sphinx-press-theme sphinx-jsonschema
|
- pip install -U sphinx sphinx-press-theme sphinx-jsonschema
|
||||||
- pushd scripts && python update_schemas.py && popd
|
- pushd scripts && python update_schemas.py && popd
|
||||||
- pushd docs && sphinx-build -b html . ../public && popd
|
- pushd docs && sphinx-build -b html . ../public && popd
|
||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- public
|
- public
|
||||||
rules:
|
rules:
|
||||||
- if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
|
- if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
|
||||||
|
|
||||||
|
|
|
@ -5,9 +5,6 @@ import yaml
|
||||||
from app.extensions import db
|
from app.extensions import db
|
||||||
from app.extensions import migrate
|
from app.extensions import migrate
|
||||||
from app.extensions import bootstrap
|
from app.extensions import bootstrap
|
||||||
from app.lists.bc2 import mirror_sites
|
|
||||||
from app.models.mirrors import Origin, Proxy, Mirror
|
|
||||||
from app.models.base import Group
|
|
||||||
from app.portal import portal
|
from app.portal import portal
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
|
|
|
@ -4,7 +4,7 @@ from typing import List, Dict, Union
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from app.models.mirrors import Origin
|
from app.models.mirrors import Origin, Proxy, Mirror
|
||||||
|
|
||||||
|
|
||||||
class BC2Alternative(BaseModel):
|
class BC2Alternative(BaseModel):
|
||||||
|
@ -18,52 +18,48 @@ class BC2Alternative(BaseModel):
|
||||||
class BC2Site(BaseModel):
|
class BC2Site(BaseModel):
|
||||||
main_domain: str = Field(
|
main_domain: str = Field(
|
||||||
description="The main domain name of the website, excluding \"www.\" if present.",
|
description="The main domain name of the website, excluding \"www.\" if present.",
|
||||||
examples=["bbc.co.uk", "bbc.com", "guardianproject.info"]
|
examples=["bbc.co.uk", "bbc.com", "guardianproject.info"])
|
||||||
)
|
|
||||||
available_alternatives: List[BC2Alternative]
|
available_alternatives: List[BC2Alternative]
|
||||||
|
|
||||||
|
|
||||||
class BypassCensorship2(BaseModel):
|
class BypassCensorship2(BaseModel):
|
||||||
version: str = Field(
|
version: str = Field(description="Version number of the Bypass Censorship Extension schema in use", )
|
||||||
description="Version number of the Bypass Censorship Extension schema in use",
|
|
||||||
)
|
|
||||||
sites: List[BC2Site]
|
sites: List[BC2Site]
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
title = "Bypass Censorship Version 2"
|
title = "Bypass Censorship Version 2"
|
||||||
|
|
||||||
|
|
||||||
def mirror_sites(provider: str = "cloudfront") -> Dict[
|
def mirror_alternative(mirror: Mirror):
|
||||||
str, Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]:
|
|
||||||
return {
|
return {
|
||||||
"version": "2.0",
|
"proto": "tor" if ".onion" in mirror.url else "https",
|
||||||
"sites": [{
|
"type": "eotk" if ".onion" in mirror.url else "mirror",
|
||||||
"main_domain": x.description[len("proxy:"):].replace("www.", "") if x.description.startswith(
|
"created_at": str(mirror.added),
|
||||||
"proxy:") else x.domain_name.replace("www.", ""),
|
"updated_at": str(mirror.updated),
|
||||||
"available_alternatives": [
|
"url": mirror.url
|
||||||
{
|
|
||||||
"proto": "tor" if ".onion" in a.url else "https",
|
|
||||||
"type": "eotk" if ".onion" in a.url else "mirror",
|
|
||||||
"created_at": str(a.added),
|
|
||||||
"updated_at": str(a.updated),
|
|
||||||
"url": a.url
|
|
||||||
} for a in x.mirrors if not a.deprecated and not a.destroyed
|
|
||||||
] + [
|
|
||||||
{
|
|
||||||
"proto": "https",
|
|
||||||
"type": "mirror",
|
|
||||||
"created_at": str(a.added),
|
|
||||||
"updated_at": str(a.updated),
|
|
||||||
"url": a.url
|
|
||||||
} for a in x.proxies if
|
|
||||||
a.url is not None
|
|
||||||
and not a.deprecated
|
|
||||||
and not a.destroyed
|
|
||||||
and a.provider == provider
|
|
||||||
]} for x in Origin.query.order_by(Origin.domain_name).all() if x.destroyed is None
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def proxy_alternative(proxy: Proxy):
|
||||||
|
return {
|
||||||
|
"proto": "https",
|
||||||
|
"type": "mirror",
|
||||||
|
"created_at": str(proxy.added),
|
||||||
|
"updated_at": str(proxy.updated),
|
||||||
|
"url": proxy.url
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def mirror_sites(provider: str = "cloudfront") -> Dict[str,
|
||||||
|
Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]:
|
||||||
|
return {"version": "2.0", "sites": [{
|
||||||
|
"main_domain": x.description[len("proxy:"):].replace("www.", "") if x.description.startswith(
|
||||||
|
"proxy:") else x.domain_name.replace("www.", ""),
|
||||||
|
"available_alternatives": [mirror_alternative(a) for a in x.mirrors if not a.deprecated and not a.destroyed] + [
|
||||||
|
proxy_alternative(a) for a in x.proxies if
|
||||||
|
a.url is not None and not a.deprecated and not a.destroyed and a.provider == provider]} for x in
|
||||||
|
Origin.query.order_by(Origin.domain_name).all() if x.destroyed is None]}
|
||||||
|
|
||||||
|
|
||||||
if getattr(builtins, "__sphinx_build__", False):
|
if getattr(builtins, "__sphinx_build__", False):
|
||||||
schema = BypassCensorship2.schema_json()
|
schema = BypassCensorship2.schema_json()
|
||||||
|
|
|
@ -28,9 +28,9 @@ class Bridgelines(BaseModel):
|
||||||
|
|
||||||
def bridgelines(*, distribution_method: Optional[str] = None) -> Dict[str, Any]:
|
def bridgelines(*, distribution_method: Optional[str] = None) -> Dict[str, Any]:
|
||||||
bridges: Iterable[Bridge] = Bridge.query.filter(
|
bridges: Iterable[Bridge] = Bridge.query.filter(
|
||||||
Bridge.destroyed == None,
|
Bridge.destroyed.is_(None),
|
||||||
Bridge.deprecated == None,
|
Bridge.deprecated.is_(None),
|
||||||
Bridge.bridgeline != None
|
Bridge.bridgeline.is_not(None)
|
||||||
).all()
|
).all()
|
||||||
if distribution_method is not None:
|
if distribution_method is not None:
|
||||||
bridges = [b for b in bridges
|
bridges = [b for b in bridges
|
||||||
|
|
|
@ -46,7 +46,7 @@ def mirror_mapping() -> Dict[str, Union[str, Dict[str, str]]]:
|
||||||
},
|
},
|
||||||
s3_buckets=[
|
s3_buckets=[
|
||||||
f"{app.config['GLOBAL_NAMESPACE']}-{g.group_name.lower()}-logs-cloudfront"
|
f"{app.config['GLOBAL_NAMESPACE']}-{g.group_name.lower()}-logs-cloudfront"
|
||||||
for g in Group.query.filter(Group.destroyed == None).all()
|
for g in Group.query.filter(Group.destroyed.is_(None)).all()
|
||||||
]
|
]
|
||||||
).dict()
|
).dict()
|
||||||
|
|
||||||
|
|
|
@ -37,7 +37,7 @@ class Activity(db.Model): # type: ignore
|
||||||
def notify(self) -> int:
|
def notify(self) -> int:
|
||||||
count = 0
|
count = 0
|
||||||
hooks = Webhook.query.filter(
|
hooks = Webhook.query.filter(
|
||||||
Webhook.destroyed == None
|
Webhook.destroyed.is_(None)
|
||||||
)
|
)
|
||||||
for hook in hooks:
|
for hook in hooks:
|
||||||
hook.send(self.text)
|
hook.send(self.text)
|
||||||
|
@ -55,4 +55,4 @@ class Webhook(AbstractConfiguration):
|
||||||
else:
|
else:
|
||||||
# Matrix as default
|
# Matrix as default
|
||||||
data = {"body": text}
|
data = {"body": text}
|
||||||
r = requests.post(self.url, json=data)
|
requests.post(self.url, json=data)
|
||||||
|
|
|
@ -10,7 +10,6 @@ from app.models.alarms import Alarm, AlarmState
|
||||||
from app.models.bridges import Bridge
|
from app.models.bridges import Bridge
|
||||||
from app.models.mirrors import Origin, Proxy
|
from app.models.mirrors import Origin, Proxy
|
||||||
from app.models.base import Group
|
from app.models.base import Group
|
||||||
from app.portal.list import NewMirrorListForm
|
|
||||||
from app.portal.automation import bp as automation
|
from app.portal.automation import bp as automation
|
||||||
from app.portal.bridgeconf import bp as bridgeconf
|
from app.portal.bridgeconf import bp as bridgeconf
|
||||||
from app.portal.bridge import bp as bridge
|
from app.portal.bridge import bp as bridge
|
||||||
|
@ -53,7 +52,7 @@ def format_datetime(s: Optional[datetime]) -> str:
|
||||||
|
|
||||||
def total_origins_blocked() -> int:
|
def total_origins_blocked() -> int:
|
||||||
count = 0
|
count = 0
|
||||||
for o in Origin.query.filter(Origin.destroyed == None).all():
|
for o in Origin.query.filter(Origin.destroyed.is_(None)).all():
|
||||||
for a in o.alarms:
|
for a in o.alarms:
|
||||||
if a.alarm_type.startswith("origin-block-ooni-"):
|
if a.alarm_type.startswith("origin-block-ooni-"):
|
||||||
if a.alarm_state == AlarmState.WARNING:
|
if a.alarm_state == AlarmState.WARNING:
|
||||||
|
@ -66,7 +65,7 @@ def total_origins_blocked() -> int:
|
||||||
def portal_home() -> ResponseReturnValue:
|
def portal_home() -> ResponseReturnValue:
|
||||||
groups = Group.query.order_by(Group.group_name).all()
|
groups = Group.query.order_by(Group.group_name).all()
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
proxies = Proxy.query.filter(Proxy.destroyed == None).all()
|
proxies = Proxy.query.filter(Proxy.destroyed.is_(None)).all()
|
||||||
last24 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=1))).all())
|
last24 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=1))).all())
|
||||||
last72 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=3))).all())
|
last72 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=3))).all())
|
||||||
lastweek = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=7))).all())
|
lastweek = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=7))).all())
|
||||||
|
@ -74,15 +73,15 @@ def portal_home() -> ResponseReturnValue:
|
||||||
s: len(Alarm.query.filter(Alarm.alarm_state == s.upper(), Alarm.last_updated > (now - timedelta(days=1))).all())
|
s: len(Alarm.query.filter(Alarm.alarm_state == s.upper(), Alarm.last_updated > (now - timedelta(days=1))).all())
|
||||||
for s in ["critical", "warning", "ok", "unknown"]
|
for s in ["critical", "warning", "ok", "unknown"]
|
||||||
}
|
}
|
||||||
bridges = Bridge.query.filter(Bridge.destroyed == None).all()
|
bridges = Bridge.query.filter(Bridge.destroyed.is_(None)).all()
|
||||||
br_last = {
|
br_last = {
|
||||||
d: len(Bridge.query.filter(Bridge.deprecated > (now - timedelta(days=d))).all())
|
d: len(Bridge.query.filter(Bridge.deprecated > (now - timedelta(days=d))).all())
|
||||||
for d in [1, 3, 7]
|
for d in [1, 3, 7]
|
||||||
}
|
}
|
||||||
activity = Activity.query.filter(Activity.added > (now - timedelta(days=2))).order_by(desc(Activity.added)).all()
|
activity = Activity.query.filter(Activity.added > (now - timedelta(days=2))).order_by(desc(Activity.added)).all()
|
||||||
onionified = len([o for o in Origin.query.filter(Origin.destroyed == None).all() if o.onion() != None])
|
onionified = len([o for o in Origin.query.filter(Origin.destroyed.is_(None)).all() if o.onion() is not None])
|
||||||
ooni_blocked = total_origins_blocked()
|
ooni_blocked = total_origins_blocked()
|
||||||
total_origins = len(Origin.query.filter(Origin.destroyed == None).all())
|
total_origins = len(Origin.query.filter(Origin.destroyed.is_(None)).all())
|
||||||
return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72,
|
return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72,
|
||||||
lastweek=lastweek, proxies=proxies, **alarms, activity=activity, total_origins=total_origins,
|
lastweek=lastweek, proxies=proxies, **alarms, activity=activity, total_origins=total_origins,
|
||||||
onionified=onionified, br_last=br_last, ooni_blocked=ooni_blocked, bridges=bridges)
|
onionified=onionified, br_last=br_last, ooni_blocked=ooni_blocked, bridges=bridges)
|
||||||
|
@ -91,7 +90,7 @@ def portal_home() -> ResponseReturnValue:
|
||||||
@portal.route("/search")
|
@portal.route("/search")
|
||||||
def search() -> ResponseReturnValue:
|
def search() -> ResponseReturnValue:
|
||||||
query = request.args.get("query")
|
query = request.args.get("query")
|
||||||
proxies = Proxy.query.filter(or_(Proxy.url.contains(query)), Proxy.destroyed == None).all()
|
proxies = Proxy.query.filter(or_(Proxy.url.contains(query)), Proxy.destroyed.is_(None)).all()
|
||||||
origins = Origin.query.filter(or_(Origin.description.contains(query), Origin.domain_name.contains(query))).all()
|
origins = Origin.query.filter(or_(Origin.description.contains(query), Origin.domain_name.contains(query))).all()
|
||||||
return render_template("search.html.j2", section="home", proxies=proxies, origins=origins)
|
return render_template("search.html.j2", section="home", proxies=proxies, origins=origins)
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,7 @@ class EditAutomationForm(FlaskForm): # type: ignore
|
||||||
@bp.route("/list")
|
@bp.route("/list")
|
||||||
def automation_list() -> ResponseReturnValue:
|
def automation_list() -> ResponseReturnValue:
|
||||||
automations = Automation.query.filter(
|
automations = Automation.query.filter(
|
||||||
Automation.destroyed == None).order_by(Automation.description).all()
|
Automation.destroyed.is_(None)).order_by(Automation.description).all()
|
||||||
return render_template("list.html.j2",
|
return render_template("list.html.j2",
|
||||||
section="automation",
|
section="automation",
|
||||||
title="Automation Jobs",
|
title="Automation Jobs",
|
||||||
|
@ -57,11 +57,11 @@ def automation_edit(automation_id: int) -> ResponseReturnValue:
|
||||||
def automation_kick(automation_id: int) -> ResponseReturnValue:
|
def automation_kick(automation_id: int) -> ResponseReturnValue:
|
||||||
automation = Automation.query.filter(
|
automation = Automation.query.filter(
|
||||||
Automation.id == automation_id,
|
Automation.id == automation_id,
|
||||||
Automation.destroyed == None).first()
|
Automation.destroyed.is_(None)).first()
|
||||||
if automation is None:
|
if automation is None:
|
||||||
return response_404("The requested bridge configuration could not be found.")
|
return response_404("The requested bridge configuration could not be found.")
|
||||||
return view_lifecycle(
|
return view_lifecycle(
|
||||||
header=f"Kick automation timer?",
|
header="Kick automation timer?",
|
||||||
message=automation.description,
|
message=automation.description,
|
||||||
success_view="portal.automation.automation_list",
|
success_view="portal.automation.automation_list",
|
||||||
success_message="This automation job will next run within 1 minute.",
|
success_message="This automation job will next run within 1 minute.",
|
||||||
|
|
|
@ -12,7 +12,7 @@ bp = Blueprint("bridge", __name__)
|
||||||
|
|
||||||
@bp.route("/list")
|
@bp.route("/list")
|
||||||
def bridge_list() -> ResponseReturnValue:
|
def bridge_list() -> ResponseReturnValue:
|
||||||
bridges = Bridge.query.filter(Bridge.destroyed == None).all()
|
bridges = Bridge.query.filter(Bridge.destroyed.is_(None)).all()
|
||||||
return render_template("list.html.j2",
|
return render_template("list.html.j2",
|
||||||
section="bridge",
|
section="bridge",
|
||||||
title="Tor Bridges",
|
title="Tor Bridges",
|
||||||
|
@ -22,7 +22,7 @@ def bridge_list() -> ResponseReturnValue:
|
||||||
|
|
||||||
@bp.route("/block/<bridge_id>", methods=['GET', 'POST'])
|
@bp.route("/block/<bridge_id>", methods=['GET', 'POST'])
|
||||||
def bridge_blocked(bridge_id: int) -> ResponseReturnValue:
|
def bridge_blocked(bridge_id: int) -> ResponseReturnValue:
|
||||||
bridge: Optional[Bridge] = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first()
|
bridge: Optional[Bridge] = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed.is_(None)).first()
|
||||||
if bridge is None:
|
if bridge is None:
|
||||||
return Response(render_template("error.html.j2",
|
return Response(render_template("error.html.j2",
|
||||||
header="404 Proxy Not Found",
|
header="404 Proxy Not Found",
|
||||||
|
|
|
@ -33,7 +33,7 @@ class EditBridgeConfForm(FlaskForm): # type: ignore
|
||||||
|
|
||||||
@bp.route("/list")
|
@bp.route("/list")
|
||||||
def bridgeconf_list() -> ResponseReturnValue:
|
def bridgeconf_list() -> ResponseReturnValue:
|
||||||
bridgeconfs: List[BridgeConf] = BridgeConf.query.filter(BridgeConf.destroyed == None).all()
|
bridgeconfs: List[BridgeConf] = BridgeConf.query.filter(BridgeConf.destroyed.is_(None)).all()
|
||||||
return render_template("list.html.j2",
|
return render_template("list.html.j2",
|
||||||
section="bridgeconf",
|
section="bridgeconf",
|
||||||
title="Tor Bridge Configurations",
|
title="Tor Bridge Configurations",
|
||||||
|
@ -110,11 +110,11 @@ def bridgeconf_edit(bridgeconf_id: int) -> ResponseReturnValue:
|
||||||
|
|
||||||
@bp.route("/destroy/<bridgeconf_id>", methods=['GET', 'POST'])
|
@bp.route("/destroy/<bridgeconf_id>", methods=['GET', 'POST'])
|
||||||
def bridgeconf_destroy(bridgeconf_id: int) -> ResponseReturnValue:
|
def bridgeconf_destroy(bridgeconf_id: int) -> ResponseReturnValue:
|
||||||
bridgeconf = BridgeConf.query.filter(BridgeConf.id == bridgeconf_id, BridgeConf.destroyed == None).first()
|
bridgeconf = BridgeConf.query.filter(BridgeConf.id == bridgeconf_id, BridgeConf.destroyed.is_(None)).first()
|
||||||
if bridgeconf is None:
|
if bridgeconf is None:
|
||||||
return response_404("The requested bridge configuration could not be found.")
|
return response_404("The requested bridge configuration could not be found.")
|
||||||
return view_lifecycle(
|
return view_lifecycle(
|
||||||
header=f"Destroy bridge configuration?",
|
header="Destroy bridge configuration?",
|
||||||
message=bridgeconf.description,
|
message=bridgeconf.description,
|
||||||
success_view="portal.bridgeconf.bridgeconf_list",
|
success_view="portal.bridgeconf.bridgeconf_list",
|
||||||
success_message="All bridges from the destroyed configuration will shortly be destroyed at their providers.",
|
success_message="All bridges from the destroyed configuration will shortly be destroyed at their providers.",
|
||||||
|
|
|
@ -10,7 +10,7 @@ bp = Blueprint("eotk", __name__)
|
||||||
|
|
||||||
@bp.route("/list")
|
@bp.route("/list")
|
||||||
def eotk_list() -> ResponseReturnValue:
|
def eotk_list() -> ResponseReturnValue:
|
||||||
instances = Eotk.query.filter(Eotk.destroyed == None).order_by(desc(Eotk.added)).all()
|
instances = Eotk.query.filter(Eotk.destroyed.is_(None)).order_by(desc(Eotk.added)).all()
|
||||||
return render_template("list.html.j2",
|
return render_template("list.html.j2",
|
||||||
section="eotk",
|
section="eotk",
|
||||||
title="EOTK Instances",
|
title="EOTK Instances",
|
||||||
|
|
|
@ -31,7 +31,7 @@ def list_format_name(s: str) -> str:
|
||||||
|
|
||||||
@bp.route('/list')
|
@bp.route('/list')
|
||||||
def list_list() -> ResponseReturnValue:
|
def list_list() -> ResponseReturnValue:
|
||||||
lists = MirrorList.query.filter(MirrorList.destroyed == None).all()
|
lists = MirrorList.query.filter(MirrorList.destroyed.is_(None)).all()
|
||||||
return render_template("list.html.j2",
|
return render_template("list.html.j2",
|
||||||
section="list",
|
section="list",
|
||||||
title="Mirror Lists",
|
title="Mirror Lists",
|
||||||
|
@ -62,11 +62,11 @@ def list_preview(format_: str) -> ResponseReturnValue:
|
||||||
|
|
||||||
@bp.route("/destroy/<list_id>", methods=['GET', 'POST'])
|
@bp.route("/destroy/<list_id>", methods=['GET', 'POST'])
|
||||||
def list_destroy(list_id: int) -> ResponseReturnValue:
|
def list_destroy(list_id: int) -> ResponseReturnValue:
|
||||||
list_ = MirrorList.query.filter(MirrorList.id == list_id, MirrorList.destroyed == None).first()
|
list_ = MirrorList.query.filter(MirrorList.id == list_id, MirrorList.destroyed.is_(None)).first()
|
||||||
if list_ is None:
|
if list_ is None:
|
||||||
return response_404("The requested bridge configuration could not be found.")
|
return response_404("The requested bridge configuration could not be found.")
|
||||||
return view_lifecycle(
|
return view_lifecycle(
|
||||||
header=f"Destroy mirror list?",
|
header="Destroy mirror list?",
|
||||||
message=list_.description,
|
message=list_.description,
|
||||||
success_view="portal.list.list_list",
|
success_view="portal.list.list_list",
|
||||||
success_message="This list will no longer be updated and may be deleted depending on the provider.",
|
success_message="This list will no longer be updated and may be deleted depending on the provider.",
|
||||||
|
@ -96,7 +96,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue:
|
||||||
try:
|
try:
|
||||||
db.session.add(list_)
|
db.session.add(list_)
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
flash(f"Created new mirror list.", "success")
|
flash("Created new mirror list.", "success")
|
||||||
return redirect(url_for("portal.list.list_list"))
|
return redirect(url_for("portal.list.list_list"))
|
||||||
except exc.SQLAlchemyError as e:
|
except exc.SQLAlchemyError as e:
|
||||||
print(e)
|
print(e)
|
||||||
|
|
|
@ -97,7 +97,7 @@ def onion_list() -> ResponseReturnValue:
|
||||||
|
|
||||||
@bp.route("/destroy/<onion_id>", methods=['GET', 'POST'])
|
@bp.route("/destroy/<onion_id>", methods=['GET', 'POST'])
|
||||||
def onion_destroy(onion_id: int) -> ResponseReturnValue:
|
def onion_destroy(onion_id: int) -> ResponseReturnValue:
|
||||||
onion = Onion.query.filter(Onion.id == onion_id, Onion.destroyed == None).first()
|
onion = Onion.query.filter(Onion.id == onion_id, Onion.destroyed.is_(None)).first()
|
||||||
if onion is None:
|
if onion is None:
|
||||||
return response_404("The requested onion service could not be found.")
|
return response_404("The requested onion service could not be found.")
|
||||||
return view_lifecycle(
|
return view_lifecycle(
|
||||||
|
|
|
@ -115,7 +115,7 @@ def origin_onion() -> ResponseReturnValue:
|
||||||
|
|
||||||
@bp.route("/destroy/<origin_id>", methods=['GET', 'POST'])
|
@bp.route("/destroy/<origin_id>", methods=['GET', 'POST'])
|
||||||
def origin_destroy(origin_id: int) -> ResponseReturnValue:
|
def origin_destroy(origin_id: int) -> ResponseReturnValue:
|
||||||
origin = Origin.query.filter(Origin.id == origin_id, Origin.destroyed == None).first()
|
origin = Origin.query.filter(Origin.id == origin_id, Origin.destroyed.is_(None)).first()
|
||||||
if origin is None:
|
if origin is None:
|
||||||
return response_404("The requested origin could not be found.")
|
return response_404("The requested origin could not be found.")
|
||||||
return view_lifecycle(
|
return view_lifecycle(
|
||||||
|
|
|
@ -11,7 +11,7 @@ bp = Blueprint("proxy", __name__)
|
||||||
|
|
||||||
@bp.route("/list")
|
@bp.route("/list")
|
||||||
def proxy_list() -> ResponseReturnValue:
|
def proxy_list() -> ResponseReturnValue:
|
||||||
proxies = Proxy.query.filter(Proxy.destroyed == None).order_by(desc(Proxy.added)).all()
|
proxies = Proxy.query.filter(Proxy.destroyed.is_(None)).order_by(desc(Proxy.added)).all()
|
||||||
return render_template("list.html.j2",
|
return render_template("list.html.j2",
|
||||||
section="proxy",
|
section="proxy",
|
||||||
title="Proxies",
|
title="Proxies",
|
||||||
|
@ -21,7 +21,7 @@ def proxy_list() -> ResponseReturnValue:
|
||||||
|
|
||||||
@bp.route("/block/<proxy_id>", methods=['GET', 'POST'])
|
@bp.route("/block/<proxy_id>", methods=['GET', 'POST'])
|
||||||
def proxy_block(proxy_id: int) -> ResponseReturnValue:
|
def proxy_block(proxy_id: int) -> ResponseReturnValue:
|
||||||
proxy = Proxy.query.filter(Proxy.id == proxy_id, Proxy.destroyed == None).first()
|
proxy = Proxy.query.filter(Proxy.id == proxy_id, Proxy.destroyed.is_(None)).first()
|
||||||
if proxy is None:
|
if proxy is None:
|
||||||
return Response(render_template("error.html.j2",
|
return Response(render_template("error.html.j2",
|
||||||
header="404 Proxy Not Found",
|
header="404 Proxy Not Found",
|
||||||
|
|
|
@ -48,7 +48,7 @@ def webhook_new() -> ResponseReturnValue:
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
flash(f"Created new webhook {webhook.url}.", "success")
|
flash(f"Created new webhook {webhook.url}.", "success")
|
||||||
return redirect(url_for("portal.webhook.webhook_edit", webhook_id=webhook.id))
|
return redirect(url_for("portal.webhook.webhook_edit", webhook_id=webhook.id))
|
||||||
except exc.SQLAlchemyError as e:
|
except exc.SQLAlchemyError:
|
||||||
flash("Failed to create new webhook.", "danger")
|
flash("Failed to create new webhook.", "danger")
|
||||||
return redirect(url_for("portal.webhook.webhook_list"))
|
return redirect(url_for("portal.webhook.webhook_list"))
|
||||||
return render_template("new.html.j2", section="webhook", form=form)
|
return render_template("new.html.j2", section="webhook", form=form)
|
||||||
|
@ -95,7 +95,7 @@ def webhook_list() -> ResponseReturnValue:
|
||||||
|
|
||||||
@bp.route("/destroy/<webhook_id>", methods=['GET', 'POST'])
|
@bp.route("/destroy/<webhook_id>", methods=['GET', 'POST'])
|
||||||
def webhook_destroy(webhook_id: int) -> ResponseReturnValue:
|
def webhook_destroy(webhook_id: int) -> ResponseReturnValue:
|
||||||
webhook: Optional[Webhook] = Webhook.query.filter(Webhook.id == webhook_id, Webhook.destroyed == None).first()
|
webhook: Optional[Webhook] = Webhook.query.filter(Webhook.id == webhook_id, Webhook.destroyed.is_(None)).first()
|
||||||
if webhook is None:
|
if webhook is None:
|
||||||
return response_404("The requested webhook could not be found.")
|
return response_404("The requested webhook could not be found.")
|
||||||
return view_lifecycle(
|
return view_lifecycle(
|
||||||
|
|
|
@ -28,7 +28,7 @@ class AlarmProxyAzureCdnAutomation(BaseAutomation):
|
||||||
if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
|
if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
|
||||||
for proxy in Proxy.query.filter(
|
for proxy in Proxy.query.filter(
|
||||||
Proxy.provider == "azure_cdn",
|
Proxy.provider == "azure_cdn",
|
||||||
Proxy.destroyed == None
|
Proxy.destroyed.is_(None)
|
||||||
):
|
):
|
||||||
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
|
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
|
||||||
if proxy.origin.group.group_name.lower() not in firing:
|
if proxy.origin.group.group_name.lower() not in firing:
|
||||||
|
|
|
@ -47,7 +47,7 @@ class AlarmProxyCloudfrontAutomation(BaseAutomation):
|
||||||
db.session.add(alarm)
|
db.session.add(alarm)
|
||||||
alarm.last_updated = datetime.datetime.utcnow()
|
alarm.last_updated = datetime.datetime.utcnow()
|
||||||
deployed_count = len(Proxy.query.filter(
|
deployed_count = len(Proxy.query.filter(
|
||||||
Proxy.destroyed == None).all())
|
Proxy.destroyed.is_(None)).all())
|
||||||
old_state = alarm.alarm_state
|
old_state = alarm.alarm_state
|
||||||
if deployed_count > 370:
|
if deployed_count > 370:
|
||||||
alarm.alarm_state = AlarmState.CRITICAL
|
alarm.alarm_state = AlarmState.CRITICAL
|
||||||
|
|
|
@ -29,7 +29,7 @@ class AlarmProxyHTTPStatusAutomation(BaseAutomation):
|
||||||
|
|
||||||
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
||||||
proxies = Proxy.query.filter(
|
proxies = Proxy.query.filter(
|
||||||
Proxy.destroyed == None
|
Proxy.destroyed.is_(None)
|
||||||
)
|
)
|
||||||
for proxy in proxies:
|
for proxy in proxies:
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from typing import Dict, Tuple, Union, Any
|
from typing import Dict, Tuple, Any
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
@ -78,7 +78,7 @@ class BlockOONIAutomation(BaseAutomation):
|
||||||
frequency = 240
|
frequency = 240
|
||||||
|
|
||||||
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
||||||
origins = Origin.query.filter(Origin.destroyed == None).all()
|
origins = Origin.query.filter(Origin.destroyed.is_(None)).all()
|
||||||
for origin in origins:
|
for origin in origins:
|
||||||
ooni = threshold_origin(origin.domain_name)
|
ooni = threshold_origin(origin.domain_name)
|
||||||
for country in ooni:
|
for country in ooni:
|
||||||
|
|
|
@ -17,8 +17,8 @@ class BlockRoskomsvobodaAutomation(BaseAutomation):
|
||||||
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
def automate(self, full: bool = False) -> Tuple[bool, str]:
|
||||||
activities = []
|
activities = []
|
||||||
proxies: List[Proxy] = Proxy.query.filter(
|
proxies: List[Proxy] = Proxy.query.filter(
|
||||||
Proxy.deprecated == None,
|
Proxy.deprecated.is_(None),
|
||||||
Proxy.destroyed == None
|
Proxy.destroyed.is_(None)
|
||||||
).all()
|
).all()
|
||||||
patterns = requests.get("https://reestr.rublacklist.net/api/v2/domains/json").json()
|
patterns = requests.get("https://reestr.rublacklist.net/api/v2/domains/json").json()
|
||||||
for pattern in patterns:
|
for pattern in patterns:
|
||||||
|
|
|
@ -23,12 +23,12 @@ class BridgeAutomation(TerraformAutomation):
|
||||||
def create_missing(self) -> None:
|
def create_missing(self) -> None:
|
||||||
bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter(
|
bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter(
|
||||||
BridgeConf.provider == self.provider,
|
BridgeConf.provider == self.provider,
|
||||||
BridgeConf.destroyed == None
|
BridgeConf.destroyed.is_(None)
|
||||||
).all()
|
).all()
|
||||||
for bridgeconf in bridgeconfs:
|
for bridgeconf in bridgeconfs:
|
||||||
active_bridges = Bridge.query.filter(
|
active_bridges = Bridge.query.filter(
|
||||||
Bridge.conf_id == bridgeconf.id,
|
Bridge.conf_id == bridgeconf.id,
|
||||||
Bridge.deprecated == None
|
Bridge.deprecated.is_(None)
|
||||||
).all()
|
).all()
|
||||||
if len(active_bridges) < bridgeconf.number:
|
if len(active_bridges) < bridgeconf.number:
|
||||||
for i in range(bridgeconf.number - len(active_bridges)):
|
for i in range(bridgeconf.number - len(active_bridges)):
|
||||||
|
@ -49,7 +49,7 @@ class BridgeAutomation(TerraformAutomation):
|
||||||
def destroy_expired(self) -> None:
|
def destroy_expired(self) -> None:
|
||||||
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=0)
|
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=0)
|
||||||
bridges = [b for b in Bridge.query.filter(
|
bridges = [b for b in Bridge.query.filter(
|
||||||
Bridge.destroyed == None,
|
Bridge.destroyed.is_(None),
|
||||||
Bridge.deprecated < cutoff
|
Bridge.deprecated < cutoff
|
||||||
).all() if b.conf.provider == self.provider]
|
).all() if b.conf.provider == self.provider]
|
||||||
for bridge in bridges:
|
for bridge in bridges:
|
||||||
|
@ -66,7 +66,7 @@ class BridgeAutomation(TerraformAutomation):
|
||||||
self.template,
|
self.template,
|
||||||
groups=Group.query.all(),
|
groups=Group.query.all(),
|
||||||
bridgeconfs=BridgeConf.query.filter(
|
bridgeconfs=BridgeConf.query.filter(
|
||||||
BridgeConf.destroyed == None,
|
BridgeConf.destroyed.is_(None),
|
||||||
BridgeConf.provider == self.provider
|
BridgeConf.provider == self.provider
|
||||||
).all(),
|
).all(),
|
||||||
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
||||||
|
|
|
@ -15,7 +15,7 @@ def update_eotk_instance(group_id: int,
|
||||||
Eotk.group_id == group_id,
|
Eotk.group_id == group_id,
|
||||||
Eotk.region == region,
|
Eotk.region == region,
|
||||||
Eotk.provider == "aws",
|
Eotk.provider == "aws",
|
||||||
Eotk.destroyed == None
|
Eotk.destroyed.is_(None)
|
||||||
).first()
|
).first()
|
||||||
if instance is None:
|
if instance is None:
|
||||||
instance = Eotk()
|
instance = Eotk()
|
||||||
|
@ -80,8 +80,8 @@ class EotkAWSAutomation(TerraformAutomation):
|
||||||
self.tf_write(
|
self.tf_write(
|
||||||
self.template,
|
self.template,
|
||||||
groups=Group.query.filter(
|
groups=Group.query.filter(
|
||||||
Group.eotk == True,
|
Group.eotk.is_(True),
|
||||||
Group.destroyed == None
|
Group.destroyed.is_(None)
|
||||||
).all(),
|
).all(),
|
||||||
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
||||||
**{
|
**{
|
||||||
|
|
|
@ -25,7 +25,7 @@ class ListAutomation(TerraformAutomation):
|
||||||
self.tf_write(
|
self.tf_write(
|
||||||
self.template,
|
self.template,
|
||||||
lists=MirrorList.query.filter(
|
lists=MirrorList.query.filter(
|
||||||
MirrorList.destroyed == None,
|
MirrorList.destroyed.is_(None),
|
||||||
MirrorList.provider == self.provider,
|
MirrorList.provider == self.provider,
|
||||||
).all(),
|
).all(),
|
||||||
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
||||||
|
|
|
@ -83,8 +83,8 @@ class ProxyAutomation(TerraformAutomation):
|
||||||
|
|
||||||
def deprecate_orphaned_proxies(self) -> None:
|
def deprecate_orphaned_proxies(self) -> None:
|
||||||
proxies = Proxy.query.filter(
|
proxies = Proxy.query.filter(
|
||||||
Proxy.deprecated == None,
|
Proxy.deprecated.is_(None),
|
||||||
Proxy.destroyed == None,
|
Proxy.destroyed.is_(None),
|
||||||
Proxy.provider == self.provider
|
Proxy.provider == self.provider
|
||||||
).all()
|
).all()
|
||||||
for proxy in proxies:
|
for proxy in proxies:
|
||||||
|
@ -95,7 +95,7 @@ class ProxyAutomation(TerraformAutomation):
|
||||||
def destroy_expired_proxies(self) -> None:
|
def destroy_expired_proxies(self) -> None:
|
||||||
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
|
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
|
||||||
proxies = Proxy.query.filter(
|
proxies = Proxy.query.filter(
|
||||||
Proxy.destroyed == None,
|
Proxy.destroyed.is_(None),
|
||||||
Proxy.provider == self.provider,
|
Proxy.provider == self.provider,
|
||||||
Proxy.deprecated < cutoff
|
Proxy.deprecated < cutoff
|
||||||
).all()
|
).all()
|
||||||
|
@ -123,7 +123,7 @@ class ProxyAutomation(TerraformAutomation):
|
||||||
groups=Group.query.all(),
|
groups=Group.query.all(),
|
||||||
proxies=Proxy.query.filter(
|
proxies=Proxy.query.filter(
|
||||||
Proxy.provider == self.provider,
|
Proxy.provider == self.provider,
|
||||||
Proxy.destroyed == None
|
Proxy.destroyed.is_(None)
|
||||||
).all(),
|
).all(),
|
||||||
subgroups=self.get_subgroups(),
|
subgroups=self.get_subgroups(),
|
||||||
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
||||||
|
|
|
@ -34,7 +34,6 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
|
||||||
|
|
||||||
provider "azurerm" {
|
provider "azurerm" {
|
||||||
features {}
|
features {}
|
||||||
|
|
||||||
client_id = "{{ azure_client_id }}"
|
client_id = "{{ azure_client_id }}"
|
||||||
client_secret = "{{ azure_client_secret }}"
|
client_secret = "{{ azure_client_secret }}"
|
||||||
subscription_id = "{{ azure_subscription_id }}"
|
subscription_id = "{{ azure_subscription_id }}"
|
||||||
|
@ -69,8 +68,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
|
||||||
location = "{{ azure_location }}"
|
location = "{{ azure_location }}"
|
||||||
resource_group_name = data.azurerm_resource_group.this.name
|
resource_group_name = data.azurerm_resource_group.this.name
|
||||||
sku = "Standard_Microsoft"
|
sku = "Standard_Microsoft"
|
||||||
|
tags = module.label_{{ group.id }}.tags
|
||||||
tags = module.label_{{ group.id }}.tags
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}_{{ subgroup }}" {
|
resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}_{{ subgroup }}" {
|
||||||
|
@ -162,7 +160,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
|
||||||
def import_state(self, state: Optional[Any]) -> None:
|
def import_state(self, state: Optional[Any]) -> None:
|
||||||
proxies = Proxy.query.filter(
|
proxies = Proxy.query.filter(
|
||||||
Proxy.provider == self.provider,
|
Proxy.provider == self.provider,
|
||||||
Proxy.destroyed == None
|
Proxy.destroyed.is_(None)
|
||||||
).all()
|
).all()
|
||||||
for proxy in proxies:
|
for proxy in proxies:
|
||||||
proxy.url = f"https://{proxy.slug}.azureedge.net"
|
proxy.url = f"https://{proxy.slug}.azureedge.net"
|
||||||
|
|
|
@ -110,7 +110,7 @@ def create_missing_proxies():
|
||||||
def destroy_expired_proxies():
|
def destroy_expired_proxies():
|
||||||
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
|
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
|
||||||
proxies = Proxy.query.filter(
|
proxies = Proxy.query.filter(
|
||||||
Proxy.destroyed == None,
|
Proxy.destroyed.is_(None),
|
||||||
Proxy.provider == "fastly",
|
Proxy.provider == "fastly",
|
||||||
Proxy.deprecated < cutoff
|
Proxy.deprecated < cutoff
|
||||||
).all()
|
).all()
|
||||||
|
|
2
setup.cfg
Normal file
2
setup.cfg
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
[flake8]
|
||||||
|
ignore = E501
|
Loading…
Add table
Add a link
Reference in a new issue