jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ci: add flake8

Browse source
This commit is contained in:
Iain Learmonth 2022-05-16 13:29:48 +01:00
parent 014596d271
commit dd501a6e4e
32 changed files with 170 additions and 171 deletions
Download patch file Download diff file Expand all files Collapse all files

61
.gitlab-ci.yml
View file

@ -2,44 +2,51 @@ image: python:3.8-bullseye
test:bandit:
script:
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt --quiet
- pip install bandit --quiet
- bandit -r app
test:mypy:
script:
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt --quiet
- pip install -r requirements-types.txt --quiet
- mypy app
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt --quiet
- pip install bandit --quiet
- bandit -r app
test:docs:
stage: test
script:
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt
- pip install -U sphinx sphinx-press-theme sphinx-jsonschema
- pushd scripts && python update_schemas.py && popd
- pushd docs && sphinx-build -b html . ../public && popd
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt
- pip install -U sphinx sphinx-press-theme sphinx-jsonschema
- pushd scripts && python update_schemas.py && popd
- pushd docs && sphinx-build -b html . ../public && popd
rules:
- if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH
test:flake8:
script:
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt --quiet
- pip install flake8 --quiet
- flake8 app
test:mypy:
script:
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt --quiet
- pip install -r requirements-types.txt --quiet
- mypy app
pages:
stage: deploy
script:
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt
- pip install -U sphinx sphinx-press-theme sphinx-jsonschema
- pushd scripts && python update_schemas.py && popd
- pushd docs && sphinx-build -b html . ../public && popd
- cp config.yaml.example config.yaml
- apt update && apt install build-essential
- pip install -r requirements.txt
- pip install -U sphinx sphinx-press-theme sphinx-jsonschema
- pushd scripts && python update_schemas.py && popd
- pushd docs && sphinx-build -b html . ../public && popd
artifacts:
paths:
- public
rules:
- if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH

3
app/__init__.py
View file

@ -5,9 +5,6 @@ import yaml
from app.extensions import db
from app.extensions import migrate
from app.extensions import bootstrap
from app.lists.bc2 import mirror_sites
from app.models.mirrors import Origin, Proxy, Mirror
from app.models.base import Group
from app.portal import portal
app = Flask(__name__)

64
app/lists/bc2.py
View file

@ -4,7 +4,7 @@ from typing import List, Dict, Union
from pydantic import BaseModel, Field
from app.models.mirrors import Origin
from app.models.mirrors import Origin, Proxy, Mirror
class BC2Alternative(BaseModel):
@ -18,52 +18,48 @@ class BC2Alternative(BaseModel):
class BC2Site(BaseModel):
main_domain: str = Field(
description="The main domain name of the website, excluding \"www.\" if present.",
examples=["bbc.co.uk", "bbc.com", "guardianproject.info"]
)
examples=["bbc.co.uk", "bbc.com", "guardianproject.info"])
available_alternatives: List[BC2Alternative]
class BypassCensorship2(BaseModel):
version: str = Field(
description="Version number of the Bypass Censorship Extension schema in use",
)
version: str = Field(description="Version number of the Bypass Censorship Extension schema in use", )
sites: List[BC2Site]
class Config:
title = "Bypass Censorship Version 2"
def mirror_sites(provider: str = "cloudfront") -> Dict[
str, Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]:
def mirror_alternative(mirror: Mirror):
return {
"version": "2.0",
"sites": [{
"main_domain": x.description[len("proxy:"):].replace("www.", "") if x.description.startswith(
"proxy:") else x.domain_name.replace("www.", ""),
"available_alternatives": [
{
"proto": "tor" if ".onion" in a.url else "https",
"type": "eotk" if ".onion" in a.url else "mirror",
"created_at": str(a.added),
"updated_at": str(a.updated),
"url": a.url
} for a in x.mirrors if not a.deprecated and not a.destroyed
] + [
{
"proto": "https",
"type": "mirror",
"created_at": str(a.added),
"updated_at": str(a.updated),
"url": a.url
} for a in x.proxies if
a.url is not None
and not a.deprecated
and not a.destroyed
and a.provider == provider
]} for x in Origin.query.order_by(Origin.domain_name).all() if x.destroyed is None
]
"proto": "tor" if ".onion" in mirror.url else "https",
"type": "eotk" if ".onion" in mirror.url else "mirror",
"created_at": str(mirror.added),
"updated_at": str(mirror.updated),
"url": mirror.url
}
def proxy_alternative(proxy: Proxy):
return {
"proto": "https",
"type": "mirror",
"created_at": str(proxy.added),
"updated_at": str(proxy.updated),
"url": proxy.url
}
def mirror_sites(provider: str = "cloudfront") -> Dict[str,
Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]]:
return {"version": "2.0", "sites": [{
"main_domain": x.description[len("proxy:"):].replace("www.", "") if x.description.startswith(
"proxy:") else x.domain_name.replace("www.", ""),
"available_alternatives": [mirror_alternative(a) for a in x.mirrors if not a.deprecated and not a.destroyed] + [
proxy_alternative(a) for a in x.proxies if
a.url is not None and not a.deprecated and not a.destroyed and a.provider == provider]} for x in
Origin.query.order_by(Origin.domain_name).all() if x.destroyed is None]}
if getattr(builtins, "__sphinx_build__", False):
schema = BypassCensorship2.schema_json()

6
app/lists/bridgelines.py
View file

@ -28,9 +28,9 @@ class Bridgelines(BaseModel):
def bridgelines(*, distribution_method: Optional[str] = None) -> Dict[str, Any]:
bridges: Iterable[Bridge] = Bridge.query.filter(
Bridge.destroyed == None,
Bridge.deprecated == None,
Bridge.bridgeline != None
Bridge.destroyed.is_(None),
Bridge.deprecated.is_(None),
Bridge.bridgeline.is_not(None)
).all()
if distribution_method is not None:
bridges = [b for b in bridges

2
app/lists/mirror_mapping.py
View file

@ -46,7 +46,7 @@ def mirror_mapping() -> Dict[str, Union[str, Dict[str, str]]]:
},
s3_buckets=[
f"{app.config['GLOBAL_NAMESPACE']}-{g.group_name.lower()}-logs-cloudfront"
for g in Group.query.filter(Group.destroyed == None).all()
for g in Group.query.filter(Group.destroyed.is_(None)).all()
]
).dict()

4
app/models/activity.py
View file

@ -37,7 +37,7 @@ class Activity(db.Model): # type: ignore
def notify(self) -> int:
count = 0
hooks = Webhook.query.filter(
Webhook.destroyed == None
Webhook.destroyed.is_(None)
)
for hook in hooks:
hook.send(self.text)
@ -55,4 +55,4 @@ class Webhook(AbstractConfiguration):
else:
# Matrix as default
data = {"body": text}
r = requests.post(self.url, json=data)
requests.post(self.url, json=data)

13
app/portal/__init__.py
View file

@ -10,7 +10,6 @@ from app.models.alarms import Alarm, AlarmState
from app.models.bridges import Bridge
from app.models.mirrors import Origin, Proxy
from app.models.base import Group
from app.portal.list import NewMirrorListForm
from app.portal.automation import bp as automation
from app.portal.bridgeconf import bp as bridgeconf
from app.portal.bridge import bp as bridge
@ -53,7 +52,7 @@ def format_datetime(s: Optional[datetime]) -> str:
def total_origins_blocked() -> int:
count = 0
for o in Origin.query.filter(Origin.destroyed == None).all():
for o in Origin.query.filter(Origin.destroyed.is_(None)).all():
for a in o.alarms:
if a.alarm_type.startswith("origin-block-ooni-"):
if a.alarm_state == AlarmState.WARNING:
@ -66,7 +65,7 @@ def total_origins_blocked() -> int:
def portal_home() -> ResponseReturnValue:
groups = Group.query.order_by(Group.group_name).all()
now = datetime.now(timezone.utc)
proxies = Proxy.query.filter(Proxy.destroyed == None).all()
proxies = Proxy.query.filter(Proxy.destroyed.is_(None)).all()
last24 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=1))).all())
last72 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=3))).all())
lastweek = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=7))).all())
@ -74,15 +73,15 @@ def portal_home() -> ResponseReturnValue:
s: len(Alarm.query.filter(Alarm.alarm_state == s.upper(), Alarm.last_updated > (now - timedelta(days=1))).all())
for s in ["critical", "warning", "ok", "unknown"]
}
bridges = Bridge.query.filter(Bridge.destroyed == None).all()
bridges = Bridge.query.filter(Bridge.destroyed.is_(None)).all()
br_last = {
d: len(Bridge.query.filter(Bridge.deprecated > (now - timedelta(days=d))).all())
for d in [1, 3, 7]
}
activity = Activity.query.filter(Activity.added > (now - timedelta(days=2))).order_by(desc(Activity.added)).all()
onionified = len([o for o in Origin.query.filter(Origin.destroyed == None).all() if o.onion() != None])
onionified = len([o for o in Origin.query.filter(Origin.destroyed.is_(None)).all() if o.onion() is not None])
ooni_blocked = total_origins_blocked()
total_origins = len(Origin.query.filter(Origin.destroyed == None).all())
total_origins = len(Origin.query.filter(Origin.destroyed.is_(None)).all())
return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72,
lastweek=lastweek, proxies=proxies, **alarms, activity=activity, total_origins=total_origins,
onionified=onionified, br_last=br_last, ooni_blocked=ooni_blocked, bridges=bridges)
@ -91,7 +90,7 @@ def portal_home() -> ResponseReturnValue:
@portal.route("/search")
def search() -> ResponseReturnValue:
query = request.args.get("query")
proxies = Proxy.query.filter(or_(Proxy.url.contains(query)), Proxy.destroyed == None).all()
proxies = Proxy.query.filter(or_(Proxy.url.contains(query)), Proxy.destroyed.is_(None)).all()
origins = Origin.query.filter(or_(Origin.description.contains(query), Origin.domain_name.contains(query))).all()
return render_template("search.html.j2", section="home", proxies=proxies, origins=origins)

6
app/portal/automation.py
View file

@ -22,7 +22,7 @@ class EditAutomationForm(FlaskForm): # type: ignore
@bp.route("/list")
def automation_list() -> ResponseReturnValue:
automations = Automation.query.filter(
Automation.destroyed == None).order_by(Automation.description).all()
Automation.destroyed.is_(None)).order_by(Automation.description).all()
return render_template("list.html.j2",
section="automation",
title="Automation Jobs",
@ -57,11 +57,11 @@ def automation_edit(automation_id: int) -> ResponseReturnValue:
def automation_kick(automation_id: int) -> ResponseReturnValue:
automation = Automation.query.filter(
Automation.id == automation_id,
Automation.destroyed == None).first()
Automation.destroyed.is_(None)).first()
if automation is None:
return response_404("The requested bridge configuration could not be found.")
return view_lifecycle(
header=f"Kick automation timer?",
header="Kick automation timer?",
message=automation.description,
success_view="portal.automation.automation_list",
success_message="This automation job will next run within 1 minute.",

4
app/portal/bridge.py
View file

@ -12,7 +12,7 @@ bp = Blueprint("bridge", __name__)
@bp.route("/list")
def bridge_list() -> ResponseReturnValue:
bridges = Bridge.query.filter(Bridge.destroyed == None).all()
bridges = Bridge.query.filter(Bridge.destroyed.is_(None)).all()
return render_template("list.html.j2",
section="bridge",
title="Tor Bridges",
@ -22,7 +22,7 @@ def bridge_list() -> ResponseReturnValue:
@bp.route("/block/<bridge_id>", methods=['GET', 'POST'])
def bridge_blocked(bridge_id: int) -> ResponseReturnValue:
bridge: Optional[Bridge] = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first()
bridge: Optional[Bridge] = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed.is_(None)).first()
if bridge is None:
return Response(render_template("error.html.j2",
header="404 Proxy Not Found",

6
app/portal/bridgeconf.py
View file

@ -33,7 +33,7 @@ class EditBridgeConfForm(FlaskForm): # type: ignore
@bp.route("/list")
def bridgeconf_list() -> ResponseReturnValue:
bridgeconfs: List[BridgeConf] = BridgeConf.query.filter(BridgeConf.destroyed == None).all()
bridgeconfs: List[BridgeConf] = BridgeConf.query.filter(BridgeConf.destroyed.is_(None)).all()
return render_template("list.html.j2",
section="bridgeconf",
title="Tor Bridge Configurations",
@ -110,11 +110,11 @@ def bridgeconf_edit(bridgeconf_id: int) -> ResponseReturnValue:
@bp.route("/destroy/<bridgeconf_id>", methods=['GET', 'POST'])
def bridgeconf_destroy(bridgeconf_id: int) -> ResponseReturnValue:
bridgeconf = BridgeConf.query.filter(BridgeConf.id == bridgeconf_id, BridgeConf.destroyed == None).first()
bridgeconf = BridgeConf.query.filter(BridgeConf.id == bridgeconf_id, BridgeConf.destroyed.is_(None)).first()
if bridgeconf is None:
return response_404("The requested bridge configuration could not be found.")
return view_lifecycle(
header=f"Destroy bridge configuration?",
header="Destroy bridge configuration?",
message=bridgeconf.description,
success_view="portal.bridgeconf.bridgeconf_list",
success_message="All bridges from the destroyed configuration will shortly be destroyed at their providers.",

2
app/portal/eotk.py
View file

@ -10,7 +10,7 @@ bp = Blueprint("eotk", __name__)
@bp.route("/list")
def eotk_list() -> ResponseReturnValue:
instances = Eotk.query.filter(Eotk.destroyed == None).order_by(desc(Eotk.added)).all()
instances = Eotk.query.filter(Eotk.destroyed.is_(None)).order_by(desc(Eotk.added)).all()
return render_template("list.html.j2",
section="eotk",
title="EOTK Instances",

8
app/portal/list.py
View file

@ -31,7 +31,7 @@ def list_format_name(s: str) -> str:
@bp.route('/list')
def list_list() -> ResponseReturnValue:
lists = MirrorList.query.filter(MirrorList.destroyed == None).all()
lists = MirrorList.query.filter(MirrorList.destroyed.is_(None)).all()
return render_template("list.html.j2",
section="list",
title="Mirror Lists",
@ -62,11 +62,11 @@ def list_preview(format_: str) -> ResponseReturnValue:
@bp.route("/destroy/<list_id>", methods=['GET', 'POST'])
def list_destroy(list_id: int) -> ResponseReturnValue:
list_ = MirrorList.query.filter(MirrorList.id == list_id, MirrorList.destroyed == None).first()
list_ = MirrorList.query.filter(MirrorList.id == list_id, MirrorList.destroyed.is_(None)).first()
if list_ is None:
return response_404("The requested bridge configuration could not be found.")
return view_lifecycle(
header=f"Destroy mirror list?",
header="Destroy mirror list?",
message=list_.description,
success_view="portal.list.list_list",
success_message="This list will no longer be updated and may be deleted depending on the provider.",
@ -96,7 +96,7 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue:
try:
db.session.add(list_)
db.session.commit()
flash(f"Created new mirror list.", "success")
flash("Created new mirror list.", "success")
return redirect(url_for("portal.list.list_list"))
except exc.SQLAlchemyError as e:
print(e)

2
app/portal/onion.py
View file

@ -97,7 +97,7 @@ def onion_list() -> ResponseReturnValue:
@bp.route("/destroy/<onion_id>", methods=['GET', 'POST'])
def onion_destroy(onion_id: int) -> ResponseReturnValue:
onion = Onion.query.filter(Onion.id == onion_id, Onion.destroyed == None).first()
onion = Onion.query.filter(Onion.id == onion_id, Onion.destroyed.is_(None)).first()
if onion is None:
return response_404("The requested onion service could not be found.")
return view_lifecycle(

2
app/portal/origin.py
View file

@ -115,7 +115,7 @@ def origin_onion() -> ResponseReturnValue:
@bp.route("/destroy/<origin_id>", methods=['GET', 'POST'])
def origin_destroy(origin_id: int) -> ResponseReturnValue:
origin = Origin.query.filter(Origin.id == origin_id, Origin.destroyed == None).first()
origin = Origin.query.filter(Origin.id == origin_id, Origin.destroyed.is_(None)).first()
if origin is None:
return response_404("The requested origin could not be found.")
return view_lifecycle(

4
app/portal/proxy.py
View file

@ -11,7 +11,7 @@ bp = Blueprint("proxy", __name__)
@bp.route("/list")
def proxy_list() -> ResponseReturnValue:
proxies = Proxy.query.filter(Proxy.destroyed == None).order_by(desc(Proxy.added)).all()
proxies = Proxy.query.filter(Proxy.destroyed.is_(None)).order_by(desc(Proxy.added)).all()
return render_template("list.html.j2",
section="proxy",
title="Proxies",
@ -21,7 +21,7 @@ def proxy_list() -> ResponseReturnValue:
@bp.route("/block/<proxy_id>", methods=['GET', 'POST'])
def proxy_block(proxy_id: int) -> ResponseReturnValue:
proxy = Proxy.query.filter(Proxy.id == proxy_id, Proxy.destroyed == None).first()
proxy = Proxy.query.filter(Proxy.id == proxy_id, Proxy.destroyed.is_(None)).first()
if proxy is None:
return Response(render_template("error.html.j2",
header="404 Proxy Not Found",

4
app/portal/webhook.py
View file

@ -48,7 +48,7 @@ def webhook_new() -> ResponseReturnValue:
db.session.commit()
flash(f"Created new webhook {webhook.url}.", "success")
return redirect(url_for("portal.webhook.webhook_edit", webhook_id=webhook.id))
except exc.SQLAlchemyError as e:
except exc.SQLAlchemyError:
flash("Failed to create new webhook.", "danger")
return redirect(url_for("portal.webhook.webhook_list"))
return render_template("new.html.j2", section="webhook", form=form)
@ -95,7 +95,7 @@ def webhook_list() -> ResponseReturnValue:
@bp.route("/destroy/<webhook_id>", methods=['GET', 'POST'])
def webhook_destroy(webhook_id: int) -> ResponseReturnValue:
webhook: Optional[Webhook] = Webhook.query.filter(Webhook.id == webhook_id, Webhook.destroyed == None).first()
webhook: Optional[Webhook] = Webhook.query.filter(Webhook.id == webhook_id, Webhook.destroyed.is_(None)).first()
if webhook is None:
return response_404("The requested webhook could not be found.")
return view_lifecycle(

4
app/terraform/alarms/proxy_azure_cdn.py
View file

@ -28,11 +28,11 @@ class AlarmProxyAzureCdnAutomation(BaseAutomation):
if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
for proxy in Proxy.query.filter(
Proxy.provider == "azure_cdn",
Proxy.destroyed == None
Proxy.destroyed.is_(None)
):
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
if proxy.origin.group.group_name.lower() not in firing:
alarm.update_state(AlarmState.OK, "Azure monitor alert not firing")
else:
alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing")
return True, ""
return True, ""

2
app/terraform/alarms/proxy_cloudfront.py
View file

@ -47,7 +47,7 @@ class AlarmProxyCloudfrontAutomation(BaseAutomation):
db.session.add(alarm)
alarm.last_updated = datetime.datetime.utcnow()
deployed_count = len(Proxy.query.filter(
Proxy.destroyed == None).all())
Proxy.destroyed.is_(None)).all())
old_state = alarm.alarm_state
if deployed_count > 370:
alarm.alarm_state = AlarmState.CRITICAL

2
app/terraform/alarms/proxy_http_status.py
View file

@ -29,7 +29,7 @@ class AlarmProxyHTTPStatusAutomation(BaseAutomation):
def automate(self, full: bool = False) -> Tuple[bool, str]:
proxies = Proxy.query.filter(
Proxy.destroyed == None
Proxy.destroyed.is_(None)
)
for proxy in proxies:
try:

4
app/terraform/block_ooni.py
View file

@ -1,7 +1,7 @@
from collections import defaultdict
from datetime import datetime
from datetime import timedelta
from typing import Dict, Tuple, Union, Any
from typing import Dict, Tuple, Any
import requests
@ -78,7 +78,7 @@ class BlockOONIAutomation(BaseAutomation):
frequency = 240
def automate(self, full: bool = False) -> Tuple[bool, str]:
origins = Origin.query.filter(Origin.destroyed == None).all()
origins = Origin.query.filter(Origin.destroyed.is_(None)).all()
for origin in origins:
ooni = threshold_origin(origin.domain_name)
for country in ooni:

4
app/terraform/block_roskomsvoboda.py
View file

@ -17,8 +17,8 @@ class BlockRoskomsvobodaAutomation(BaseAutomation):
def automate(self, full: bool = False) -> Tuple[bool, str]:
activities = []
proxies: List[Proxy] = Proxy.query.filter(
Proxy.deprecated == None,
Proxy.destroyed == None
Proxy.deprecated.is_(None),
Proxy.destroyed.is_(None)
).all()
patterns = requests.get("https://reestr.rublacklist.net/api/v2/domains/json").json()
for pattern in patterns:

8
app/terraform/bridge/__init__.py
View file

@ -23,12 +23,12 @@ class BridgeAutomation(TerraformAutomation):
def create_missing(self) -> None:
bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter(
BridgeConf.provider == self.provider,
BridgeConf.destroyed == None
BridgeConf.destroyed.is_(None)
).all()
for bridgeconf in bridgeconfs:
active_bridges = Bridge.query.filter(
Bridge.conf_id == bridgeconf.id,
Bridge.deprecated == None
Bridge.deprecated.is_(None)
).all()
if len(active_bridges) < bridgeconf.number:
for i in range(bridgeconf.number - len(active_bridges)):
@ -49,7 +49,7 @@ class BridgeAutomation(TerraformAutomation):
def destroy_expired(self) -> None:
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=0)
bridges = [b for b in Bridge.query.filter(
Bridge.destroyed == None,
Bridge.destroyed.is_(None),
Bridge.deprecated < cutoff
).all() if b.conf.provider == self.provider]
for bridge in bridges:
@ -66,7 +66,7 @@ class BridgeAutomation(TerraformAutomation):
self.template,
groups=Group.query.all(),
bridgeconfs=BridgeConf.query.filter(
BridgeConf.destroyed == None,
BridgeConf.destroyed.is_(None),
BridgeConf.provider == self.provider
).all(),
global_namespace=app.config['GLOBAL_NAMESPACE'],

12
app/terraform/bridge/aws.py
View file

@ -20,17 +20,17 @@ class BridgeAWSAutomation(BridgeAutomation):
}
}
}
provider "aws" {
access_key = "{{ aws_access_key }}"
secret_key = "{{ aws_secret_key }}"
region = "us-east-1"
}
locals {
ssh_key = file("{{ ssh_public_key_path }}")
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
@ -40,7 +40,7 @@ class BridgeAWSAutomation(BridgeAutomation):
label_order = ["namespace", "tenant", "name", "attributes"]
}
{% endfor %}
{% for bridgeconf in bridgeconfs %}
{% for bridge in bridgeconf.bridges %}
{% if not bridge.destroyed %}
@ -54,11 +54,11 @@ class BridgeAWSAutomation(BridgeAutomation):
attributes = ["{{ bridge.id }}"]
distribution_method = "{{ bridge.conf.method }}"
}
output "bridge_hashed_fingerprint_{{ bridge.id }}" {
value = module.bridge_{{ bridge.id }}.hashed_fingerprint
}
output "bridge_bridgeline_{{ bridge.id }}" {
value = module.bridge_{{ bridge.id }}.bridgeline
sensitive = true

20
app/terraform/bridge/ovh.py
View file

@ -34,7 +34,7 @@ class BridgeOvhAutomation(BridgeAutomation):
}
}
}
provider "openstack" {
auth_url = "https://auth.cloud.ovh.net/v3/"
domain_name = "Default" # Domain name - Always at 'default' for OVHcloud
@ -42,24 +42,24 @@ class BridgeOvhAutomation(BridgeAutomation):
password = "{{ ovh_openstack_password }}"
tenant_id = "{{ ovh_openstack_tenant_id }}"
}
provider "ovh" {
endpoint = "ovh-eu"
application_key = "{{ ovh_cloud_application_key }}"
application_secret = "{{ ovh_cloud_application_secret }}"
consumer_key = "{{ ovh_cloud_consumer_key }}"
}
locals {
public_ssh_key = file("{{ ssh_public_key_path }}")
private_ssh_key = file("{{ ssh_private_key_path }}")
}
data "ovh_cloud_project_regions" "regions" {
service_name = "{{ ovh_openstack_tenant_id }}"
has_services_up = ["instance"]
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
@ -69,19 +69,19 @@ class BridgeOvhAutomation(BridgeAutomation):
label_order = ["namespace", "tenant", "name", "attributes"]
}
{% endfor %}
{% for bridgeconf in bridgeconfs %}
{% for bridge in bridgeconf.bridges %}
{% if not bridge.destroyed %}
resource "random_shuffle" "region_{{ bridge.id }}" {
input = data.ovh_cloud_project_regions.regions.names
result_count = 1
lifecycle {
ignore_changes = [input] # don't replace all the bridges if a new region appears
}
}
module "bridge_{{ bridge.id }}" {
source = "sr2c/tor-bridge/openstack"
version = "0.0.7"
@ -94,11 +94,11 @@ class BridgeOvhAutomation(BridgeAutomation):
contact_info = "hi"
distribution_method = "{{ bridge.conf.method }}"
}
output "bridge_hashed_fingerprint_{{ bridge.id }}" {
value = module.bridge_{{ bridge.id }}.hashed_fingerprint
}
output "bridge_bridgeline_{{ bridge.id }}" {
value = module.bridge_{{ bridge.id }}.bridgeline
sensitive = true

12
app/terraform/eotk/aws.py
View file

@ -15,7 +15,7 @@ def update_eotk_instance(group_id: int,
Eotk.group_id == group_id,
Eotk.region == region,
Eotk.provider == "aws",
Eotk.destroyed == None
Eotk.destroyed.is_(None)
).first()
if instance is None:
instance = Eotk()
@ -45,20 +45,20 @@ class EotkAWSAutomation(TerraformAutomation):
}
}
}
provider "aws" {
access_key = "{{ aws_access_key }}"
secret_key = "{{ aws_secret_key }}"
region = "us-east-2"
}
provider "aws" {
access_key = "{{ aws_access_key }}"
secret_key = "{{ aws_secret_key }}"
region = "eu-central-1"
alias = "second_region"
}
{% for group in groups %}
module "eotk_{{ group.id }}" {
providers = {
@ -80,8 +80,8 @@ class EotkAWSAutomation(TerraformAutomation):
self.tf_write(
self.template,
groups=Group.query.filter(
Group.eotk == True,
Group.destroyed == None
Group.eotk.is_(True),
Group.destroyed.is_(None)
).all(),
global_namespace=app.config['GLOBAL_NAMESPACE'],
**{

2
app/terraform/list/__init__.py
View file

@ -25,7 +25,7 @@ class ListAutomation(TerraformAutomation):
self.tf_write(
self.template,
lists=MirrorList.query.filter(
MirrorList.destroyed == None,
MirrorList.destroyed.is_(None),
MirrorList.provider == self.provider,
).all(),
global_namespace=app.config['GLOBAL_NAMESPACE'],

8
app/terraform/list/gitlab.py
View file

@ -22,16 +22,16 @@ class ListGitlabAutomation(ListAutomation):
}
}
}
provider "gitlab" {
token = "{{ gitlab_token }}"
}
{% for list in lists %}
data "gitlab_project" "project_{{ list.id }}" {
id = "{{ list.container }}"
}
resource "gitlab_repository_file" "file_{{ list.id }}" {
project = data.gitlab_project.project_{{ list.id }}.id
file_path = "{{ list.filename }}"
@ -41,6 +41,6 @@ class ListGitlabAutomation(ListAutomation):
author_name = "{{ gitlab_author_name }}"
commit_message = "{{ gitlab_commit_message }}"
}
{% endfor %}
"""

8
app/terraform/proxy/__init__.py
View file

@ -83,8 +83,8 @@ class ProxyAutomation(TerraformAutomation):
def deprecate_orphaned_proxies(self) -> None:
proxies = Proxy.query.filter(
Proxy.deprecated == None,
Proxy.destroyed == None,
Proxy.deprecated.is_(None),
Proxy.destroyed.is_(None),
Proxy.provider == self.provider
).all()
for proxy in proxies:
@ -95,7 +95,7 @@ class ProxyAutomation(TerraformAutomation):
def destroy_expired_proxies(self) -> None:
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
proxies = Proxy.query.filter(
Proxy.destroyed == None,
Proxy.destroyed.is_(None),
Proxy.provider == self.provider,
Proxy.deprecated < cutoff
).all()
@ -123,7 +123,7 @@ class ProxyAutomation(TerraformAutomation):
groups=Group.query.all(),
proxies=Proxy.query.filter(
Proxy.provider == self.provider,
Proxy.destroyed == None
Proxy.destroyed.is_(None)
).all(),
subgroups=self.get_subgroups(),
global_namespace=app.config['GLOBAL_NAMESPACE'],

46
app/terraform/proxy/azure_cdn.py
View file

@ -31,21 +31,20 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
}
}
}
provider "azurerm" {
features {}
client_id = "{{ azure_client_id }}"
client_secret = "{{ azure_client_secret }}"
subscription_id = "{{ azure_subscription_id }}"
tenant_id = "{{ azure_tenant_id }}"
skip_provider_registration = true
}
data "azurerm_resource_group" "this" {
name = "{{ azure_resource_group_name }}"
}
resource "azurerm_storage_account" "this" {
name = "{{ azure_storage_account_name }}"
resource_group_name = data.azurerm_resource_group.this.name
@ -53,7 +52,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
account_tier = "Standard"
account_replication_type = "RAGRS"
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
@ -62,49 +61,48 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
tenant = "{{ group.group_name }}"
label_order = ["namespace", "tenant", "name", "attributes"]
}
{% for subgroup in subgroups[group.id] %}
resource "azurerm_cdn_profile" "profile_{{ group.id }}_{{ subgroup }}" {
name = "${module.label_{{ group.id }}.id}-sub{{ subgroup }}"
location = "{{ azure_location }}"
resource_group_name = data.azurerm_resource_group.this.name
sku = "Standard_Microsoft"
tags = module.label_{{ group.id }}.tags
tags = module.label_{{ group.id }}.tags
}
resource "azurerm_monitor_diagnostic_setting" "profile_diagnostic_{{ group.id }}_{{ subgroup }}" {
name = "cdn-diagnostics"
target_resource_id = azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id
storage_account_id = azurerm_storage_account.this.id
log {
category = "AzureCDNAccessLog"
enabled = true
retention_policy {
enabled = true
days = 90
}
}
metric {
category = "AllMetrics"
enabled = true
retention_policy {
enabled = true
days = 90
}
}
}
resource "azurerm_monitor_metric_alert" "response_alert_{{ group.id }}_{{ subgroup }}" {
name = "bandwidth-out-high-${module.label_{{ group.id }}.id}-sub{{ subgroup }}"
resource_group_name = data.azurerm_resource_group.this.name
scopes = [azurerm_cdn_profile.profile_{{ group.id }}_{{ subgroup }}.id]
description = "Action will be triggered when response size is too high."
criteria {
metric_namespace = "Microsoft.Cdn/profiles"
metric_name = "ResponseSize"
@ -112,26 +110,26 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
operator = "GreaterThan"
threshold = 21474836481
}
window_size = "PT1H"
}
{% endfor %}
{% endfor %}
{% for proxy in proxies %}
resource "azurerm_cdn_endpoint" "endpoint_{{ proxy.id }}" {
name = "{{ proxy.slug }}"
profile_name = azurerm_cdn_profile.profile_{{ proxy.origin.group.id }}_{{ proxy.psg }}.name
location = "{{ azure_location }}"
resource_group_name = data.azurerm_resource_group.this.name
origin_host_header = "{{ proxy.origin.domain_name }}"
origin {
name = "upstream"
host_name = "{{ proxy.origin.domain_name }}"
}
global_delivery_rule {
modify_request_header_action {
action = "Append"
@ -140,16 +138,16 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
}
}
}
resource "azurerm_monitor_diagnostic_setting" "diagnostic_{{ proxy.id }}" {
name = "cdn-diagnostics"
target_resource_id = azurerm_cdn_endpoint.endpoint_{{ proxy.id }}.id
storage_account_id = azurerm_storage_account.this.id
log {
category = "CoreAnalytics"
enabled = true
retention_policy {
enabled = true
days = 90
@ -162,7 +160,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
def import_state(self, state: Optional[Any]) -> None:
proxies = Proxy.query.filter(
Proxy.provider == self.provider,
Proxy.destroyed == None
Proxy.destroyed.is_(None)
).all()
for proxy in proxies:
proxy.url = f"https://{proxy.slug}.azureedge.net"

10
app/terraform/proxy/cloudfront.py
View file

@ -24,13 +24,13 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
}
}
}
provider "aws" {
access_key = "{{ aws_access_key }}"
secret_key = "{{ aws_secret_key }}"
region = "us-east-2"
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
@ -39,7 +39,7 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
tenant = "{{ group.group_name }}"
label_order = ["namespace", "tenant", "name", "attributes"]
}
module "log_bucket_{{ group.id }}" {
source = "cloudposse/s3-log-storage/aws"
version = "0.28.0"
@ -51,12 +51,12 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
glacier_transition_days = 60
expiration_days = 90
}
resource "aws_sns_topic" "alarms_{{ group.id }}" {
name = "${module.label_{{ group.id }}.id}-cloudfront-alarms"
}
{% endfor %}
{% for proxy in proxies %}
module "cloudfront_{{ proxy.id }}" {
source = "sr2c/bc-proxy/aws"

6
app/terraform/proxy/fastly.py
View file

@ -61,7 +61,7 @@ module "log_bucket_{{ group.id }}" {
{% if group.id == 3 %}
resource "fastly_service_vcl" "service_{{ group.id }}" {
name = module.label_{{ group.id }}.id
{% for origin in group.origins %}
{% for proxy in origin.proxies %}
{% if proxy.destroyed == None and proxy.provider == "fastly" %}
@ -71,7 +71,7 @@ resource "fastly_service_vcl" "service_{{ group.id }}" {
}
{% endif %}
{% endfor %}
backend {
address = "{{ origin.domain_name }}"
name = "{{ origin.description }}"
@ -110,7 +110,7 @@ def create_missing_proxies():
def destroy_expired_proxies():
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
proxies = Proxy.query.filter(
Proxy.destroyed == None,
Proxy.destroyed.is_(None),
Proxy.provider == "fastly",
Proxy.deprecated < cutoff
).all()

2
setup.cfg Normal file
View file

@ -0,0 +1,2 @@
[flake8]
ignore = E501