jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

automation: establish an automation framework

Browse source
This commit is contained in:
Iain Learmonth 2022-05-08 17:20:04 +01:00
parent 1b53bf451c
commit 8abe5d60fa
31 changed files with 586 additions and 274 deletions
Download patch file Download diff file Expand all files Collapse all files

2
app/cli/__main__.py
View file

@ -3,6 +3,7 @@ import logging
import sys
from os.path import basename
from app.cli.automate import AutomateCliHandler
from app.cli.db import DbCliHandler
from app.cli.list import ListCliHandler
@ -13,6 +14,7 @@ def parse_args(argv):
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbose", help="increase logging verbosity", action="store_true")
subparsers = parser.add_subparsers(title="command", help="command to run")
AutomateCliHandler.add_subparser_to(subparsers)
DbCliHandler.add_subparser_to(subparsers)
ListCliHandler.add_subparser_to(subparsers)
args = parser.parse_args(argv[1:])

117
app/cli/automate.py Normal file
View file

@ -0,0 +1,117 @@
import argparse
import datetime
import json
import logging
from app import app
from app.extensions import db
from app.models.automation import Automation, AutomationState, AutomationLogs
from app.terraform import BaseAutomation
from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation
from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation
from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation
from app.terraform.bridge.aws import BridgeAWSAutomation
from app.terraform.bridge.gandi import BridgeGandiAutomation
from app.terraform.bridge.hcloud import BridgeHcloudAutomation
from app.terraform.bridge.ovh import BridgeOvhAutomation
from app.terraform.list.github import ListGithubAutomation
from app.terraform.list.gitlab import ListGitlabAutomation
from app.terraform.list.s3 import ListS3Automation
from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
jobs = {
x.short_name: x
for x in [
AlarmProxyAzureCdnAutomation,
AlarmProxyCloudfrontAutomation,
AlarmProxyHTTPStatusAutomation,
BridgeAWSAutomation,
BridgeGandiAutomation,
BridgeHcloudAutomation,
BridgeOvhAutomation,
ListGithubAutomation,
ListGitlabAutomation,
ListS3Automation,
ProxyAzureCdnAutomation,
ProxyCloudfrontAutomation
]
}
def run_all(**kwargs):
for job in jobs.values():
run_job(job, **kwargs)
def run_job(job: BaseAutomation, *, force: bool = False, ignore_schedule: bool = False):
automation = Automation.query.filter(Automation.short_name == job.short_name).first()
if automation is None:
automation = Automation()
automation.short_name = job.short_name
automation.description = job.description
automation.enabled = True
automation.next_is_full = False
automation.added = datetime.datetime.utcnow()
automation.updated = automation.added
db.session.add(automation)
else:
if automation.state == AutomationState.RUNNING and not force:
logging.warning("Not running an already running automation")
return
if not ignore_schedule and not force:
if automation.next_run is not None and automation.next_run > datetime.datetime.utcnow():
logging.warning("Not time to run this job yet")
return
if not automation.enabled and not force:
db.session.rollback()
logging.warning(f"job {job.short_name} is disabled and --force not specified")
return
automation.state = AutomationState.RUNNING
db.session.commit()
job = job()
try:
success, logs = job.automate()
except Exception as e:
success = False
logs = repr(e)
if success:
automation.state = AutomationState.IDLE
automation.next_run = datetime.datetime.utcnow() + datetime.timedelta(minutes=7)
else:
automation.state = AutomationState.ERROR
automation.enabled = False
automation.next_run = None
log = AutomationLogs()
log.automation_id = automation.id
log.added = datetime.datetime.utcnow()
log.updated = datetime.datetime.utcnow()
log.logs = json.dumps(logs)
db.session.add(log)
automation.last_run = datetime.datetime.utcnow()
db.session.commit()
class AutomateCliHandler:
@classmethod
def add_subparser_to(cls, subparsers: argparse._SubParsersAction) -> None:
parser = subparsers.add_parser("automate", help="automation operations")
parser.add_argument("-a", "--all", dest="all", help="run all automation jobs", action="store_true")
parser.add_argument("-j", "--job", dest="job", choices=sorted(jobs.keys()),
help="run a specific automation job")
parser.add_argument("--force", help="run job even if disabled and it's not time yet", action="store_true")
parser.add_argument("--ignore-schedule", help="run job even if it's not time yet", action="store_true")
parser.set_defaults(cls=cls)
def __init__(self, args):
self.args = args
def run(self):
with app.app_context():
if self.args.job:
run_job(jobs[self.args.job], force=self.args.force, ignore_schedule=self.args.ignore_schedule)
elif self.args.all:
run_all(force=self.args.force, ignore_schedule=self.args.ignore_schedule)
else:
logging.error("No action requested")

34
app/models/automation.py Normal file
View file

@ -0,0 +1,34 @@
import datetime
import enum
from app.extensions import db
from app.models import AbstractConfiguration, AbstractResource
class AutomationState(enum.Enum):
IDLE = 0
RUNNING = 1
ERROR = 3
class Automation(AbstractConfiguration):
short_name = db.Column(db.String(25), nullable=False)
state = db.Column(db.Enum(AutomationState), default=AutomationState.IDLE, nullable=False)
enabled = db.Column(db.Boolean, nullable=False)
last_run = db.Column(db.DateTime(), nullable=True)
next_run = db.Column(db.DateTime(), nullable=True)
next_is_full = db.Column(db.Boolean(), nullable=False)
logs = db.relationship("AutomationLogs", back_populates="automation")
def kick(self):
self.enabled = True
self.next_run = datetime.datetime.utcnow()
self.updated = datetime.datetime.utcnow()
class AutomationLogs(AbstractResource):
automation_id = db.Column(db.Integer, db.ForeignKey(Automation.id), nullable=False)
logs = db.Column(db.Text)
automation = db.relationship("Automation", back_populates="logs")

3
app/portal/__init__.py
View file

@ -8,6 +8,7 @@ from app.models.alarms import Alarm
from app import Origin, Proxy
from app.models.base import Group, MirrorList
from app.portal.forms import LifecycleForm, NewMirrorListForm
from app.portal.automation import bp as automation
from app.portal.bridgeconf import bp as bridgeconf
from app.portal.bridge import bp as bridge
from app.portal.group import bp as group
@ -17,7 +18,7 @@ from app.portal.proxy import bp as proxy
from app.portal.util import response_404, view_lifecycle
portal = Blueprint("portal", __name__, template_folder="templates", static_folder="static")
portal.register_blueprint(automation, url_prefix="/automation")
portal.register_blueprint(bridgeconf, url_prefix="/bridgeconf")
portal.register_blueprint(bridge, url_prefix="/bridge")
portal.register_blueprint(group, url_prefix="/group")

70
app/portal/automation.py Normal file
View file

@ -0,0 +1,70 @@
from datetime import datetime
from flask import render_template, flash, Response, Blueprint
from flask_wtf import FlaskForm
from sqlalchemy import exc
from wtforms import SubmitField, BooleanField
from wtforms.validators import DataRequired
from app.extensions import db
from app.models.automation import Automation
from app.portal.util import view_lifecycle, response_404
bp = Blueprint("automation", __name__)
class EditAutomationForm(FlaskForm):
enabled = BooleanField('Enabled')
submit = SubmitField('Save Changes')
@bp.route("/list")
def automation_list():
automations = Automation.query.filter(
Automation.destroyed == None).order_by(Automation.description).all()
return render_template("list.html.j2",
section="automation",
title="Automation Jobs",
item="automation",
items=automations)
@bp.route('/edit/<automation_id>', methods=['GET', 'POST'])
def automation_edit(automation_id):
automation = Automation.query.filter(Automation.id == automation_id).first()
if automation is None:
return Response(render_template("error.html.j2",
section="automation",
header="404 Automation Job Not Found",
message="The requested automation job could not be found."),
status=404)
form = EditAutomationForm(enabled=automation.enabled)
if form.validate_on_submit():
automation.enabled = form.enabled.data
automation.updated = datetime.utcnow()
try:
db.session.commit()
flash("Saved changes to bridge configuration.", "success")
except exc.SQLAlchemyError:
flash("An error occurred saving the changes to the bridge configuration.", "danger")
return render_template("automation.html.j2",
section="automation",
automation=automation, form=form)
@bp.route("/kick/<automation_id>", methods=['GET', 'POST'])
def automation_kick(automation_id: int):
automation = Automation.query.filter(
Automation.id == automation_id,
Automation.destroyed == None).first()
if automation is None:
return response_404("The requested bridge configuration could not be found.")
return view_lifecycle(
header=f"Kick automation timer?",
message=automation.description,
success_view="portal.automation.automation_list",
success_message="This automation job will next run within 1 minute.",
section="automation",
resource=automation,
action="kick"
)

16
app/portal/templates/automation.html.j2 Normal file
View file

@ -0,0 +1,16 @@
{% extends "base.html.j2" %}
{% from 'bootstrap5/form.html' import render_form %}
{% from "tables.html.j2" import automation_logs_table %}
{% block content %}
<h1 class="h2 mt-3">Automation Job</h1>
<h2 class="h3">{{ automation.description }} ({{ automation.short_name }})</h2>
<div style="border: 1px solid #666;" class="p-3">
{{ render_form(form) }}
</div>
<h3>Logs</h3>
{{ automation_logs_table(automation.logs) }}
{% endblock %}

6
app/portal/templates/base.html.j2
View file

@ -128,6 +128,12 @@
<span>Monitoring</span>
</h6>
<ul class="nav flex-column">
<li class="nav-item">
<a class="nav-link{% if section == "automation" %} active{% endif %}"
href="{{ url_for("portal.automation.automation_list") }}">
Automation
</a>
</li>
<li class="nav-item">
<a class="nav-link{% if section == "alarm" %} active{% endif %}"
href="{{ url_for("portal.view_alarms") }}">

6
app/portal/templates/list.html.j2
View file

@ -1,5 +1,5 @@
{% extends "base.html.j2" %}
{% from "tables.html.j2" import bridgeconfs_table, bridges_table,
{% from "tables.html.j2" import automations_table, bridgeconfs_table, bridges_table,
groups_table, mirrorlists_table, origins_table, origin_onion_table,
onions_table, proxies_table %}
@ -8,7 +8,9 @@
{% if new_link %}
<a href="{{ new_link }}" class="btn btn-success">Create new {{ item }}</a>
{% endif %}
{% if item == "bridge configuration" %}
{% if item == "automation" %}
{{ automations_table(items) }}
{% elif item == "bridge configuration" %}
{{ bridgeconfs_table(items) }}
{% elif item == "bridge" %}
{{ bridges_table(items) }}

63
app/portal/templates/tables.html.j2
View file

@ -26,6 +26,67 @@
<div class="alert alert-danger">Not implemented yet.</div>
{% endmacro %}
{% macro automations_table(automations) %}
<div class="table-responsive">
<table class="table table-striped table-sm">
<thead>
<tr>
<th scope="col">Description</th>
<th scope="col">Status</th>
<th scope="col">Enabled</th>
<th scope="col">Last Run</th>
<th scope="col">Next Run</th>
<th scope="col">Actions</th>
</tr>
</thead>
<tbody>
{% for automation in automations %}
<tr>
<td title="{{ automation.short_name }}">{{ automation.description }}</td>
<td title="{{ automation.state.name }}">
{% if automation.state.name == "IDLE" %}
🕰️
{% elif automation.state.name == "RUNNING" %}
🏃
{% else %}
💥
{% endif %}
</td>
<td>{% if automation.enabled %}✅{% else %}❌{% endif %}</td>
<td>{{ automation.last_run | format_datetime }}</td>
<td>{{ automation.next_run | format_datetime }}</td>
<td>
<a href="{{ url_for("portal.automation.automation_edit", automation_id=automation.id) }}" class="btn btn-primary btn-sm">View/Edit</a>
<a href="{{ url_for("portal.automation.automation_kick", automation_id=automation.id) }}" class="btn btn-success btn-sm">Kick Timer</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endmacro %}
{% macro automation_logs_table(logs) %}
<div class="table-responsive">
<table class="table table-striped table-sm">
<thead>
<tr>
<th scope="col">Log Created</th>
<th scope="col">Log</th>
</tr>
</thead>
<tbody>
{% for log in logs %}
<tr>
<td>{{ log.added | format_datetime }}</td>
<td>{{ log.logs }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endmacro %}
{% macro groups_table(groups) %}
<div class="table-responsive">
<table class="table table-striped table-sm">
@ -84,6 +145,8 @@
<td>
<a href="{{ url_for("portal.origin.origin_edit", origin_id=origin.id) }}"
class="btn btn-primary btn-sm">View/Edit</a>
<a href="{{ url_for("portal.origin.origin_destroy", origin_id=origin.id) }}"
class="btn btn-danger btn-sm">Destroy</a>
</td>
</tr>
{% endif %}

8
app/portal/util.py
View file

@ -20,11 +20,19 @@ def view_lifecycle(*,
resource: AbstractResource,
action: str):
form = LifecycleForm()
if action == "destroy":
form.submit.render_kw = {"class": "btn btn-danger"}
elif action == "deprecate":
form.submit.render_kw = {"class": "btn btn-warning"}
elif action == "kick":
form.submit.render_kw = {"class": "btn btn-success"}
if form.validate_on_submit():
if action == "destroy":
resource.destroy()
elif action == "deprecate":
resource.deprecate(reason="manual")
elif action == "kick":
resource.kick()
else:
flash("Unknown action")
return redirect(url_for("portal.portal_home"))

3
app/terraform/__init__.py
View file

@ -1,4 +1,5 @@
import os
from typing import Tuple
from app import app
@ -10,7 +11,7 @@ class BaseAutomation:
the portal system.
"""
def automate(self):
def automate(self, full: bool = False) -> Tuple[bool, str]:
raise NotImplementedError()
def working_directory(self, filename=None) -> str:

0
app/terraform/alarms/__init__.py Normal file
View file

36
app/terraform/alarms/proxy_azure_cdn.py Normal file
View file

@ -0,0 +1,36 @@
from azure.identity import ClientSecretCredential
from azure.mgmt.alertsmanagement import AlertsManagementClient
from app import app
from app.alarms import get_proxy_alarm
from app.models.alarms import AlarmState
from app.models.mirrors import Proxy
from app.terraform import BaseAutomation
class AlarmProxyAzureCdnAutomation(BaseAutomation):
short_name = "monitor_proxy_azure_cdn"
description = "Import alarms for Azure CDN proxies"
def automate(self):
credential = ClientSecretCredential(
tenant_id=app.config['AZURE_TENANT_ID'],
client_id=app.config['AZURE_CLIENT_ID'],
client_secret=app.config['AZURE_CLIENT_SECRET'])
client = AlertsManagementClient(
credential,
app.config['AZURE_SUBSCRIPTION_ID']
)
firing = [x.name[len("bandwidth-out-high-bc-"):]
for x in client.alerts.get_all()
if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
for proxy in Proxy.query.filter(
Proxy.provider == "azure_cdn",
Proxy.destroyed == None
):
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
if proxy.origin.group.group_name.lower() not in firing:
alarm.update_state(AlarmState.OK, "Azure monitor alert not firing")
else:
alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing")
return True, []

60
app/terraform/alarms/proxy_cloudfront.py Normal file
View file

@ -0,0 +1,60 @@
import datetime
import boto3
from app import app
from app.alarms import get_proxy_alarm
from app.extensions import db
from app.models.mirrors import Proxy
from app.models.alarms import AlarmState, Alarm
from app.terraform import BaseAutomation
class AlarmProxyCloudfrontAutomation(BaseAutomation):
short_name = "monitor_proxy_cloudfront"
description = "Import alarms for AWS CloudFront proxies"
def automate(self):
cloudwatch = boto3.client('cloudwatch',
aws_access_key_id=app.config['AWS_ACCESS_KEY'],
aws_secret_access_key=app.config['AWS_SECRET_KEY'],
region_name='us-east-2')
dist_paginator = cloudwatch.get_paginator('describe_alarms')
page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-")
for page in page_iterator:
for cw_alarm in page['MetricAlarms']:
dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):]
proxy = Proxy.query.filter(Proxy.slug == dist_id).first()
if proxy is None:
print("Skipping unknown proxy " + dist_id)
continue
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
if cw_alarm['StateValue'] == "OK":
alarm.update_state(AlarmState.OK, "CloudWatch alarm OK")
elif cw_alarm['StateValue'] == "ALARM":
alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM")
else:
alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}")
alarm = Alarm.query.filter(
Alarm.alarm_type == "cloudfront-quota"
).first()
if alarm is None:
alarm = Alarm()
alarm.target = "service/cloudfront"
alarm.alarm_type = "cloudfront-quota"
alarm.state_changed = datetime.datetime.utcnow()
db.session.add(alarm)
alarm.last_updated = datetime.datetime.utcnow()
deployed_count = len(Proxy.query.filter(
Proxy.destroyed == None).all())
old_state = alarm.alarm_state
if deployed_count > 370:
alarm.alarm_state = AlarmState.CRITICAL
elif deployed_count > 320:
alarm.alarm_state = AlarmState.WARNING
else:
alarm.alarm_state = AlarmState.OK
if alarm.alarm_state != old_state:
alarm.state_changed = datetime.datetime.utcnow()
db.session.commit()
return True, []

64
app/terraform/alarms/proxy_http_status.py Normal file
View file

@ -0,0 +1,64 @@
from typing import Tuple
import requests
from app.extensions import db
from app.models.alarms import Alarm, AlarmState
from app.models.mirrors import Proxy
from app.terraform import BaseAutomation
def set_http_alarm(proxy_id: int, state: AlarmState, text: str):
alarm = Alarm.query.filter(
Alarm.proxy_id == proxy_id,
Alarm.alarm_type == "http-status"
).first()
if alarm is None:
alarm = Alarm()
alarm.proxy_id = proxy_id
alarm.alarm_type = "http-status"
alarm.target = "proxy"
db.session.add(alarm)
alarm.update_state(state, text)
class AlarmProxyHTTPStatusAutomation(BaseAutomation):
short_name = "alarm_http_status"
description = "Check all deployed proxies for HTTP status code"
def automate(self, full: bool = False) -> Tuple[bool, str]:
proxies = Proxy.query.filter(
Proxy.destroyed == None
)
for proxy in proxies:
try:
if proxy.url is None:
continue
r = requests.get(proxy.url,
allow_redirects=False,
timeout=5)
r.raise_for_status()
if r.is_redirect:
set_http_alarm(
proxy.id,
AlarmState.CRITICAL,
f"{r.status_code} {r.reason}"
)
else:
set_http_alarm(
proxy.id,
AlarmState.OK,
f"{r.status_code} {r.reason}"
)
except (requests.ConnectionError, requests.Timeout):
set_http_alarm(
proxy.id,
AlarmState.CRITICAL,
f"Connection failure")
except requests.HTTPError:
set_http_alarm(
proxy.id,
AlarmState.CRITICAL,
f"{r.status_code} {r.reason}"
)
return True, []

18
app/terraform/bridge/__init__.py
View file

@ -1,14 +1,14 @@
import datetime
from typing import Iterable
from typing import Iterable, Optional, Any
from app import app
from app.extensions import db
from app.models.bridges import BridgeConf, Bridge
from app.models.base import Group
from app.terraform import BaseAutomation
from app.terraform.terraform import TerraformAutomation
class BridgeAutomation(BaseAutomation):
class BridgeAutomation(TerraformAutomation):
def create_missing(self):
bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter(
BridgeConf.provider == self.provider,
@ -45,8 +45,12 @@ class BridgeAutomation(BaseAutomation):
bridge.destroy()
db.session.commit()
def generate_terraform(self):
self.write_terraform_config(
def tf_prehook(self) -> Optional[Any]:
self.create_missing()
self.destroy_expired()
def tf_generate(self):
self.tf_write(
self.template,
groups=Group.query.all(),
bridgeconfs=BridgeConf.query.filter(
@ -60,8 +64,8 @@ class BridgeAutomation(BaseAutomation):
}
)
def import_terraform(self):
outputs = self.terraform_output()
def tf_posthook(self, *, prehook_result: Any = None) -> None:
outputs = self.tf_output()
for output in outputs:
if output.startswith('bridge_hashed_fingerprint_'):
parts = outputs[output]['value'].split(" ")

17
app/terraform/bridge/aws.py
View file

@ -1,9 +1,9 @@
from app import app
from app.terraform.bridge import BridgeAutomation
class BridgeAWSAutomation(BridgeAutomation):
short_name = "bridge_aws"
description = "Deploy Tor bridges on AWS Lightsail"
provider = "aws"
template_parameters = [
@ -67,18 +67,3 @@ class BridgeAWSAutomation(BridgeAutomation):
{% endfor %}
{% endfor %}
"""
def automate():
auto = BridgeAWSAutomation()
auto.destroy_expired()
auto.create_missing()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply()
auto.import_terraform()
if __name__ == "__main__":
with app.app_context():
automate()

17
app/terraform/bridge/gandi.py
View file

@ -1,9 +1,9 @@
from app import app
from app.terraform.bridge import BridgeAutomation
class BridgeGandiAutomation(BridgeAutomation):
short_name = "bridge_gandi"
description = "Deploy Tor bridges on GandiCloud VPS"
provider = "gandi"
template_parameters = [
@ -78,18 +78,3 @@ class BridgeGandiAutomation(BridgeAutomation):
{% endfor %}
{% endfor %}
"""
def automate():
auto = BridgeGandiAutomation()
auto.destroy_expired()
auto.create_missing()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply()
auto.import_terraform()
if __name__ == "__main__":
with app.app_context():
automate()

17
app/terraform/bridge/hcloud.py
View file

@ -1,9 +1,9 @@
from app import app
from app.terraform.bridge import BridgeAutomation
class BridgeHcloudAutomation(BridgeAutomation):
short_name = "bridge_hcloud"
description = "Deploy Tor bridges on Hetzner Cloud"
provider = "hcloud"
template_parameters = [
@ -81,18 +81,3 @@ class BridgeHcloudAutomation(BridgeAutomation):
{% endfor %}
{% endfor %}
"""
def automate():
auto = BridgeHcloudAutomation()
auto.destroy_expired()
auto.create_missing()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply()
auto.import_terraform()
if __name__ == "__main__":
with app.app_context():
automate()

17
app/terraform/bridge/ovh.py
View file

@ -1,9 +1,9 @@
from app import app
from app.terraform.bridge import BridgeAutomation
class BridgeOvhAutomation(BridgeAutomation):
short_name = "bridge_ovh"
description = "Deploy Tor bridges on OVH Public Cloud"
provider = "ovh"
template_parameters = [
@ -104,18 +104,3 @@ class BridgeOvhAutomation(BridgeAutomation):
{% endfor %}
{% endfor %}
"""
def automate():
auto = BridgeOvhAutomation()
auto.destroy_expired()
auto.create_missing()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply()
auto.import_terraform()
if __name__ == "__main__":
with app.app_context():
automate()

8
app/terraform/list/__init__.py
View file

@ -5,12 +5,12 @@ from app.lists.mirror_mapping import mirror_mapping
from app.lists.bc2 import mirror_sites
from app.lists.bridgelines import bridgelines
from app.models.base import MirrorList
from app.terraform import BaseAutomation
from app.terraform.terraform import TerraformAutomation
class ListAutomation(BaseAutomation):
def generate_terraform(self):
self.write_terraform_config(
class ListAutomation(TerraformAutomation):
def tf_generate(self):
self.tf_write(
self.template,
lists=MirrorList.query.filter(
MirrorList.destroyed == None,

10
app/terraform/list/github.py
View file

@ -1,9 +1,9 @@
from app import app
from app.terraform.list import ListAutomation
class ListGithubAutomation(ListAutomation):
short_name = "list_github"
description = "Update mirror lists in GitHub repositories"
provider = "github"
template_parameters = [
@ -45,11 +45,3 @@ class ListGithubAutomation(ListAutomation):
}
{% endfor %}
"""
if __name__ == "__main__":
with app.app_context():
auto = ListGithubAutomation()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply()

10
app/terraform/list/gitlab.py
View file

@ -1,9 +1,9 @@
from app import app
from app.terraform.list import ListAutomation
class ListGitlabAutomation(ListAutomation):
short_name = "list_gitlab"
description = "Update mirror lists in GitLab repositories"
provider = "gitlab"
template_parameters = [
@ -44,11 +44,3 @@ class ListGitlabAutomation(ListAutomation):
{% endfor %}
"""
if __name__ == "__main__":
with app.app_context():
auto = ListGitlabAutomation()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply()

12
app/terraform/list/s3.py
View file

@ -1,9 +1,9 @@
from app import app
from app.terraform.list import ListAutomation
class ListGithubAutomation(ListAutomation):
class ListS3Automation(ListAutomation):
short_name = "list_s3"
description = "Update mirror lists in AWS S3 buckets"
provider = "s3"
template_parameters = [
@ -36,11 +36,3 @@ class ListGithubAutomation(ListAutomation):
}
{% endfor %}
"""
if __name__ == "__main__":
with app.app_context():
auto = ListGithubAutomation()
auto.generate_terraform()
auto.terraform_init()
auto.terraform_apply()

4
app/terraform/proxy/__init__.py
View file

@ -37,6 +37,8 @@ class ProxyAutomation(TerraformAutomation):
for group in groups:
subgroup = 0
for origin in group.origins:
if origin.destroyed is not None:
continue
while True:
if subgroups[group.id][subgroup] >= self.subgroup_max:
subgroup += 1
@ -87,7 +89,7 @@ class ProxyAutomation(TerraformAutomation):
self.deprecate_orphaned_proxies()
self.destroy_expired_proxies()
def tf_posthook(self):
def tf_posthook(self, *, prehook_result):
self.import_state(self.tf_show())
def tf_generate(self):

37
app/terraform/proxy/azure_cdn.py
View file

@ -1,16 +1,11 @@
from azure.identity import ClientSecretCredential
from azure.mgmt.alertsmanagement import AlertsManagementClient
from app import app
from app.alarms import get_proxy_alarm
from app.extensions import db
from app.models.mirrors import Proxy
from app.models.alarms import AlarmState
from app.terraform.proxy import ProxyAutomation
class ProxyAzureCdnAutomation(ProxyAutomation):
short_name = "proxy_azure_cdn"
description = "Deploy proxies to Azure CDN"
provider = "azure_cdn"
subgroup_max = 25
parallelism = 1
@ -170,33 +165,3 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
for proxy in proxies:
proxy.url = f"https://{proxy.slug}.azureedge.net"
db.session.commit()
def import_monitor_alerts():
credential = ClientSecretCredential(
tenant_id=app.config['AZURE_TENANT_ID'],
client_id=app.config['AZURE_CLIENT_ID'],
client_secret=app.config['AZURE_CLIENT_SECRET'])
client = AlertsManagementClient(
credential,
app.config['AZURE_SUBSCRIPTION_ID']
)
firing = [x.name[len("bandwidth-out-high-bc-"):]
for x in client.alerts.get_all()
if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
for proxy in Proxy.query.filter(
Proxy.provider == "azure_cdn",
Proxy.destroyed == None
):
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
if proxy.origin.group.group_name.lower() not in firing:
alarm.update_state(AlarmState.OK, "Azure monitor alert not firing")
else:
alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing")
if __name__ == "__main__":
with app.app_context():
auto = ProxyAzureCdnAutomation()
auto.automate()
import_monitor_alerts()

58
app/terraform/proxy/cloudfront.py
View file

@ -1,17 +1,13 @@
import datetime
import boto3
from app import app
from app.alarms import get_proxy_alarm
from app.extensions import db
from app.models.alarms import Alarm, AlarmState
from app.models.mirrors import Proxy
from app.terraform.proxy import ProxyAutomation
class ProxyCloudfrontAutomation(ProxyAutomation):
short_name = "proxy_cloudfront"
description = "Deploy proxies to AWS CloudFront"
provider = "cloudfront"
template_parameters = [
@ -87,55 +83,3 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
proxy.terraform_updated = datetime.datetime.utcnow()
break
db.session.commit()
def import_cloudwatch_alarms():
cloudwatch = boto3.client('cloudwatch',
aws_access_key_id=app.config['AWS_ACCESS_KEY'],
aws_secret_access_key=app.config['AWS_SECRET_KEY'],
region_name='us-east-2')
dist_paginator = cloudwatch.get_paginator('describe_alarms')
page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-")
for page in page_iterator:
for cw_alarm in page['MetricAlarms']:
dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):]
proxy = Proxy.query.filter(Proxy.slug == dist_id).first()
if proxy is None:
print("Skipping unknown proxy " + dist_id)
continue
alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
if cw_alarm['StateValue'] == "OK":
alarm.update_state(AlarmState.OK, "CloudWatch alarm OK")
elif cw_alarm['StateValue'] == "ALARM":
alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM")
else:
alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}")
alarm = Alarm.query.filter(
Alarm.alarm_type == "cloudfront-quota"
).first()
if alarm is None:
alarm = Alarm()
alarm.target = "service/cloudfront"
alarm.alarm_type = "cloudfront-quota"
alarm.state_changed = datetime.datetime.utcnow()
db.session.add(alarm)
alarm.last_updated = datetime.datetime.utcnow()
deployed_count = len(Proxy.query.filter(
Proxy.destroyed == None).all())
old_state = alarm.alarm_state
if deployed_count > 370:
alarm.alarm_state = AlarmState.CRITICAL
elif deployed_count > 320:
alarm.alarm_state = AlarmState.WARNING
else:
alarm.alarm_state = AlarmState.OK
if alarm.alarm_state != old_state:
alarm.state_changed = datetime.datetime.utcnow()
db.session.commit()
if __name__ == "__main__":
with app.app_context():
auto = ProxyCloudfrontAutomation()
auto.automate()
import_cloudwatch_alarms()

62
app/terraform/proxy_check.py
View file

@ -1,62 +0,0 @@
import requests
from app import app
from app.extensions import db
from app.models.alarms import Alarm, AlarmState
from app.models.mirrors import Proxy
def set_http_alarm(proxy_id: int, state: AlarmState, text: str):
alarm = Alarm.query.filter(
Alarm.proxy_id == proxy_id,
Alarm.alarm_type == "http-status"
).first()
if alarm is None:
alarm = Alarm()
alarm.proxy_id = proxy_id
alarm.alarm_type = "http-status"
alarm.target = "proxy"
db.session.add(alarm)
alarm.update_state(state, text)
def check_http():
proxies = Proxy.query.filter(
Proxy.destroyed == None
)
for proxy in proxies:
try:
if proxy.url is None:
continue
r = requests.get(proxy.url,
allow_redirects=False,
timeout=5)
r.raise_for_status()
if r.is_redirect:
set_http_alarm(
proxy.id,
AlarmState.CRITICAL,
f"{r.status_code} {r.reason}"
)
else:
set_http_alarm(
proxy.id,
AlarmState.OK,
f"{r.status_code} {r.reason}"
)
except (requests.ConnectionError, requests.Timeout):
set_http_alarm(
proxy.id,
AlarmState.CRITICAL,
f"Connection failure")
except requests.HTTPError:
set_http_alarm(
proxy.id,
AlarmState.CRITICAL,
f"{r.status_code} {r.reason}"
)
if __name__ == "__main__":
with app.app_context():
check_http()

27
app/terraform/terraform.py
View file

@ -1,6 +1,6 @@
import json
import subprocess
from typing import Dict, Any, Optional
from typing import Any, Dict, List, Optional, Tuple
import jinja2
@ -18,20 +18,27 @@ class TerraformAutomation(BaseAutomation):
Default parallelism for remote API calls.
"""
def automate(self):
self.tf_prehook()
def automate(self, full: bool = False):
prehook_result = self.tf_prehook()
self.tf_generate()
self.tf_init()
self.tf_apply(refresh=False)
self.tf_posthook()
returncode, logs = self.tf_apply(refresh=full)
self.tf_posthook(prehook_result=prehook_result)
return True if returncode == 0 else False, logs
def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None):
def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None) -> Tuple[int, List[Dict[str, Any]]]:
if not parallelism:
parallelism = self.parallelism
subprocess.run(
tf = subprocess.run(
['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve',
f'-parallelism={str(parallelism)}'],
cwd=self.working_directory())
f'-parallelism={str(parallelism)}', '-json'],
cwd=self.working_directory(),
stdout=subprocess.PIPE)
logs = []
for line in tf.stdout.decode('utf-8').split('\n'):
if line.strip():
logs.append(json.loads(line))
return tf.returncode, logs
def tf_generate(self):
raise NotImplementedError()
@ -56,7 +63,7 @@ class TerraformAutomation(BaseAutomation):
# more like JSON-ND, task is to figure out how to yield those records
# as plan runs, the same is probably also true for apply
def tf_posthook(self, prehook_result: Any = None) -> None:
def tf_posthook(self, *, prehook_result: Any = None) -> None:
"""
This hook function is called as part of normal automation, after the
completion of :func:`tf_apply`.

4
docs/admin/external.rst
View file

@ -70,7 +70,9 @@ The expiry can be set according to your threat model.
GitLab will send an email warning to token owners 7 days before expiry
allowing you to generate a new token and update your configuration.
Your access token will need the "read_repository" and "write_repository" scopes.
Your access token will need the "api" scope. Unforunately the "write_repository" scope
only works for Git-over-HTTPS, but the portal uses the API to update mirror lists in
GitLab.
Once you've generated your token, you can add it to your ``config.yaml``:

54
migrations/versions/0a0a65db7f01_add_automations.py Normal file
View file

@ -0,0 +1,54 @@
"""add automations
Revision ID: 0a0a65db7f01
Revises: c3d6e95caa79
Create Date: 2022-05-08 16:24:53.779353
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '0a0a65db7f01'
down_revision = 'c3d6e95caa79'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('automation',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('description', sa.String(length=255), nullable=False),
sa.Column('added', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('destroyed', sa.DateTime(), nullable=True),
sa.Column('short_name', sa.String(length=25), nullable=False),
sa.Column('state', sa.Enum('IDLE', 'RUNNING', 'ERROR', name='automationstate'), nullable=False),
sa.Column('enabled', sa.Boolean(), nullable=False),
sa.Column('last_run', sa.DateTime(), nullable=True),
sa.Column('next_run', sa.DateTime(), nullable=True),
sa.Column('next_is_full', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_automation'))
)
op.create_table('automation_logs',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('added', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('deprecated', sa.DateTime(), nullable=True),
sa.Column('deprecation_reason', sa.String(), nullable=True),
sa.Column('destroyed', sa.DateTime(), nullable=True),
sa.Column('automation_id', sa.Integer(), nullable=False),
sa.Column('logs', sa.Text(), nullable=True),
sa.ForeignKeyConstraint(['automation_id'], ['automation.id'], name=op.f('fk_automation_logs_automation_id_automation')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_automation_logs'))
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('automation_logs')
op.drop_table('automation')
# ### end Alembic commands ###