From 8abe5d60fa91c06eb979ab57559932643a38ac11 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Sun, 8 May 2022 17:20:04 +0100 Subject: [PATCH] automation: establish an automation framework --- app/cli/__main__.py | 2 + app/cli/automate.py | 117 ++++++++++++++++++ app/models/automation.py | 34 +++++ app/portal/__init__.py | 3 +- app/portal/automation.py | 70 +++++++++++ app/portal/templates/automation.html.j2 | 16 +++ app/portal/templates/base.html.j2 | 6 + app/portal/templates/list.html.j2 | 6 +- app/portal/templates/tables.html.j2 | 63 ++++++++++ app/portal/util.py | 8 ++ app/terraform/__init__.py | 3 +- app/terraform/alarms/__init__.py | 0 app/terraform/alarms/proxy_azure_cdn.py | 36 ++++++ app/terraform/alarms/proxy_cloudfront.py | 60 +++++++++ app/terraform/alarms/proxy_http_status.py | 64 ++++++++++ app/terraform/bridge/__init__.py | 18 +-- app/terraform/bridge/aws.py | 17 +-- app/terraform/bridge/gandi.py | 17 +-- app/terraform/bridge/hcloud.py | 17 +-- app/terraform/bridge/ovh.py | 17 +-- app/terraform/list/__init__.py | 8 +- app/terraform/list/github.py | 10 +- app/terraform/list/gitlab.py | 10 +- app/terraform/list/s3.py | 12 +- app/terraform/proxy/__init__.py | 4 +- app/terraform/proxy/azure_cdn.py | 37 +----- app/terraform/proxy/cloudfront.py | 58 +-------- app/terraform/proxy_check.py | 62 ---------- app/terraform/terraform.py | 27 ++-- docs/admin/external.rst | 4 +- .../versions/0a0a65db7f01_add_automations.py | 54 ++++++++ 31 files changed, 586 insertions(+), 274 deletions(-) create mode 100644 app/cli/automate.py create mode 100644 app/models/automation.py create mode 100644 app/portal/automation.py create mode 100644 app/portal/templates/automation.html.j2 create mode 100644 app/terraform/alarms/__init__.py create mode 100644 app/terraform/alarms/proxy_azure_cdn.py create mode 100644 app/terraform/alarms/proxy_cloudfront.py create mode 100644 app/terraform/alarms/proxy_http_status.py delete mode 100644 app/terraform/proxy_check.py create mode 100644 migrations/versions/0a0a65db7f01_add_automations.py diff --git a/app/cli/__main__.py b/app/cli/__main__.py index 87bfcb3..8c0ec6f 100644 --- a/app/cli/__main__.py +++ b/app/cli/__main__.py @@ -3,6 +3,7 @@ import logging import sys from os.path import basename +from app.cli.automate import AutomateCliHandler from app.cli.db import DbCliHandler from app.cli.list import ListCliHandler @@ -13,6 +14,7 @@ def parse_args(argv): parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", help="increase logging verbosity", action="store_true") subparsers = parser.add_subparsers(title="command", help="command to run") + AutomateCliHandler.add_subparser_to(subparsers) DbCliHandler.add_subparser_to(subparsers) ListCliHandler.add_subparser_to(subparsers) args = parser.parse_args(argv[1:]) diff --git a/app/cli/automate.py b/app/cli/automate.py new file mode 100644 index 0000000..ffd7335 --- /dev/null +++ b/app/cli/automate.py @@ -0,0 +1,117 @@ +import argparse +import datetime +import json +import logging + +from app import app +from app.extensions import db +from app.models.automation import Automation, AutomationState, AutomationLogs +from app.terraform import BaseAutomation +from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation +from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation +from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation +from app.terraform.bridge.aws import BridgeAWSAutomation +from app.terraform.bridge.gandi import BridgeGandiAutomation +from app.terraform.bridge.hcloud import BridgeHcloudAutomation +from app.terraform.bridge.ovh import BridgeOvhAutomation +from app.terraform.list.github import ListGithubAutomation +from app.terraform.list.gitlab import ListGitlabAutomation +from app.terraform.list.s3 import ListS3Automation +from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation +from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation + + +jobs = { + x.short_name: x + for x in [ + AlarmProxyAzureCdnAutomation, + AlarmProxyCloudfrontAutomation, + AlarmProxyHTTPStatusAutomation, + BridgeAWSAutomation, + BridgeGandiAutomation, + BridgeHcloudAutomation, + BridgeOvhAutomation, + ListGithubAutomation, + ListGitlabAutomation, + ListS3Automation, + ProxyAzureCdnAutomation, + ProxyCloudfrontAutomation + ] +} + + +def run_all(**kwargs): + for job in jobs.values(): + run_job(job, **kwargs) + + +def run_job(job: BaseAutomation, *, force: bool = False, ignore_schedule: bool = False): + automation = Automation.query.filter(Automation.short_name == job.short_name).first() + if automation is None: + automation = Automation() + automation.short_name = job.short_name + automation.description = job.description + automation.enabled = True + automation.next_is_full = False + automation.added = datetime.datetime.utcnow() + automation.updated = automation.added + db.session.add(automation) + else: + if automation.state == AutomationState.RUNNING and not force: + logging.warning("Not running an already running automation") + return + if not ignore_schedule and not force: + if automation.next_run is not None and automation.next_run > datetime.datetime.utcnow(): + logging.warning("Not time to run this job yet") + return + if not automation.enabled and not force: + db.session.rollback() + logging.warning(f"job {job.short_name} is disabled and --force not specified") + return + automation.state = AutomationState.RUNNING + db.session.commit() + job = job() + try: + success, logs = job.automate() + except Exception as e: + success = False + logs = repr(e) + if success: + automation.state = AutomationState.IDLE + automation.next_run = datetime.datetime.utcnow() + datetime.timedelta(minutes=7) + else: + automation.state = AutomationState.ERROR + automation.enabled = False + automation.next_run = None + log = AutomationLogs() + log.automation_id = automation.id + log.added = datetime.datetime.utcnow() + log.updated = datetime.datetime.utcnow() + log.logs = json.dumps(logs) + db.session.add(log) + automation.last_run = datetime.datetime.utcnow() + db.session.commit() + + +class AutomateCliHandler: + @classmethod + def add_subparser_to(cls, subparsers: argparse._SubParsersAction) -> None: + parser = subparsers.add_parser("automate", help="automation operations") + parser.add_argument("-a", "--all", dest="all", help="run all automation jobs", action="store_true") + parser.add_argument("-j", "--job", dest="job", choices=sorted(jobs.keys()), + help="run a specific automation job") + parser.add_argument("--force", help="run job even if disabled and it's not time yet", action="store_true") + parser.add_argument("--ignore-schedule", help="run job even if it's not time yet", action="store_true") + parser.set_defaults(cls=cls) + + def __init__(self, args): + self.args = args + + def run(self): + with app.app_context(): + if self.args.job: + run_job(jobs[self.args.job], force=self.args.force, ignore_schedule=self.args.ignore_schedule) + elif self.args.all: + run_all(force=self.args.force, ignore_schedule=self.args.ignore_schedule) + else: + logging.error("No action requested") diff --git a/app/models/automation.py b/app/models/automation.py new file mode 100644 index 0000000..bde6542 --- /dev/null +++ b/app/models/automation.py @@ -0,0 +1,34 @@ +import datetime +import enum + +from app.extensions import db +from app.models import AbstractConfiguration, AbstractResource + + +class AutomationState(enum.Enum): + IDLE = 0 + RUNNING = 1 + ERROR = 3 + + +class Automation(AbstractConfiguration): + short_name = db.Column(db.String(25), nullable=False) + state = db.Column(db.Enum(AutomationState), default=AutomationState.IDLE, nullable=False) + enabled = db.Column(db.Boolean, nullable=False) + last_run = db.Column(db.DateTime(), nullable=True) + next_run = db.Column(db.DateTime(), nullable=True) + next_is_full = db.Column(db.Boolean(), nullable=False) + + logs = db.relationship("AutomationLogs", back_populates="automation") + + def kick(self): + self.enabled = True + self.next_run = datetime.datetime.utcnow() + self.updated = datetime.datetime.utcnow() + + +class AutomationLogs(AbstractResource): + automation_id = db.Column(db.Integer, db.ForeignKey(Automation.id), nullable=False) + logs = db.Column(db.Text) + + automation = db.relationship("Automation", back_populates="logs") diff --git a/app/portal/__init__.py b/app/portal/__init__.py index 0154d15..3cbb73f 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -8,6 +8,7 @@ from app.models.alarms import Alarm from app import Origin, Proxy from app.models.base import Group, MirrorList from app.portal.forms import LifecycleForm, NewMirrorListForm +from app.portal.automation import bp as automation from app.portal.bridgeconf import bp as bridgeconf from app.portal.bridge import bp as bridge from app.portal.group import bp as group @@ -17,7 +18,7 @@ from app.portal.proxy import bp as proxy from app.portal.util import response_404, view_lifecycle portal = Blueprint("portal", __name__, template_folder="templates", static_folder="static") - +portal.register_blueprint(automation, url_prefix="/automation") portal.register_blueprint(bridgeconf, url_prefix="/bridgeconf") portal.register_blueprint(bridge, url_prefix="/bridge") portal.register_blueprint(group, url_prefix="/group") diff --git a/app/portal/automation.py b/app/portal/automation.py new file mode 100644 index 0000000..df39003 --- /dev/null +++ b/app/portal/automation.py @@ -0,0 +1,70 @@ +from datetime import datetime + +from flask import render_template, flash, Response, Blueprint +from flask_wtf import FlaskForm +from sqlalchemy import exc +from wtforms import SubmitField, BooleanField +from wtforms.validators import DataRequired + +from app.extensions import db +from app.models.automation import Automation +from app.portal.util import view_lifecycle, response_404 + +bp = Blueprint("automation", __name__) + + +class EditAutomationForm(FlaskForm): + enabled = BooleanField('Enabled') + submit = SubmitField('Save Changes') + + +@bp.route("/list") +def automation_list(): + automations = Automation.query.filter( + Automation.destroyed == None).order_by(Automation.description).all() + return render_template("list.html.j2", + section="automation", + title="Automation Jobs", + item="automation", + items=automations) + + +@bp.route('/edit/', methods=['GET', 'POST']) +def automation_edit(automation_id): + automation = Automation.query.filter(Automation.id == automation_id).first() + if automation is None: + return Response(render_template("error.html.j2", + section="automation", + header="404 Automation Job Not Found", + message="The requested automation job could not be found."), + status=404) + form = EditAutomationForm(enabled=automation.enabled) + if form.validate_on_submit(): + automation.enabled = form.enabled.data + automation.updated = datetime.utcnow() + try: + db.session.commit() + flash("Saved changes to bridge configuration.", "success") + except exc.SQLAlchemyError: + flash("An error occurred saving the changes to the bridge configuration.", "danger") + return render_template("automation.html.j2", + section="automation", + automation=automation, form=form) + + +@bp.route("/kick/", methods=['GET', 'POST']) +def automation_kick(automation_id: int): + automation = Automation.query.filter( + Automation.id == automation_id, + Automation.destroyed == None).first() + if automation is None: + return response_404("The requested bridge configuration could not be found.") + return view_lifecycle( + header=f"Kick automation timer?", + message=automation.description, + success_view="portal.automation.automation_list", + success_message="This automation job will next run within 1 minute.", + section="automation", + resource=automation, + action="kick" + ) diff --git a/app/portal/templates/automation.html.j2 b/app/portal/templates/automation.html.j2 new file mode 100644 index 0000000..1917520 --- /dev/null +++ b/app/portal/templates/automation.html.j2 @@ -0,0 +1,16 @@ +{% extends "base.html.j2" %} +{% from 'bootstrap5/form.html' import render_form %} +{% from "tables.html.j2" import automation_logs_table %} + +{% block content %} +

Automation Job

+

{{ automation.description }} ({{ automation.short_name }})

+ +
+ {{ render_form(form) }} +
+ +

Logs

+ {{ automation_logs_table(automation.logs) }} + +{% endblock %} diff --git a/app/portal/templates/base.html.j2 b/app/portal/templates/base.html.j2 index 52d9910..1db9162 100644 --- a/app/portal/templates/base.html.j2 +++ b/app/portal/templates/base.html.j2 @@ -128,6 +128,12 @@ Monitoring
    +
  • + + Automation + +
  • diff --git a/app/portal/templates/list.html.j2 b/app/portal/templates/list.html.j2 index b7ed4e4..9d80e57 100644 --- a/app/portal/templates/list.html.j2 +++ b/app/portal/templates/list.html.j2 @@ -1,5 +1,5 @@ {% extends "base.html.j2" %} -{% from "tables.html.j2" import bridgeconfs_table, bridges_table, +{% from "tables.html.j2" import automations_table, bridgeconfs_table, bridges_table, groups_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, proxies_table %} @@ -8,7 +8,9 @@ {% if new_link %} Create new {{ item }} {% endif %} - {% if item == "bridge configuration" %} + {% if item == "automation" %} + {{ automations_table(items) }} + {% elif item == "bridge configuration" %} {{ bridgeconfs_table(items) }} {% elif item == "bridge" %} {{ bridges_table(items) }} diff --git a/app/portal/templates/tables.html.j2 b/app/portal/templates/tables.html.j2 index bdeb5ed..9c9939f 100644 --- a/app/portal/templates/tables.html.j2 +++ b/app/portal/templates/tables.html.j2 @@ -26,6 +26,67 @@
    Not implemented yet.
    {% endmacro %} +{% macro automations_table(automations) %} +
    + + + + + + + + + + + + + {% for automation in automations %} + + + + + + + + + {% endfor %} + +
    DescriptionStatusEnabledLast RunNext RunActions
    {{ automation.description }} + {% if automation.state.name == "IDLE" %} + 🕰️ + {% elif automation.state.name == "RUNNING" %} + 🏃 + {% else %} + 💥 + {% endif %} + {% if automation.enabled %}✅{% else %}❌{% endif %}{{ automation.last_run | format_datetime }}{{ automation.next_run | format_datetime }} + View/Edit + Kick Timer +
    +
    +{% endmacro %} + +{% macro automation_logs_table(logs) %} +
    + + + + + + + + + {% for log in logs %} + + + + + {% endfor %} + +
    Log CreatedLog
    {{ log.added | format_datetime }}{{ log.logs }}
    +
    +{% endmacro %} + {% macro groups_table(groups) %}
    @@ -84,6 +145,8 @@ {% endif %} diff --git a/app/portal/util.py b/app/portal/util.py index 9097c26..e188fbf 100644 --- a/app/portal/util.py +++ b/app/portal/util.py @@ -20,11 +20,19 @@ def view_lifecycle(*, resource: AbstractResource, action: str): form = LifecycleForm() + if action == "destroy": + form.submit.render_kw = {"class": "btn btn-danger"} + elif action == "deprecate": + form.submit.render_kw = {"class": "btn btn-warning"} + elif action == "kick": + form.submit.render_kw = {"class": "btn btn-success"} if form.validate_on_submit(): if action == "destroy": resource.destroy() elif action == "deprecate": resource.deprecate(reason="manual") + elif action == "kick": + resource.kick() else: flash("Unknown action") return redirect(url_for("portal.portal_home")) diff --git a/app/terraform/__init__.py b/app/terraform/__init__.py index 36d4ce8..c696137 100644 --- a/app/terraform/__init__.py +++ b/app/terraform/__init__.py @@ -1,4 +1,5 @@ import os +from typing import Tuple from app import app @@ -10,7 +11,7 @@ class BaseAutomation: the portal system. """ - def automate(self): + def automate(self, full: bool = False) -> Tuple[bool, str]: raise NotImplementedError() def working_directory(self, filename=None) -> str: diff --git a/app/terraform/alarms/__init__.py b/app/terraform/alarms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/terraform/alarms/proxy_azure_cdn.py b/app/terraform/alarms/proxy_azure_cdn.py new file mode 100644 index 0000000..73029a9 --- /dev/null +++ b/app/terraform/alarms/proxy_azure_cdn.py @@ -0,0 +1,36 @@ +from azure.identity import ClientSecretCredential +from azure.mgmt.alertsmanagement import AlertsManagementClient + +from app import app +from app.alarms import get_proxy_alarm +from app.models.alarms import AlarmState +from app.models.mirrors import Proxy +from app.terraform import BaseAutomation + + +class AlarmProxyAzureCdnAutomation(BaseAutomation): + short_name = "monitor_proxy_azure_cdn" + description = "Import alarms for Azure CDN proxies" + + def automate(self): + credential = ClientSecretCredential( + tenant_id=app.config['AZURE_TENANT_ID'], + client_id=app.config['AZURE_CLIENT_ID'], + client_secret=app.config['AZURE_CLIENT_SECRET']) + client = AlertsManagementClient( + credential, + app.config['AZURE_SUBSCRIPTION_ID'] + ) + firing = [x.name[len("bandwidth-out-high-bc-"):] + for x in client.alerts.get_all() + if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"] + for proxy in Proxy.query.filter( + Proxy.provider == "azure_cdn", + Proxy.destroyed == None + ): + alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high") + if proxy.origin.group.group_name.lower() not in firing: + alarm.update_state(AlarmState.OK, "Azure monitor alert not firing") + else: + alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing") + return True, [] diff --git a/app/terraform/alarms/proxy_cloudfront.py b/app/terraform/alarms/proxy_cloudfront.py new file mode 100644 index 0000000..15b52e2 --- /dev/null +++ b/app/terraform/alarms/proxy_cloudfront.py @@ -0,0 +1,60 @@ +import datetime + +import boto3 + +from app import app +from app.alarms import get_proxy_alarm +from app.extensions import db +from app.models.mirrors import Proxy +from app.models.alarms import AlarmState, Alarm +from app.terraform import BaseAutomation + + +class AlarmProxyCloudfrontAutomation(BaseAutomation): + short_name = "monitor_proxy_cloudfront" + description = "Import alarms for AWS CloudFront proxies" + + def automate(self): + cloudwatch = boto3.client('cloudwatch', + aws_access_key_id=app.config['AWS_ACCESS_KEY'], + aws_secret_access_key=app.config['AWS_SECRET_KEY'], + region_name='us-east-2') + dist_paginator = cloudwatch.get_paginator('describe_alarms') + page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-") + for page in page_iterator: + for cw_alarm in page['MetricAlarms']: + dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):] + proxy = Proxy.query.filter(Proxy.slug == dist_id).first() + if proxy is None: + print("Skipping unknown proxy " + dist_id) + continue + alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high") + if cw_alarm['StateValue'] == "OK": + alarm.update_state(AlarmState.OK, "CloudWatch alarm OK") + elif cw_alarm['StateValue'] == "ALARM": + alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM") + else: + alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}") + alarm = Alarm.query.filter( + Alarm.alarm_type == "cloudfront-quota" + ).first() + if alarm is None: + alarm = Alarm() + alarm.target = "service/cloudfront" + alarm.alarm_type = "cloudfront-quota" + alarm.state_changed = datetime.datetime.utcnow() + db.session.add(alarm) + alarm.last_updated = datetime.datetime.utcnow() + deployed_count = len(Proxy.query.filter( + Proxy.destroyed == None).all()) + old_state = alarm.alarm_state + if deployed_count > 370: + alarm.alarm_state = AlarmState.CRITICAL + elif deployed_count > 320: + alarm.alarm_state = AlarmState.WARNING + else: + alarm.alarm_state = AlarmState.OK + if alarm.alarm_state != old_state: + alarm.state_changed = datetime.datetime.utcnow() + db.session.commit() + return True, [] diff --git a/app/terraform/alarms/proxy_http_status.py b/app/terraform/alarms/proxy_http_status.py new file mode 100644 index 0000000..5e31dbb --- /dev/null +++ b/app/terraform/alarms/proxy_http_status.py @@ -0,0 +1,64 @@ +from typing import Tuple + +import requests + +from app.extensions import db +from app.models.alarms import Alarm, AlarmState +from app.models.mirrors import Proxy +from app.terraform import BaseAutomation + + +def set_http_alarm(proxy_id: int, state: AlarmState, text: str): + alarm = Alarm.query.filter( + Alarm.proxy_id == proxy_id, + Alarm.alarm_type == "http-status" + ).first() + if alarm is None: + alarm = Alarm() + alarm.proxy_id = proxy_id + alarm.alarm_type = "http-status" + alarm.target = "proxy" + db.session.add(alarm) + alarm.update_state(state, text) + + +class AlarmProxyHTTPStatusAutomation(BaseAutomation): + short_name = "alarm_http_status" + description = "Check all deployed proxies for HTTP status code" + + def automate(self, full: bool = False) -> Tuple[bool, str]: + proxies = Proxy.query.filter( + Proxy.destroyed == None + ) + for proxy in proxies: + try: + if proxy.url is None: + continue + r = requests.get(proxy.url, + allow_redirects=False, + timeout=5) + r.raise_for_status() + if r.is_redirect: + set_http_alarm( + proxy.id, + AlarmState.CRITICAL, + f"{r.status_code} {r.reason}" + ) + else: + set_http_alarm( + proxy.id, + AlarmState.OK, + f"{r.status_code} {r.reason}" + ) + except (requests.ConnectionError, requests.Timeout): + set_http_alarm( + proxy.id, + AlarmState.CRITICAL, + f"Connection failure") + except requests.HTTPError: + set_http_alarm( + proxy.id, + AlarmState.CRITICAL, + f"{r.status_code} {r.reason}" + ) + return True, [] \ No newline at end of file diff --git a/app/terraform/bridge/__init__.py b/app/terraform/bridge/__init__.py index d22f9f2..2ccf214 100644 --- a/app/terraform/bridge/__init__.py +++ b/app/terraform/bridge/__init__.py @@ -1,14 +1,14 @@ import datetime -from typing import Iterable +from typing import Iterable, Optional, Any from app import app from app.extensions import db from app.models.bridges import BridgeConf, Bridge from app.models.base import Group -from app.terraform import BaseAutomation +from app.terraform.terraform import TerraformAutomation -class BridgeAutomation(BaseAutomation): +class BridgeAutomation(TerraformAutomation): def create_missing(self): bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter( BridgeConf.provider == self.provider, @@ -45,8 +45,12 @@ class BridgeAutomation(BaseAutomation): bridge.destroy() db.session.commit() - def generate_terraform(self): - self.write_terraform_config( + def tf_prehook(self) -> Optional[Any]: + self.create_missing() + self.destroy_expired() + + def tf_generate(self): + self.tf_write( self.template, groups=Group.query.all(), bridgeconfs=BridgeConf.query.filter( @@ -60,8 +64,8 @@ class BridgeAutomation(BaseAutomation): } ) - def import_terraform(self): - outputs = self.terraform_output() + def tf_posthook(self, *, prehook_result: Any = None) -> None: + outputs = self.tf_output() for output in outputs: if output.startswith('bridge_hashed_fingerprint_'): parts = outputs[output]['value'].split(" ") diff --git a/app/terraform/bridge/aws.py b/app/terraform/bridge/aws.py index 66f7834..e9509d5 100644 --- a/app/terraform/bridge/aws.py +++ b/app/terraform/bridge/aws.py @@ -1,9 +1,9 @@ -from app import app from app.terraform.bridge import BridgeAutomation class BridgeAWSAutomation(BridgeAutomation): short_name = "bridge_aws" + description = "Deploy Tor bridges on AWS Lightsail" provider = "aws" template_parameters = [ @@ -67,18 +67,3 @@ class BridgeAWSAutomation(BridgeAutomation): {% endfor %} {% endfor %} """ - - -def automate(): - auto = BridgeAWSAutomation() - auto.destroy_expired() - auto.create_missing() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply() - auto.import_terraform() - - -if __name__ == "__main__": - with app.app_context(): - automate() diff --git a/app/terraform/bridge/gandi.py b/app/terraform/bridge/gandi.py index 11a28cb..b26cd97 100644 --- a/app/terraform/bridge/gandi.py +++ b/app/terraform/bridge/gandi.py @@ -1,9 +1,9 @@ -from app import app from app.terraform.bridge import BridgeAutomation class BridgeGandiAutomation(BridgeAutomation): short_name = "bridge_gandi" + description = "Deploy Tor bridges on GandiCloud VPS" provider = "gandi" template_parameters = [ @@ -78,18 +78,3 @@ class BridgeGandiAutomation(BridgeAutomation): {% endfor %} {% endfor %} """ - - -def automate(): - auto = BridgeGandiAutomation() - auto.destroy_expired() - auto.create_missing() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply() - auto.import_terraform() - - -if __name__ == "__main__": - with app.app_context(): - automate() diff --git a/app/terraform/bridge/hcloud.py b/app/terraform/bridge/hcloud.py index e3c06e4..9d64e01 100644 --- a/app/terraform/bridge/hcloud.py +++ b/app/terraform/bridge/hcloud.py @@ -1,9 +1,9 @@ -from app import app from app.terraform.bridge import BridgeAutomation class BridgeHcloudAutomation(BridgeAutomation): short_name = "bridge_hcloud" + description = "Deploy Tor bridges on Hetzner Cloud" provider = "hcloud" template_parameters = [ @@ -81,18 +81,3 @@ class BridgeHcloudAutomation(BridgeAutomation): {% endfor %} {% endfor %} """ - - -def automate(): - auto = BridgeHcloudAutomation() - auto.destroy_expired() - auto.create_missing() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply() - auto.import_terraform() - - -if __name__ == "__main__": - with app.app_context(): - automate() diff --git a/app/terraform/bridge/ovh.py b/app/terraform/bridge/ovh.py index cea1727..d1a6a2e 100644 --- a/app/terraform/bridge/ovh.py +++ b/app/terraform/bridge/ovh.py @@ -1,9 +1,9 @@ -from app import app from app.terraform.bridge import BridgeAutomation class BridgeOvhAutomation(BridgeAutomation): short_name = "bridge_ovh" + description = "Deploy Tor bridges on OVH Public Cloud" provider = "ovh" template_parameters = [ @@ -104,18 +104,3 @@ class BridgeOvhAutomation(BridgeAutomation): {% endfor %} {% endfor %} """ - - -def automate(): - auto = BridgeOvhAutomation() - auto.destroy_expired() - auto.create_missing() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply() - auto.import_terraform() - - -if __name__ == "__main__": - with app.app_context(): - automate() diff --git a/app/terraform/list/__init__.py b/app/terraform/list/__init__.py index 95e2348..e83ab63 100644 --- a/app/terraform/list/__init__.py +++ b/app/terraform/list/__init__.py @@ -5,12 +5,12 @@ from app.lists.mirror_mapping import mirror_mapping from app.lists.bc2 import mirror_sites from app.lists.bridgelines import bridgelines from app.models.base import MirrorList -from app.terraform import BaseAutomation +from app.terraform.terraform import TerraformAutomation -class ListAutomation(BaseAutomation): - def generate_terraform(self): - self.write_terraform_config( +class ListAutomation(TerraformAutomation): + def tf_generate(self): + self.tf_write( self.template, lists=MirrorList.query.filter( MirrorList.destroyed == None, diff --git a/app/terraform/list/github.py b/app/terraform/list/github.py index 852c70c..60573dc 100644 --- a/app/terraform/list/github.py +++ b/app/terraform/list/github.py @@ -1,9 +1,9 @@ -from app import app from app.terraform.list import ListAutomation class ListGithubAutomation(ListAutomation): short_name = "list_github" + description = "Update mirror lists in GitHub repositories" provider = "github" template_parameters = [ @@ -45,11 +45,3 @@ class ListGithubAutomation(ListAutomation): } {% endfor %} """ - - -if __name__ == "__main__": - with app.app_context(): - auto = ListGithubAutomation() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply() diff --git a/app/terraform/list/gitlab.py b/app/terraform/list/gitlab.py index db19ebc..924cc22 100644 --- a/app/terraform/list/gitlab.py +++ b/app/terraform/list/gitlab.py @@ -1,9 +1,9 @@ -from app import app from app.terraform.list import ListAutomation class ListGitlabAutomation(ListAutomation): short_name = "list_gitlab" + description = "Update mirror lists in GitLab repositories" provider = "gitlab" template_parameters = [ @@ -44,11 +44,3 @@ class ListGitlabAutomation(ListAutomation): {% endfor %} """ - - -if __name__ == "__main__": - with app.app_context(): - auto = ListGitlabAutomation() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply() diff --git a/app/terraform/list/s3.py b/app/terraform/list/s3.py index c3131e2..2db1641 100644 --- a/app/terraform/list/s3.py +++ b/app/terraform/list/s3.py @@ -1,9 +1,9 @@ -from app import app from app.terraform.list import ListAutomation -class ListGithubAutomation(ListAutomation): +class ListS3Automation(ListAutomation): short_name = "list_s3" + description = "Update mirror lists in AWS S3 buckets" provider = "s3" template_parameters = [ @@ -36,11 +36,3 @@ class ListGithubAutomation(ListAutomation): } {% endfor %} """ - - -if __name__ == "__main__": - with app.app_context(): - auto = ListGithubAutomation() - auto.generate_terraform() - auto.terraform_init() - auto.terraform_apply() diff --git a/app/terraform/proxy/__init__.py b/app/terraform/proxy/__init__.py index f9674dd..23bd576 100644 --- a/app/terraform/proxy/__init__.py +++ b/app/terraform/proxy/__init__.py @@ -37,6 +37,8 @@ class ProxyAutomation(TerraformAutomation): for group in groups: subgroup = 0 for origin in group.origins: + if origin.destroyed is not None: + continue while True: if subgroups[group.id][subgroup] >= self.subgroup_max: subgroup += 1 @@ -87,7 +89,7 @@ class ProxyAutomation(TerraformAutomation): self.deprecate_orphaned_proxies() self.destroy_expired_proxies() - def tf_posthook(self): + def tf_posthook(self, *, prehook_result): self.import_state(self.tf_show()) def tf_generate(self): diff --git a/app/terraform/proxy/azure_cdn.py b/app/terraform/proxy/azure_cdn.py index 5acd8e3..ce9c176 100644 --- a/app/terraform/proxy/azure_cdn.py +++ b/app/terraform/proxy/azure_cdn.py @@ -1,16 +1,11 @@ -from azure.identity import ClientSecretCredential -from azure.mgmt.alertsmanagement import AlertsManagementClient - -from app import app -from app.alarms import get_proxy_alarm from app.extensions import db from app.models.mirrors import Proxy -from app.models.alarms import AlarmState from app.terraform.proxy import ProxyAutomation class ProxyAzureCdnAutomation(ProxyAutomation): short_name = "proxy_azure_cdn" + description = "Deploy proxies to Azure CDN" provider = "azure_cdn" subgroup_max = 25 parallelism = 1 @@ -170,33 +165,3 @@ class ProxyAzureCdnAutomation(ProxyAutomation): for proxy in proxies: proxy.url = f"https://{proxy.slug}.azureedge.net" db.session.commit() - - -def import_monitor_alerts(): - credential = ClientSecretCredential( - tenant_id=app.config['AZURE_TENANT_ID'], - client_id=app.config['AZURE_CLIENT_ID'], - client_secret=app.config['AZURE_CLIENT_SECRET']) - client = AlertsManagementClient( - credential, - app.config['AZURE_SUBSCRIPTION_ID'] - ) - firing = [x.name[len("bandwidth-out-high-bc-"):] - for x in client.alerts.get_all() - if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"] - for proxy in Proxy.query.filter( - Proxy.provider == "azure_cdn", - Proxy.destroyed == None - ): - alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high") - if proxy.origin.group.group_name.lower() not in firing: - alarm.update_state(AlarmState.OK, "Azure monitor alert not firing") - else: - alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing") - - -if __name__ == "__main__": - with app.app_context(): - auto = ProxyAzureCdnAutomation() - auto.automate() - import_monitor_alerts() diff --git a/app/terraform/proxy/cloudfront.py b/app/terraform/proxy/cloudfront.py index 1c1fa42..a10d315 100644 --- a/app/terraform/proxy/cloudfront.py +++ b/app/terraform/proxy/cloudfront.py @@ -1,17 +1,13 @@ import datetime -import boto3 - -from app import app -from app.alarms import get_proxy_alarm from app.extensions import db -from app.models.alarms import Alarm, AlarmState from app.models.mirrors import Proxy from app.terraform.proxy import ProxyAutomation class ProxyCloudfrontAutomation(ProxyAutomation): short_name = "proxy_cloudfront" + description = "Deploy proxies to AWS CloudFront" provider = "cloudfront" template_parameters = [ @@ -87,55 +83,3 @@ class ProxyCloudfrontAutomation(ProxyAutomation): proxy.terraform_updated = datetime.datetime.utcnow() break db.session.commit() - - -def import_cloudwatch_alarms(): - cloudwatch = boto3.client('cloudwatch', - aws_access_key_id=app.config['AWS_ACCESS_KEY'], - aws_secret_access_key=app.config['AWS_SECRET_KEY'], - region_name='us-east-2') - dist_paginator = cloudwatch.get_paginator('describe_alarms') - page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-") - for page in page_iterator: - for cw_alarm in page['MetricAlarms']: - dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):] - proxy = Proxy.query.filter(Proxy.slug == dist_id).first() - if proxy is None: - print("Skipping unknown proxy " + dist_id) - continue - alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high") - if cw_alarm['StateValue'] == "OK": - alarm.update_state(AlarmState.OK, "CloudWatch alarm OK") - elif cw_alarm['StateValue'] == "ALARM": - alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM") - else: - alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}") - alarm = Alarm.query.filter( - Alarm.alarm_type == "cloudfront-quota" - ).first() - if alarm is None: - alarm = Alarm() - alarm.target = "service/cloudfront" - alarm.alarm_type = "cloudfront-quota" - alarm.state_changed = datetime.datetime.utcnow() - db.session.add(alarm) - alarm.last_updated = datetime.datetime.utcnow() - deployed_count = len(Proxy.query.filter( - Proxy.destroyed == None).all()) - old_state = alarm.alarm_state - if deployed_count > 370: - alarm.alarm_state = AlarmState.CRITICAL - elif deployed_count > 320: - alarm.alarm_state = AlarmState.WARNING - else: - alarm.alarm_state = AlarmState.OK - if alarm.alarm_state != old_state: - alarm.state_changed = datetime.datetime.utcnow() - db.session.commit() - - -if __name__ == "__main__": - with app.app_context(): - auto = ProxyCloudfrontAutomation() - auto.automate() - import_cloudwatch_alarms() diff --git a/app/terraform/proxy_check.py b/app/terraform/proxy_check.py deleted file mode 100644 index e72b1ae..0000000 --- a/app/terraform/proxy_check.py +++ /dev/null @@ -1,62 +0,0 @@ -import requests - -from app import app -from app.extensions import db -from app.models.alarms import Alarm, AlarmState -from app.models.mirrors import Proxy - - -def set_http_alarm(proxy_id: int, state: AlarmState, text: str): - alarm = Alarm.query.filter( - Alarm.proxy_id == proxy_id, - Alarm.alarm_type == "http-status" - ).first() - if alarm is None: - alarm = Alarm() - alarm.proxy_id = proxy_id - alarm.alarm_type = "http-status" - alarm.target = "proxy" - db.session.add(alarm) - alarm.update_state(state, text) - - -def check_http(): - proxies = Proxy.query.filter( - Proxy.destroyed == None - ) - for proxy in proxies: - try: - if proxy.url is None: - continue - r = requests.get(proxy.url, - allow_redirects=False, - timeout=5) - r.raise_for_status() - if r.is_redirect: - set_http_alarm( - proxy.id, - AlarmState.CRITICAL, - f"{r.status_code} {r.reason}" - ) - else: - set_http_alarm( - proxy.id, - AlarmState.OK, - f"{r.status_code} {r.reason}" - ) - except (requests.ConnectionError, requests.Timeout): - set_http_alarm( - proxy.id, - AlarmState.CRITICAL, - f"Connection failure") - except requests.HTTPError: - set_http_alarm( - proxy.id, - AlarmState.CRITICAL, - f"{r.status_code} {r.reason}" - ) - - -if __name__ == "__main__": - with app.app_context(): - check_http() diff --git a/app/terraform/terraform.py b/app/terraform/terraform.py index 8311c85..4f063c2 100644 --- a/app/terraform/terraform.py +++ b/app/terraform/terraform.py @@ -1,6 +1,6 @@ import json import subprocess -from typing import Dict, Any, Optional +from typing import Any, Dict, List, Optional, Tuple import jinja2 @@ -18,20 +18,27 @@ class TerraformAutomation(BaseAutomation): Default parallelism for remote API calls. """ - def automate(self): - self.tf_prehook() + def automate(self, full: bool = False): + prehook_result = self.tf_prehook() self.tf_generate() self.tf_init() - self.tf_apply(refresh=False) - self.tf_posthook() + returncode, logs = self.tf_apply(refresh=full) + self.tf_posthook(prehook_result=prehook_result) + return True if returncode == 0 else False, logs - def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None): + def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None) -> Tuple[int, List[Dict[str, Any]]]: if not parallelism: parallelism = self.parallelism - subprocess.run( + tf = subprocess.run( ['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve', - f'-parallelism={str(parallelism)}'], - cwd=self.working_directory()) + f'-parallelism={str(parallelism)}', '-json'], + cwd=self.working_directory(), + stdout=subprocess.PIPE) + logs = [] + for line in tf.stdout.decode('utf-8').split('\n'): + if line.strip(): + logs.append(json.loads(line)) + return tf.returncode, logs def tf_generate(self): raise NotImplementedError() @@ -56,7 +63,7 @@ class TerraformAutomation(BaseAutomation): # more like JSON-ND, task is to figure out how to yield those records # as plan runs, the same is probably also true for apply - def tf_posthook(self, prehook_result: Any = None) -> None: + def tf_posthook(self, *, prehook_result: Any = None) -> None: """ This hook function is called as part of normal automation, after the completion of :func:`tf_apply`. diff --git a/docs/admin/external.rst b/docs/admin/external.rst index 66293ef..0447da6 100644 --- a/docs/admin/external.rst +++ b/docs/admin/external.rst @@ -70,7 +70,9 @@ The expiry can be set according to your threat model. GitLab will send an email warning to token owners 7 days before expiry allowing you to generate a new token and update your configuration. -Your access token will need the "read_repository" and "write_repository" scopes. +Your access token will need the "api" scope. Unforunately the "write_repository" scope +only works for Git-over-HTTPS, but the portal uses the API to update mirror lists in +GitLab. Once you've generated your token, you can add it to your ``config.yaml``: diff --git a/migrations/versions/0a0a65db7f01_add_automations.py b/migrations/versions/0a0a65db7f01_add_automations.py new file mode 100644 index 0000000..fe71ce2 --- /dev/null +++ b/migrations/versions/0a0a65db7f01_add_automations.py @@ -0,0 +1,54 @@ +"""add automations + +Revision ID: 0a0a65db7f01 +Revises: c3d6e95caa79 +Create Date: 2022-05-08 16:24:53.779353 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0a0a65db7f01' +down_revision = 'c3d6e95caa79' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('automation', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.String(length=255), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('destroyed', sa.DateTime(), nullable=True), + sa.Column('short_name', sa.String(length=25), nullable=False), + sa.Column('state', sa.Enum('IDLE', 'RUNNING', 'ERROR', name='automationstate'), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.Column('last_run', sa.DateTime(), nullable=True), + sa.Column('next_run', sa.DateTime(), nullable=True), + sa.Column('next_is_full', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_automation')) + ) + op.create_table('automation_logs', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('deprecated', sa.DateTime(), nullable=True), + sa.Column('deprecation_reason', sa.String(), nullable=True), + sa.Column('destroyed', sa.DateTime(), nullable=True), + sa.Column('automation_id', sa.Integer(), nullable=False), + sa.Column('logs', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['automation_id'], ['automation.id'], name=op.f('fk_automation_logs_automation_id_automation')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_automation_logs')) + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('automation_logs') + op.drop_table('automation') + # ### end Alembic commands ###
    View/Edit + Destroy