automation: establish an automation framework

app/cli/__main__.py

@@ -3,6 +3,7 @@ import logging
 import sys
 from os.path import basename
 
+from app.cli.automate import AutomateCliHandler
 from app.cli.db import DbCliHandler
 from app.cli.list import ListCliHandler
 
@@ -13,6 +14,7 @@ def parse_args(argv):
     parser = argparse.ArgumentParser()
     parser.add_argument("-v", "--verbose", help="increase logging verbosity", action="store_true")
     subparsers = parser.add_subparsers(title="command", help="command to run")
+    AutomateCliHandler.add_subparser_to(subparsers)
     DbCliHandler.add_subparser_to(subparsers)
     ListCliHandler.add_subparser_to(subparsers)
     args = parser.parse_args(argv[1:])

app/cli/automate.py

@@ -0,0 +1,117 @@
+import argparse
+import datetime
+import json
+import logging
+
+from app import app
+from app.extensions import db
+from app.models.automation import Automation, AutomationState, AutomationLogs
+from app.terraform import BaseAutomation
+from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation
+from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation
+from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation
+from app.terraform.bridge.aws import BridgeAWSAutomation
+from app.terraform.bridge.gandi import BridgeGandiAutomation
+from app.terraform.bridge.hcloud import BridgeHcloudAutomation
+from app.terraform.bridge.ovh import BridgeOvhAutomation
+from app.terraform.list.github import ListGithubAutomation
+from app.terraform.list.gitlab import ListGitlabAutomation
+from app.terraform.list.s3 import ListS3Automation
+from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
+from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
+
+
+jobs = {
+    x.short_name: x
+    for x in [
+        AlarmProxyAzureCdnAutomation,
+        AlarmProxyCloudfrontAutomation,
+        AlarmProxyHTTPStatusAutomation,
+        BridgeAWSAutomation,
+        BridgeGandiAutomation,
+        BridgeHcloudAutomation,
+        BridgeOvhAutomation,
+        ListGithubAutomation,
+        ListGitlabAutomation,
+        ListS3Automation,
+        ProxyAzureCdnAutomation,
+        ProxyCloudfrontAutomation
+    ]
+}
+
+
+def run_all(**kwargs):
+    for job in jobs.values():
+        run_job(job, **kwargs)
+
+
+def run_job(job: BaseAutomation, *, force: bool = False, ignore_schedule: bool = False):
+    automation = Automation.query.filter(Automation.short_name == job.short_name).first()
+    if automation is None:
+        automation = Automation()
+        automation.short_name = job.short_name
+        automation.description = job.description
+        automation.enabled = True
+        automation.next_is_full = False
+        automation.added = datetime.datetime.utcnow()
+        automation.updated = automation.added
+        db.session.add(automation)
+    else:
+        if automation.state == AutomationState.RUNNING and not force:
+            logging.warning("Not running an already running automation")
+            return
+        if not ignore_schedule and not force:
+            if automation.next_run is not None and automation.next_run > datetime.datetime.utcnow():
+                logging.warning("Not time to run this job yet")
+                return
+    if not automation.enabled and not force:
+        db.session.rollback()
+        logging.warning(f"job {job.short_name} is disabled and --force not specified")
+        return
+    automation.state = AutomationState.RUNNING
+    db.session.commit()
+    job = job()
+    try:
+        success, logs = job.automate()
+    except Exception as e:
+        success = False
+        logs = repr(e)
+    if success:
+        automation.state = AutomationState.IDLE
+        automation.next_run = datetime.datetime.utcnow() + datetime.timedelta(minutes=7)
+    else:
+        automation.state = AutomationState.ERROR
+        automation.enabled = False
+        automation.next_run = None
+        log = AutomationLogs()
+        log.automation_id = automation.id
+        log.added = datetime.datetime.utcnow()
+        log.updated = datetime.datetime.utcnow()
+        log.logs = json.dumps(logs)
+        db.session.add(log)
+    automation.last_run = datetime.datetime.utcnow()
+    db.session.commit()
+
+
+class AutomateCliHandler:
+    @classmethod
+    def add_subparser_to(cls, subparsers: argparse._SubParsersAction) -> None:
+        parser = subparsers.add_parser("automate", help="automation operations")
+        parser.add_argument("-a", "--all", dest="all", help="run all automation jobs", action="store_true")
+        parser.add_argument("-j", "--job", dest="job", choices=sorted(jobs.keys()),
+                            help="run a specific automation job")
+        parser.add_argument("--force", help="run job even if disabled and it's not time yet", action="store_true")
+        parser.add_argument("--ignore-schedule", help="run job even if it's not time yet", action="store_true")
+        parser.set_defaults(cls=cls)
+
+    def __init__(self, args):
+        self.args = args
+
+    def run(self):
+        with app.app_context():
+            if self.args.job:
+                run_job(jobs[self.args.job], force=self.args.force, ignore_schedule=self.args.ignore_schedule)
+            elif self.args.all:
+                run_all(force=self.args.force, ignore_schedule=self.args.ignore_schedule)
+            else:
+                logging.error("No action requested")

app/models/automation.py

@@ -0,0 +1,34 @@
+import datetime
+import enum
+
+from app.extensions import db
+from app.models import AbstractConfiguration, AbstractResource
+
+
+class AutomationState(enum.Enum):
+    IDLE = 0
+    RUNNING = 1
+    ERROR = 3
+
+
+class Automation(AbstractConfiguration):
+    short_name = db.Column(db.String(25), nullable=False)
+    state = db.Column(db.Enum(AutomationState), default=AutomationState.IDLE, nullable=False)
+    enabled = db.Column(db.Boolean, nullable=False)
+    last_run = db.Column(db.DateTime(), nullable=True)
+    next_run = db.Column(db.DateTime(), nullable=True)
+    next_is_full = db.Column(db.Boolean(), nullable=False)
+
+    logs = db.relationship("AutomationLogs", back_populates="automation")
+
+    def kick(self):
+        self.enabled = True
+        self.next_run = datetime.datetime.utcnow()
+        self.updated = datetime.datetime.utcnow()
+
+
+class AutomationLogs(AbstractResource):
+    automation_id = db.Column(db.Integer, db.ForeignKey(Automation.id), nullable=False)
+    logs = db.Column(db.Text)
+
+    automation = db.relationship("Automation", back_populates="logs")

app/portal/__init__.py

@@ -8,6 +8,7 @@ from app.models.alarms import Alarm
 from app import Origin, Proxy
 from app.models.base import Group, MirrorList
 from app.portal.forms import LifecycleForm, NewMirrorListForm
+from app.portal.automation import bp as automation
 from app.portal.bridgeconf import bp as bridgeconf
 from app.portal.bridge import bp as bridge
 from app.portal.group import bp as group
@@ -17,7 +18,7 @@ from app.portal.proxy import bp as proxy
 from app.portal.util import response_404, view_lifecycle
 
 portal = Blueprint("portal", __name__, template_folder="templates", static_folder="static")
-
+portal.register_blueprint(automation, url_prefix="/automation")
 portal.register_blueprint(bridgeconf, url_prefix="/bridgeconf")
 portal.register_blueprint(bridge, url_prefix="/bridge")
 portal.register_blueprint(group, url_prefix="/group")

app/portal/automation.py

@@ -0,0 +1,70 @@
+from datetime import datetime
+
+from flask import render_template, flash, Response, Blueprint
+from flask_wtf import FlaskForm
+from sqlalchemy import exc
+from wtforms import SubmitField, BooleanField
+from wtforms.validators import DataRequired
+
+from app.extensions import db
+from app.models.automation import Automation
+from app.portal.util import view_lifecycle, response_404
+
+bp = Blueprint("automation", __name__)
+
+
+class EditAutomationForm(FlaskForm):
+    enabled = BooleanField('Enabled')
+    submit = SubmitField('Save Changes')
+
+
+@bp.route("/list")
+def automation_list():
+    automations = Automation.query.filter(
+        Automation.destroyed == None).order_by(Automation.description).all()
+    return render_template("list.html.j2",
+                           section="automation",
+                           title="Automation Jobs",
+                           item="automation",
+                           items=automations)
+
+
+@bp.route('/edit/<automation_id>', methods=['GET', 'POST'])
+def automation_edit(automation_id):
+    automation = Automation.query.filter(Automation.id == automation_id).first()
+    if automation is None:
+        return Response(render_template("error.html.j2",
+                                        section="automation",
+                                        header="404 Automation Job Not Found",
+                                        message="The requested automation job could not be found."),
+                        status=404)
+    form = EditAutomationForm(enabled=automation.enabled)
+    if form.validate_on_submit():
+        automation.enabled = form.enabled.data
+        automation.updated = datetime.utcnow()
+        try:
+            db.session.commit()
+            flash("Saved changes to bridge configuration.", "success")
+        except exc.SQLAlchemyError:
+            flash("An error occurred saving the changes to the bridge configuration.", "danger")
+    return render_template("automation.html.j2",
+                           section="automation",
+                           automation=automation, form=form)
+
+
+@bp.route("/kick/<automation_id>", methods=['GET', 'POST'])
+def automation_kick(automation_id: int):
+    automation = Automation.query.filter(
+        Automation.id == automation_id,
+        Automation.destroyed == None).first()
+    if automation is None:
+        return response_404("The requested bridge configuration could not be found.")
+    return view_lifecycle(
+        header=f"Kick automation timer?",
+        message=automation.description,
+        success_view="portal.automation.automation_list",
+        success_message="This automation job will next run within 1 minute.",
+        section="automation",
+        resource=automation,
+        action="kick"
+    )

app/portal/templates/automation.html.j2

@@ -0,0 +1,16 @@
+{% extends "base.html.j2" %}
+{% from 'bootstrap5/form.html' import render_form %}
+{% from "tables.html.j2" import automation_logs_table %}
+
+{% block content %}
+    <h1 class="h2 mt-3">Automation Job</h1>
+    <h2 class="h3">{{ automation.description }} ({{ automation.short_name }})</h2>
+
+    <div style="border: 1px solid #666;" class="p-3">
+        {{ render_form(form) }}
+    </div>
+
+    <h3>Logs</h3>
+    {{ automation_logs_table(automation.logs) }}
+
+{% endblock %}

app/portal/templates/base.html.j2

@@ -128,6 +128,12 @@
                     <span>Monitoring</span>
                 </h6>
                 <ul class="nav flex-column">
+                    <li class="nav-item">
+                        <a class="nav-link{% if section == "automation" %} active{% endif %}"
+                           href="{{ url_for("portal.automation.automation_list") }}">
+                            Automation
+                        </a>
+                    </li>
                     <li class="nav-item">
                         <a class="nav-link{% if section == "alarm" %} active{% endif %}"
                            href="{{ url_for("portal.view_alarms") }}">

app/portal/templates/list.html.j2

@@ -1,5 +1,5 @@
 {% extends "base.html.j2" %}
-{% from "tables.html.j2" import bridgeconfs_table, bridges_table,
+{% from "tables.html.j2" import automations_table, bridgeconfs_table, bridges_table,
         groups_table, mirrorlists_table, origins_table, origin_onion_table,
         onions_table, proxies_table %}
 
@@ -8,7 +8,9 @@
     {% if new_link %}
     <a href="{{ new_link }}" class="btn btn-success">Create new {{ item }}</a>
     {% endif %}
-    {% if item == "bridge configuration" %}
+    {% if item == "automation" %}
+        {{ automations_table(items) }}
+    {% elif item == "bridge configuration" %}
         {{ bridgeconfs_table(items) }}
     {% elif item == "bridge" %}
         {{ bridges_table(items) }}

app/portal/templates/tables.html.j2

@@ -26,6 +26,67 @@
     <div class="alert alert-danger">Not implemented yet.</div>
 {% endmacro %}
 
+{% macro automations_table(automations) %}
+<div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Description</th>
+                <th scope="col">Status</th>
+                <th scope="col">Enabled</th>
+                <th scope="col">Last Run</th>
+                <th scope="col">Next Run</th>
+                <th scope="col">Actions</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for automation in automations %}
+                <tr>
+                    <td title="{{ automation.short_name }}">{{ automation.description }}</td>
+                    <td title="{{ automation.state.name }}">
+                        {% if automation.state.name == "IDLE" %}
+                            🕰️
+                        {% elif automation.state.name == "RUNNING" %}
+                            🏃
+                        {% else %}
+                            💥
+                        {% endif %}
+                    </td>
+                    <td>{% if automation.enabled %}✅{% else %}❌{% endif %}</td>
+                    <td>{{ automation.last_run | format_datetime }}</td>
+                    <td>{{ automation.next_run | format_datetime }}</td>
+                    <td>
+                        <a href="{{ url_for("portal.automation.automation_edit", automation_id=automation.id) }}" class="btn btn-primary btn-sm">View/Edit</a>
+                        <a href="{{ url_for("portal.automation.automation_kick", automation_id=automation.id) }}" class="btn btn-success btn-sm">Kick Timer</a>
+                    </td>
+                </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endmacro %}
+
+{% macro automation_logs_table(logs) %}
+<div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Log Created</th>
+                <th scope="col">Log</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for log in logs %}
+                <tr>
+                    <td>{{ log.added | format_datetime }}</td>
+                    <td>{{ log.logs }}</td>
+                </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endmacro %}
+
 {% macro groups_table(groups) %}
 <div class="table-responsive">
         <table class="table table-striped table-sm">
@@ -84,6 +145,8 @@
                         <td>
                             <a href="{{ url_for("portal.origin.origin_edit", origin_id=origin.id) }}"
                                class="btn btn-primary btn-sm">View/Edit</a>
+                            <a href="{{ url_for("portal.origin.origin_destroy", origin_id=origin.id) }}"
+                               class="btn btn-danger btn-sm">Destroy</a>
                         </td>
                     </tr>
                 {% endif %}

app/portal/util.py

@@ -20,11 +20,19 @@ def view_lifecycle(*,
                    resource: AbstractResource,
                    action: str):
     form = LifecycleForm()
+    if action == "destroy":
+        form.submit.render_kw = {"class": "btn btn-danger"}
+    elif action == "deprecate":
+        form.submit.render_kw = {"class": "btn btn-warning"}
+    elif action == "kick":
+        form.submit.render_kw = {"class": "btn btn-success"}
     if form.validate_on_submit():
         if action == "destroy":
             resource.destroy()
         elif action == "deprecate":
             resource.deprecate(reason="manual")
+        elif action == "kick":
+            resource.kick()
         else:
             flash("Unknown action")
             return redirect(url_for("portal.portal_home"))

app/terraform/__init__.py

@@ -1,4 +1,5 @@
 import os
+from typing import Tuple
 
 from app import app
 
@@ -10,7 +11,7 @@ class BaseAutomation:
     the portal system.
     """
 
-    def automate(self):
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
         raise NotImplementedError()
 
     def working_directory(self, filename=None) -> str:

app/terraform/alarms/proxy_azure_cdn.py

@@ -0,0 +1,36 @@
+from azure.identity import ClientSecretCredential
+from azure.mgmt.alertsmanagement import AlertsManagementClient
+
+from app import app
+from app.alarms import get_proxy_alarm
+from app.models.alarms import AlarmState
+from app.models.mirrors import Proxy
+from app.terraform import BaseAutomation
+
+
+class AlarmProxyAzureCdnAutomation(BaseAutomation):
+    short_name = "monitor_proxy_azure_cdn"
+    description = "Import alarms for Azure CDN proxies"
+
+    def automate(self):
+        credential = ClientSecretCredential(
+            tenant_id=app.config['AZURE_TENANT_ID'],
+            client_id=app.config['AZURE_CLIENT_ID'],
+            client_secret=app.config['AZURE_CLIENT_SECRET'])
+        client = AlertsManagementClient(
+            credential,
+            app.config['AZURE_SUBSCRIPTION_ID']
+        )
+        firing = [x.name[len("bandwidth-out-high-bc-"):]
+                  for x in client.alerts.get_all()
+                  if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
+        for proxy in Proxy.query.filter(
+            Proxy.provider == "azure_cdn",
+            Proxy.destroyed == None
+        ):
+            alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
+            if proxy.origin.group.group_name.lower() not in firing:
+                alarm.update_state(AlarmState.OK, "Azure monitor alert not firing")
+            else:
+                alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing")
+        return True, []

app/terraform/alarms/proxy_cloudfront.py

@@ -0,0 +1,60 @@
+import datetime
+
+import boto3
+
+from app import app
+from app.alarms import get_proxy_alarm
+from app.extensions import db
+from app.models.mirrors import Proxy
+from app.models.alarms import AlarmState, Alarm
+from app.terraform import BaseAutomation
+
+
+class AlarmProxyCloudfrontAutomation(BaseAutomation):
+    short_name = "monitor_proxy_cloudfront"
+    description = "Import alarms for AWS CloudFront proxies"
+
+    def automate(self):
+        cloudwatch = boto3.client('cloudwatch',
+                                  aws_access_key_id=app.config['AWS_ACCESS_KEY'],
+                                  aws_secret_access_key=app.config['AWS_SECRET_KEY'],
+                                  region_name='us-east-2')
+        dist_paginator = cloudwatch.get_paginator('describe_alarms')
+        page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-")
+        for page in page_iterator:
+            for cw_alarm in page['MetricAlarms']:
+                dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):]
+                proxy = Proxy.query.filter(Proxy.slug == dist_id).first()
+                if proxy is None:
+                    print("Skipping unknown proxy " + dist_id)
+                    continue
+                alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
+                if cw_alarm['StateValue'] == "OK":
+                    alarm.update_state(AlarmState.OK, "CloudWatch alarm OK")
+                elif cw_alarm['StateValue'] == "ALARM":
+                    alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM")
+                else:
+                    alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}")
+        alarm = Alarm.query.filter(
+            Alarm.alarm_type == "cloudfront-quota"
+        ).first()
+        if alarm is None:
+            alarm = Alarm()
+            alarm.target = "service/cloudfront"
+            alarm.alarm_type = "cloudfront-quota"
+            alarm.state_changed = datetime.datetime.utcnow()
+            db.session.add(alarm)
+        alarm.last_updated = datetime.datetime.utcnow()
+        deployed_count = len(Proxy.query.filter(
+            Proxy.destroyed == None).all())
+        old_state = alarm.alarm_state
+        if deployed_count > 370:
+            alarm.alarm_state = AlarmState.CRITICAL
+        elif deployed_count > 320:
+            alarm.alarm_state = AlarmState.WARNING
+        else:
+            alarm.alarm_state = AlarmState.OK
+        if alarm.alarm_state != old_state:
+            alarm.state_changed = datetime.datetime.utcnow()
+        db.session.commit()
+        return True, []

app/terraform/alarms/proxy_http_status.py

@@ -0,0 +1,64 @@
+from typing import Tuple
+
+import requests
+
+from app.extensions import db
+from app.models.alarms import Alarm, AlarmState
+from app.models.mirrors import Proxy
+from app.terraform import BaseAutomation
+
+
+def set_http_alarm(proxy_id: int, state: AlarmState, text: str):
+    alarm = Alarm.query.filter(
+        Alarm.proxy_id == proxy_id,
+        Alarm.alarm_type == "http-status"
+    ).first()
+    if alarm is None:
+        alarm = Alarm()
+        alarm.proxy_id = proxy_id
+        alarm.alarm_type = "http-status"
+        alarm.target = "proxy"
+        db.session.add(alarm)
+    alarm.update_state(state, text)
+
+
+class AlarmProxyHTTPStatusAutomation(BaseAutomation):
+    short_name = "alarm_http_status"
+    description = "Check all deployed proxies for HTTP status code"
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
+        proxies = Proxy.query.filter(
+            Proxy.destroyed == None
+        )
+        for proxy in proxies:
+            try:
+                if proxy.url is None:
+                    continue
+                r = requests.get(proxy.url,
+                                 allow_redirects=False,
+                                 timeout=5)
+                r.raise_for_status()
+                if r.is_redirect:
+                    set_http_alarm(
+                        proxy.id,
+                        AlarmState.CRITICAL,
+                        f"{r.status_code} {r.reason}"
+                    )
+                else:
+                    set_http_alarm(
+                        proxy.id,
+                        AlarmState.OK,
+                        f"{r.status_code} {r.reason}"
+                    )
+            except (requests.ConnectionError, requests.Timeout):
+                set_http_alarm(
+                    proxy.id,
+                    AlarmState.CRITICAL,
+                    f"Connection failure")
+            except requests.HTTPError:
+                set_http_alarm(
+                    proxy.id,
+                    AlarmState.CRITICAL,
+                    f"{r.status_code} {r.reason}"
+                )
+        return True, []

app/terraform/bridge/__init__.py

@@ -1,14 +1,14 @@
 import datetime
-from typing import Iterable
+from typing import Iterable, Optional, Any
 
 from app import app
 from app.extensions import db
 from app.models.bridges import BridgeConf, Bridge
 from app.models.base import Group
-from app.terraform import BaseAutomation
+from app.terraform.terraform import TerraformAutomation
 
 
-class BridgeAutomation(BaseAutomation):
+class BridgeAutomation(TerraformAutomation):
     def create_missing(self):
         bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter(
             BridgeConf.provider == self.provider,
@@ -45,8 +45,12 @@ class BridgeAutomation(BaseAutomation):
             bridge.destroy()
         db.session.commit()
 
-    def generate_terraform(self):
+    def tf_prehook(self) -> Optional[Any]:
-        self.write_terraform_config(
+        self.create_missing()
+        self.destroy_expired()
+
+    def tf_generate(self):
+        self.tf_write(
             self.template,
             groups=Group.query.all(),
             bridgeconfs=BridgeConf.query.filter(
@@ -60,8 +64,8 @@ class BridgeAutomation(BaseAutomation):
             }
         )
 
-    def import_terraform(self):
+    def tf_posthook(self, *, prehook_result: Any = None) -> None:
-        outputs = self.terraform_output()
+        outputs = self.tf_output()
         for output in outputs:
             if output.startswith('bridge_hashed_fingerprint_'):
                 parts = outputs[output]['value'].split(" ")

app/terraform/bridge/aws.py

@@ -1,9 +1,9 @@
-from app import app
 from app.terraform.bridge import BridgeAutomation
 
 
 class BridgeAWSAutomation(BridgeAutomation):
     short_name = "bridge_aws"
+    description = "Deploy Tor bridges on AWS Lightsail"
     provider = "aws"
 
     template_parameters = [
@@ -67,18 +67,3 @@ class BridgeAWSAutomation(BridgeAutomation):
     {% endfor %}
     {% endfor %}
     """
-
-
-def automate():
-    auto = BridgeAWSAutomation()
-    auto.destroy_expired()
-    auto.create_missing()
-    auto.generate_terraform()
-    auto.terraform_init()
-    auto.terraform_apply()
-    auto.import_terraform()
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        automate()

app/terraform/bridge/gandi.py

@@ -1,9 +1,9 @@
-from app import app
 from app.terraform.bridge import BridgeAutomation
 
 
 class BridgeGandiAutomation(BridgeAutomation):
     short_name = "bridge_gandi"
+    description = "Deploy Tor bridges on GandiCloud VPS"
     provider = "gandi"
 
     template_parameters = [
@@ -78,18 +78,3 @@ class BridgeGandiAutomation(BridgeAutomation):
     {% endfor %}
     {% endfor %}
     """
-
-
-def automate():
-    auto = BridgeGandiAutomation()
-    auto.destroy_expired()
-    auto.create_missing()
-    auto.generate_terraform()
-    auto.terraform_init()
-    auto.terraform_apply()
-    auto.import_terraform()
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        automate()

app/terraform/bridge/hcloud.py

@@ -1,9 +1,9 @@
-from app import app
 from app.terraform.bridge import BridgeAutomation
 
 
 class BridgeHcloudAutomation(BridgeAutomation):
     short_name = "bridge_hcloud"
+    description = "Deploy Tor bridges on Hetzner Cloud"
     provider = "hcloud"
 
     template_parameters = [
@@ -81,18 +81,3 @@ class BridgeHcloudAutomation(BridgeAutomation):
     {% endfor %}
     {% endfor %}
     """
-
-
-def automate():
-    auto = BridgeHcloudAutomation()
-    auto.destroy_expired()
-    auto.create_missing()
-    auto.generate_terraform()
-    auto.terraform_init()
-    auto.terraform_apply()
-    auto.import_terraform()
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        automate()

app/terraform/bridge/ovh.py

@@ -1,9 +1,9 @@
-from app import app
 from app.terraform.bridge import BridgeAutomation
 
 
 class BridgeOvhAutomation(BridgeAutomation):
     short_name = "bridge_ovh"
+    description = "Deploy Tor bridges on OVH Public Cloud"
     provider = "ovh"
 
     template_parameters = [
@@ -104,18 +104,3 @@ class BridgeOvhAutomation(BridgeAutomation):
     {% endfor %}
     {% endfor %}
     """
-
-
-def automate():
-    auto = BridgeOvhAutomation()
-    auto.destroy_expired()
-    auto.create_missing()
-    auto.generate_terraform()
-    auto.terraform_init()
-    auto.terraform_apply()
-    auto.import_terraform()
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        automate()

app/terraform/list/__init__.py

@@ -5,12 +5,12 @@ from app.lists.mirror_mapping import mirror_mapping
 from app.lists.bc2 import mirror_sites
 from app.lists.bridgelines import bridgelines
 from app.models.base import MirrorList
-from app.terraform import BaseAutomation
+from app.terraform.terraform import TerraformAutomation
 
 
-class ListAutomation(BaseAutomation):
+class ListAutomation(TerraformAutomation):
-    def generate_terraform(self):
+    def tf_generate(self):
-        self.write_terraform_config(
+        self.tf_write(
             self.template,
             lists=MirrorList.query.filter(
                 MirrorList.destroyed == None,

app/terraform/list/github.py

@@ -1,9 +1,9 @@
-from app import app
 from app.terraform.list import ListAutomation
 
 
 class ListGithubAutomation(ListAutomation):
     short_name = "list_github"
+    description = "Update mirror lists in GitHub repositories"
     provider = "github"
 
     template_parameters = [
@@ -45,11 +45,3 @@ class ListGithubAutomation(ListAutomation):
     }
     {% endfor %}
     """
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        auto = ListGithubAutomation()
-        auto.generate_terraform()
-        auto.terraform_init()
-        auto.terraform_apply()

app/terraform/list/gitlab.py

@@ -1,9 +1,9 @@
-from app import app
 from app.terraform.list import ListAutomation
 
 
 class ListGitlabAutomation(ListAutomation):
     short_name = "list_gitlab"
+    description = "Update mirror lists in GitLab repositories"
     provider = "gitlab"
 
     template_parameters = [
@@ -44,11 +44,3 @@ class ListGitlabAutomation(ListAutomation):
     
     {% endfor %}
     """
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        auto = ListGitlabAutomation()
-        auto.generate_terraform()
-        auto.terraform_init()
-        auto.terraform_apply()

app/terraform/list/s3.py

@@ -1,9 +1,9 @@
-from app import app
 from app.terraform.list import ListAutomation
 
 
-class ListGithubAutomation(ListAutomation):
+class ListS3Automation(ListAutomation):
     short_name = "list_s3"
+    description = "Update mirror lists in AWS S3 buckets"
     provider = "s3"
 
     template_parameters = [
@@ -36,11 +36,3 @@ class ListGithubAutomation(ListAutomation):
     }
     {% endfor %}
     """
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        auto = ListGithubAutomation()
-        auto.generate_terraform()
-        auto.terraform_init()
-        auto.terraform_apply()

app/terraform/proxy/__init__.py

@@ -37,6 +37,8 @@ class ProxyAutomation(TerraformAutomation):
         for group in groups:
             subgroup = 0
             for origin in group.origins:
+                if origin.destroyed is not None:
+                    continue
                 while True:
                     if subgroups[group.id][subgroup] >= self.subgroup_max:
                         subgroup += 1
@@ -87,7 +89,7 @@ class ProxyAutomation(TerraformAutomation):
         self.deprecate_orphaned_proxies()
         self.destroy_expired_proxies()
 
-    def tf_posthook(self):
+    def tf_posthook(self, *, prehook_result):
         self.import_state(self.tf_show())
 
     def tf_generate(self):

app/terraform/proxy/azure_cdn.py

@@ -1,16 +1,11 @@
-from azure.identity import ClientSecretCredential
-from azure.mgmt.alertsmanagement import AlertsManagementClient
-
-from app import app
-from app.alarms import get_proxy_alarm
 from app.extensions import db
 from app.models.mirrors import Proxy
-from app.models.alarms import AlarmState
 from app.terraform.proxy import ProxyAutomation
 
 
 class ProxyAzureCdnAutomation(ProxyAutomation):
     short_name = "proxy_azure_cdn"
+    description = "Deploy proxies to Azure CDN"
     provider = "azure_cdn"
     subgroup_max = 25
     parallelism = 1
@@ -170,33 +165,3 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
         for proxy in proxies:
             proxy.url = f"https://{proxy.slug}.azureedge.net"
         db.session.commit()
-
-
-def import_monitor_alerts():
-    credential = ClientSecretCredential(
-        tenant_id=app.config['AZURE_TENANT_ID'],
-        client_id=app.config['AZURE_CLIENT_ID'],
-        client_secret=app.config['AZURE_CLIENT_SECRET'])
-    client = AlertsManagementClient(
-        credential,
-        app.config['AZURE_SUBSCRIPTION_ID']
-    )
-    firing = [x.name[len("bandwidth-out-high-bc-"):]
-              for x in client.alerts.get_all()
-              if x.name.startswith("bandwidth-out-high-bc-") and x.properties.essentials.monitor_condition == "Fired"]
-    for proxy in Proxy.query.filter(
-        Proxy.provider == "azure_cdn",
-        Proxy.destroyed == None
-    ):
-        alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
-        if proxy.origin.group.group_name.lower() not in firing:
-            alarm.update_state(AlarmState.OK, "Azure monitor alert not firing")
-        else:
-            alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing")
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        auto = ProxyAzureCdnAutomation()
-        auto.automate()
-        import_monitor_alerts()

app/terraform/proxy/cloudfront.py

@@ -1,17 +1,13 @@
 import datetime
 
-import boto3
-
-from app import app
-from app.alarms import get_proxy_alarm
 from app.extensions import db
-from app.models.alarms import Alarm, AlarmState
 from app.models.mirrors import Proxy
 from app.terraform.proxy import ProxyAutomation
 
 
 class ProxyCloudfrontAutomation(ProxyAutomation):
     short_name = "proxy_cloudfront"
+    description = "Deploy proxies to AWS CloudFront"
     provider = "cloudfront"
 
     template_parameters = [
@@ -87,55 +83,3 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
                         proxy.terraform_updated = datetime.datetime.utcnow()
                         break
         db.session.commit()
-
-
-def import_cloudwatch_alarms():
-    cloudwatch = boto3.client('cloudwatch',
-                              aws_access_key_id=app.config['AWS_ACCESS_KEY'],
-                              aws_secret_access_key=app.config['AWS_SECRET_KEY'],
-                              region_name='us-east-2')
-    dist_paginator = cloudwatch.get_paginator('describe_alarms')
-    page_iterator = dist_paginator.paginate(AlarmNamePrefix="bandwidth-out-high-")
-    for page in page_iterator:
-        for cw_alarm in page['MetricAlarms']:
-            dist_id = cw_alarm["AlarmName"][len("bandwidth-out-high-"):]
-            proxy = Proxy.query.filter(Proxy.slug == dist_id).first()
-            if proxy is None:
-                print("Skipping unknown proxy " + dist_id)
-                continue
-            alarm = get_proxy_alarm(proxy.id, "bandwidth-out-high")
-            if cw_alarm['StateValue'] == "OK":
-                alarm.update_state(AlarmState.OK, "CloudWatch alarm OK")
-            elif cw_alarm['StateValue'] == "ALARM":
-                alarm.update_state(AlarmState.CRITICAL, "CloudWatch alarm ALARM")
-            else:
-                alarm.update_state(AlarmState.UNKNOWN, f"CloudWatch alarm {cw_alarm['StateValue']}")
-    alarm = Alarm.query.filter(
-        Alarm.alarm_type == "cloudfront-quota"
-    ).first()
-    if alarm is None:
-        alarm = Alarm()
-        alarm.target = "service/cloudfront"
-        alarm.alarm_type = "cloudfront-quota"
-        alarm.state_changed = datetime.datetime.utcnow()
-        db.session.add(alarm)
-    alarm.last_updated = datetime.datetime.utcnow()
-    deployed_count = len(Proxy.query.filter(
-        Proxy.destroyed == None).all())
-    old_state = alarm.alarm_state
-    if deployed_count > 370:
-        alarm.alarm_state = AlarmState.CRITICAL
-    elif deployed_count > 320:
-        alarm.alarm_state = AlarmState.WARNING
-    else:
-        alarm.alarm_state = AlarmState.OK
-    if alarm.alarm_state != old_state:
-        alarm.state_changed = datetime.datetime.utcnow()
-    db.session.commit()
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        auto = ProxyCloudfrontAutomation()
-        auto.automate()
-        import_cloudwatch_alarms()

app/terraform/proxy_check.py

@@ -1,62 +0,0 @@
-import requests
-
-from app import app
-from app.extensions import db
-from app.models.alarms import Alarm, AlarmState
-from app.models.mirrors import Proxy
-
-
-def set_http_alarm(proxy_id: int, state: AlarmState, text: str):
-    alarm = Alarm.query.filter(
-        Alarm.proxy_id == proxy_id,
-        Alarm.alarm_type == "http-status"
-    ).first()
-    if alarm is None:
-        alarm = Alarm()
-        alarm.proxy_id = proxy_id
-        alarm.alarm_type = "http-status"
-        alarm.target = "proxy"
-        db.session.add(alarm)
-    alarm.update_state(state, text)
-
-
-def check_http():
-    proxies = Proxy.query.filter(
-        Proxy.destroyed == None
-    )
-    for proxy in proxies:
-        try:
-            if proxy.url is None:
-                continue
-            r = requests.get(proxy.url,
-                             allow_redirects=False,
-                             timeout=5)
-            r.raise_for_status()
-            if r.is_redirect:
-                set_http_alarm(
-                    proxy.id,
-                    AlarmState.CRITICAL,
-                    f"{r.status_code} {r.reason}"
-                )
-            else:
-                set_http_alarm(
-                    proxy.id,
-                    AlarmState.OK,
-                    f"{r.status_code} {r.reason}"
-                )
-        except (requests.ConnectionError, requests.Timeout):
-            set_http_alarm(
-                proxy.id,
-                AlarmState.CRITICAL,
-                f"Connection failure")
-        except requests.HTTPError:
-            set_http_alarm(
-                proxy.id,
-                AlarmState.CRITICAL,
-                f"{r.status_code} {r.reason}"
-            )
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        check_http()

app/terraform/terraform.py

@@ -1,6 +1,6 @@
 import json
 import subprocess
-from typing import Dict, Any, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 import jinja2
 
@@ -18,20 +18,27 @@ class TerraformAutomation(BaseAutomation):
     Default parallelism for remote API calls.
     """
 
-    def automate(self):
+    def automate(self, full: bool = False):
-        self.tf_prehook()
+        prehook_result = self.tf_prehook()
         self.tf_generate()
         self.tf_init()
-        self.tf_apply(refresh=False)
+        returncode, logs = self.tf_apply(refresh=full)
-        self.tf_posthook()
+        self.tf_posthook(prehook_result=prehook_result)
+        return True if returncode == 0 else False, logs
 
-    def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None):
+    def tf_apply(self, refresh: bool = True, parallelism: Optional[int] = None) -> Tuple[int, List[Dict[str, Any]]]:
         if not parallelism:
             parallelism = self.parallelism
-        subprocess.run(
+        tf = subprocess.run(
             ['terraform', 'apply', f'-refresh={str(refresh).lower()}', '-auto-approve',
-             f'-parallelism={str(parallelism)}'],
+             f'-parallelism={str(parallelism)}', '-json'],
-            cwd=self.working_directory())
+            cwd=self.working_directory(),
+            stdout=subprocess.PIPE)
+        logs = []
+        for line in tf.stdout.decode('utf-8').split('\n'):
+            if line.strip():
+                logs.append(json.loads(line))
+        return tf.returncode, logs
 
     def tf_generate(self):
         raise NotImplementedError()
@@ -56,7 +63,7 @@ class TerraformAutomation(BaseAutomation):
         # more like JSON-ND, task is to figure out how to yield those records
         # as plan runs, the same is probably also true for apply
 
-    def tf_posthook(self, prehook_result: Any = None) -> None:
+    def tf_posthook(self, *, prehook_result: Any = None) -> None:
         """
         This hook function is called as part of normal automation, after the
         completion of :func:`tf_apply`.

docs/admin/external.rst

@@ -70,7 +70,9 @@ The expiry can be set according to your threat model.
 GitLab will send an email warning to token owners 7 days before expiry
 allowing you to generate a new token and update your configuration.
 
-Your access token will need the "read_repository" and "write_repository" scopes.
+Your access token will need the "api" scope. Unforunately the "write_repository" scope
+only works for Git-over-HTTPS, but the portal uses the API to update mirror lists in
+GitLab.
 
 Once you've generated your token, you can add it to your ``config.yaml``:
 

migrations/versions/0a0a65db7f01_add_automations.py

@@ -0,0 +1,54 @@
+"""add automations
+
+Revision ID: 0a0a65db7f01
+Revises: c3d6e95caa79
+Create Date: 2022-05-08 16:24:53.779353
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0a0a65db7f01'
+down_revision = 'c3d6e95caa79'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('automation',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('description', sa.String(length=255), nullable=False),
+    sa.Column('added', sa.DateTime(), nullable=False),
+    sa.Column('updated', sa.DateTime(), nullable=False),
+    sa.Column('destroyed', sa.DateTime(), nullable=True),
+    sa.Column('short_name', sa.String(length=25), nullable=False),
+    sa.Column('state', sa.Enum('IDLE', 'RUNNING', 'ERROR', name='automationstate'), nullable=False),
+    sa.Column('enabled', sa.Boolean(), nullable=False),
+    sa.Column('last_run', sa.DateTime(), nullable=True),
+    sa.Column('next_run', sa.DateTime(), nullable=True),
+    sa.Column('next_is_full', sa.Boolean(), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_automation'))
+    )
+    op.create_table('automation_logs',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('added', sa.DateTime(), nullable=False),
+    sa.Column('updated', sa.DateTime(), nullable=False),
+    sa.Column('deprecated', sa.DateTime(), nullable=True),
+    sa.Column('deprecation_reason', sa.String(), nullable=True),
+    sa.Column('destroyed', sa.DateTime(), nullable=True),
+    sa.Column('automation_id', sa.Integer(), nullable=False),
+    sa.Column('logs', sa.Text(), nullable=True),
+    sa.ForeignKeyConstraint(['automation_id'], ['automation.id'], name=op.f('fk_automation_logs_automation_id_automation')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_automation_logs'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('automation_logs')
+    op.drop_table('automation')
+    # ### end Alembic commands ###