activity: basic webhook alerts for automation failures

app/cli/automate.py

@@ -6,6 +6,7 @@ from traceback import TracebackException
 
 from app import app
 from app.extensions import db
+from app.models.activity import Activity
 from app.models.automation import Automation, AutomationState, AutomationLogs
 from app.terraform import BaseAutomation
 from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation
@@ -102,6 +103,11 @@ def run_job(job: BaseAutomation, *, force: bool = False, ignore_schedule: bool =
         log.updated = datetime.datetime.utcnow()
         log.logs = json.dumps(logs)
         db.session.add(log)
+        activity = Activity(
+            activity_type="automation",
+            text=f"FLASH! Automation Failure: {automation.short_name}. See logs for details."
+        )
+        activity.notify()  # Notify before commit because the failure occurred even if we can't commit.
     automation.last_run = datetime.datetime.utcnow()
     db.session.commit()
 

app/models/__init__.py

@@ -38,6 +38,13 @@ class AbstractResource(db.Model):
     deprecation_reason = db.Column(db.String(), nullable=True)
     destroyed = db.Column(db.DateTime(), nullable=True)
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if self.added is None:
+            self.added = datetime.utcnow()
+        if self.updated is None:
+            self.updated = datetime.utcnow()
+
     def deprecate(self, *, reason: str):
         self.deprecated = datetime.utcnow()
         self.deprecation_reason = reason

app/models/activity.py

@@ -0,0 +1,46 @@
+import datetime
+
+import requests
+
+from app.models import AbstractConfiguration
+from app.extensions import db
+
+
+class Activity(db.Model):
+    id = db.Column(db.Integer(), primary_key=True)
+    group_id = db.Column(db.Integer(), nullable=True)
+    activity_type = db.Column(db.String(20), nullable=False)
+    text = db.Column(db.Text(), nullable=False)
+    added = db.Column(db.DateTime(), nullable=False)
+
+    def __init__(self, **kwargs):
+        if type(kwargs["activity_type"]) != str or len(kwargs["activity_type"]) > 20 or kwargs["activity_type"] == "":
+            raise TypeError("expected string for activity type between 1 and 20 characters")
+        if type(kwargs["text"]) != str:
+            raise TypeError("expected string for text")
+        if "added" not in kwargs:
+            kwargs["added"] = datetime.datetime.utcnow()
+        super().__init__(**kwargs)
+
+    def notify(self) -> int:
+        count = 0
+        hooks = Webhook.query.filter(
+            Webhook.destroyed == None
+        )
+        for hook in hooks:
+            hook.send(self.text)
+            count += 1
+        return count
+
+
+class Webhook(AbstractConfiguration):
+    format = db.Column(db.String(20))
+    url = db.Column(db.String(255))
+
+    def send(self, text: str):
+        if self.format == "telegram":
+            data = {"text": text}
+        else:
+            # Matrix as default
+            data = {"body": text}
+        r = requests.post(self.url, json=data)

app/portal/__init__.py

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone
 from flask import Blueprint, render_template, request
 from sqlalchemy import desc, or_
 
+from app.models.activity import Activity
 from app.models.alarms import Alarm
 from app.models.mirrors import Origin, Proxy
 from app.models.base import Group
@@ -16,6 +17,7 @@ from app.portal.list import bp as list_
 from app.portal.origin import bp as origin
 from app.portal.onion import bp as onion
 from app.portal.proxy import bp as proxy
+from app.portal.webhook import bp as webhook
 
 portal = Blueprint("portal", __name__, template_folder="templates", static_folder="static")
 portal.register_blueprint(automation, url_prefix="/automation")
@@ -27,6 +29,7 @@ portal.register_blueprint(list_, url_prefix="/list")
 portal.register_blueprint(origin, url_prefix="/origin")
 portal.register_blueprint(onion, url_prefix="/onion")
 portal.register_blueprint(proxy, url_prefix="/proxy")
+portal.register_blueprint(webhook, url_prefix="/webhook")
 
 
 @portal.app_template_filter("mirror_expiry")
@@ -57,8 +60,9 @@ def portal_home():
         s: len(Alarm.query.filter(Alarm.alarm_state == s.upper(), Alarm.last_updated > (now - timedelta(days=1))).all())
         for s in ["critical", "warning", "ok", "unknown"]
     }
+    activity = Activity.query.filter(Activity.added > (now - timedelta(days=2))).order_by(desc(Activity.added)).all()
     return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72,
-                           lastweek=lastweek, proxies=proxies, **alarms)
+                           lastweek=lastweek, proxies=proxies, **alarms, activity=activity)
 
 
 @portal.route("/search")

app/portal/templates/base.html.j2

@@ -114,6 +114,12 @@
                             {{ icon("file-earmark-excel") }} Block Lists
                         </a>
                     </li>
+                    <li class="nav-item">
+                        <a class="nav-link{% if section == "webhook" %} active{% endif %}"
+                           href="{{ url_for("portal.webhook.webhook_list") }}">
+                            {{ icon("activity") }} Webhooks
+                        </a>
+                    </li>
                 </ul>
                 <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
                     <span>Infrastructure</span>

app/portal/templates/home.html.j2

@@ -44,4 +44,21 @@
             </div>
         </div>
     </div>
+    <div class="row mt-4">
+        <div class="col">
+            <div class="card h-100">
+                <h3 class="h4 card-header">Activity</h3>
+                <div class="card-body">
+                    <table class="table table-striped">
+                        {% for a in activity %}
+                        <tr>
+                            <td>{{ a.text }}</td>
+                            <td>{{ a.added | format_datetime }}</td>
+                        </tr>
+                        {% endfor %}
+                    </table>
+                </div>
+            </div>
+        </div>
+    </div>
 {% endblock %}

app/portal/templates/icons.html.j2

@@ -1,5 +1,11 @@
 {% macro icon(i) %}
-    {% if i == "arrow-down-up" %}
+    {% if i == "activity" %}
+        <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-activity"
+             viewBox="0 0 16 16">
+            <path fill-rule="evenodd"
+                  d="M6 2a.5.5 0 0 1 .47.33L10 12.036l1.53-4.208A.5.5 0 0 1 12 7.5h3.5a.5.5 0 0 1 0 1h-3.15l-1.88 5.17a.5.5 0 0 1-.94 0L6 3.964 4.47 8.171A.5.5 0 0 1 4 8.5H.5a.5.5 0 0 1 0-1h3.15l1.88-5.17A.5.5 0 0 1 6 2Z"/>
+        </svg>
+    {% elif i == "arrow-down-up" %}
         <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-arrow-down-up"
              viewBox="0 0 16 16">
             <path fill-rule="evenodd"

app/portal/templates/list.html.j2

@@ -1,6 +1,7 @@
 {% extends "base.html.j2" %}
 {% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, eotk_table,
-        groups_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, proxies_table %}
+        groups_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, proxies_table,
+        webhook_table %}
 
 {% block content %}
     <h1 class="h2 mt-3">{{ title }}</h1>
@@ -34,5 +35,7 @@
         {{ origins_table(items) }}
     {% elif item == "proxy" %}
         {{ proxies_table(items) }}
+    {% elif item == "webhook" %}
+        {{ webhook_table(items) }}
     {% endif %}
 {% endblock %}

app/portal/templates/tables.html.j2

@@ -529,4 +529,33 @@
             </tbody>
         </table>
     </div>
+{% endmacro %}
+
+{% macro webhook_table(webhooks) %}
+    <div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Description</th>
+                <th scope="col">Format</th>
+                <th scope="col">URL</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for webhook in webhooks %}
+                {% if not webhook.destroyed %}
+                    <tr class="align-middle">
+                        <td>{{ webhook.description }}</td>
+                        <td>{{ webhook.format | webhook_format_name }}</td>
+                        <td><code>{{ webhook.url }}</code></td>
+                        <td>
+                            <a href="{{ url_for("portal.webhook.webhook_destroy", webhook_id=webhook.id) }}"
+                               class="btn btn-danger btn-sm">Destroy</a>
+                        </td>
+                    </tr>
+                {% endif %}
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
 {% endmacro %}

app/portal/util.py

@@ -4,6 +4,7 @@ from wtforms import SubmitField
 
 from app import db
 from app.models import AbstractResource
+from app.models.activity import Activity
 
 
 def response_404(message: str):
@@ -37,7 +38,13 @@ def view_lifecycle(*,
         else:
             flash("Unknown action")
             return redirect(url_for("portal.portal_home"))
+        activity = Activity(
+            activity_type="lifecycle",
+            text=f"Portal action: {message}. {success_message}"
+        )
+        db.session.add(activity)
         db.session.commit()
+        activity.notify()
         flash(success_message, "success")
         return redirect(url_for(success_view))
     return render_template("lifecycle.html.j2",

app/portal/webhook.py

@@ -0,0 +1,105 @@
+from datetime import datetime
+
+from flask import Blueprint, flash, Response, render_template, redirect, url_for
+from flask_wtf import FlaskForm
+from sqlalchemy import exc
+from wtforms import StringField, SelectField, SubmitField
+from wtforms.validators import DataRequired
+
+from app.extensions import db
+from app.models.activity import Webhook
+from app.portal.util import response_404, view_lifecycle
+
+bp = Blueprint("webhook", __name__)
+
+@bp.app_template_filter("webhook_format_name")
+def webhook_format_name(s: str) -> str:
+    if s == "telegram":
+        return "Telegram"
+    if s == "matrix":
+        return "Matrix"
+
+
+class NewWebhookForm(FlaskForm):
+    description = StringField('Description', validators=[DataRequired()])
+    format = SelectField('Format', choices=[
+        ("telegram", "Telegram"),
+        ("matrix", "Matrix")
+    ], validators=[DataRequired()])
+    url = StringField('URL', validators=[DataRequired()])
+    submit = SubmitField('Save Changes')
+
+
+@bp.route("/new", methods=['GET', 'POST'])
+def webhook_new():
+    form = NewWebhookForm()
+    if form.validate_on_submit():
+        webhook = Webhook(
+            description=form.description.data,
+            format=form.format.data,
+            url=form.url.data
+        )
+        try:
+            db.session.add(webhook)
+            db.session.commit()
+            flash(f"Created new webhook {webhook.url}.", "success")
+            return redirect(url_for("portal.webhook.webhook_edit", webhook_id=webhook.id))
+        except exc.SQLAlchemyError as e:
+            flash("Failed to create new webhook.", "danger")
+            return redirect(url_for("portal.webhook.webhook_list"))
+    return render_template("new.html.j2", section="webhook", form=form)
+
+
+@bp.route('/edit/<webhook_id>', methods=['GET', 'POST'])
+def webhook_edit(webhook_id):
+    webhook = Webhook.query.filter(Webhook.id == webhook_id).first()
+    if webhook is None:
+        return Response(render_template("error.html.j2",
+                                        section="webhook",
+                                        header="404 Webhook Not Found",
+                                        message="The requested webhook could not be found."),
+                        status=404)
+    form = NewWebhookForm(description=webhook.description,
+                          format=webhook.format,
+                          url=webhook.url)
+    if form.validate_on_submit():
+        webhook.description = form.description.data
+        webhook.format = form.description.data
+        webhook.url = form.description.data
+        webhook.updated = datetime.utcnow()
+        try:
+            db.session.commit()
+            flash("Saved changes to webhook.", "success")
+        except exc.SQLAlchemyError:
+            flash("An error occurred saving the changes to the webhook.", "danger")
+    return render_template("edit.html.j2",
+                           section="webhook",
+                           title="Edit Webhook",
+                           item=webhook, form=form)
+
+
+@bp.route("/list")
+def webhook_list():
+    webhooks = Webhook.query.all()
+    return render_template("list.html.j2",
+                           section="webhook",
+                           title="Webhooks",
+                           item="webhook",
+                           new_link=url_for("portal.webhook.webhook_new"),
+                           items=webhooks)
+
+
+@bp.route("/destroy/<webhook_id>", methods=['GET', 'POST'])
+def webhook_destroy(webhook_id: int):
+    webhook = Webhook.query.filter(Webhook.id == webhook_id, Webhook.destroyed == None).first()
+    if webhook is None:
+        return response_404("The requested webhook could not be found.")
+    return view_lifecycle(
+        header=f"Destroy webhook {webhook.url}",
+        message=webhook.description,
+        success_message="Webhook destroyed.",
+        success_view="portal.webhook.webhook_list",
+        section="webhook",
+        resource=webhook,
+        action="destroy"
+    )

migrations/versions/6f3e327e3b87_activities.py

@@ -0,0 +1,61 @@
+"""activities
+
+Revision ID: 6f3e327e3b87
+Revises: 7ecfb305d243
+Create Date: 2022-05-14 09:10:57.320077
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '6f3e327e3b87'
+down_revision = '7ecfb305d243'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('activity',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('group_id', sa.Integer(), nullable=True),
+    sa.Column('activity_type', sa.String(length=20), nullable=False),
+    sa.Column('text', sa.Text(), nullable=False),
+    sa.Column('added', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_activity'))
+    )
+    op.create_table('webhook',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('description', sa.String(length=255), nullable=False),
+    sa.Column('added', sa.DateTime(), nullable=False),
+    sa.Column('updated', sa.DateTime(), nullable=False),
+    sa.Column('destroyed', sa.DateTime(), nullable=True),
+    sa.Column('format', sa.String(length=20), nullable=True),
+    sa.Column('url', sa.String(length=255), nullable=True),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_webhook'))
+    )
+    op.drop_table('eotk_instance')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('eotk_instance',
+    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('added', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.Column('updated', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.Column('deprecated', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('deprecation_reason', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('destroyed', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('group_id', sa.INTEGER(), autoincrement=False, nullable=False),
+    sa.Column('provider', sa.VARCHAR(length=20), autoincrement=False, nullable=False),
+    sa.Column('region', sa.VARCHAR(length=20), autoincrement=False, nullable=False),
+    sa.Column('instance_id', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
+    sa.ForeignKeyConstraint(['group_id'], ['group.id'], name='fk_eotk_instance_group_id_group'),
+    sa.PrimaryKeyConstraint('id', name='pk_eotk_instance')
+    )
+    op.drop_table('webhook')
+    op.drop_table('activity')
+    # ### end Alembic commands ###