diff --git a/app/cli/automate.py b/app/cli/automate.py index 21e0b00..6ab0905 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -6,6 +6,7 @@ from traceback import TracebackException from app import app from app.extensions import db +from app.models.activity import Activity from app.models.automation import Automation, AutomationState, AutomationLogs from app.terraform import BaseAutomation from app.terraform.block_bridge_github import BlockBridgeGitHubAutomation @@ -102,6 +103,11 @@ def run_job(job: BaseAutomation, *, force: bool = False, ignore_schedule: bool = log.updated = datetime.datetime.utcnow() log.logs = json.dumps(logs) db.session.add(log) + activity = Activity( + activity_type="automation", + text=f"FLASH! Automation Failure: {automation.short_name}. See logs for details." + ) + activity.notify() # Notify before commit because the failure occurred even if we can't commit. automation.last_run = datetime.datetime.utcnow() db.session.commit() diff --git a/app/models/__init__.py b/app/models/__init__.py index 784b65e..a448384 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -38,6 +38,13 @@ class AbstractResource(db.Model): deprecation_reason = db.Column(db.String(), nullable=True) destroyed = db.Column(db.DateTime(), nullable=True) + def __init__(self, **kwargs): + super().__init__(**kwargs) + if self.added is None: + self.added = datetime.utcnow() + if self.updated is None: + self.updated = datetime.utcnow() + def deprecate(self, *, reason: str): self.deprecated = datetime.utcnow() self.deprecation_reason = reason diff --git a/app/models/activity.py b/app/models/activity.py new file mode 100644 index 0000000..9a4b87b --- /dev/null +++ b/app/models/activity.py @@ -0,0 +1,46 @@ +import datetime + +import requests + +from app.models import AbstractConfiguration +from app.extensions import db + + +class Activity(db.Model): + id = db.Column(db.Integer(), primary_key=True) + group_id = db.Column(db.Integer(), nullable=True) + activity_type = db.Column(db.String(20), nullable=False) + text = db.Column(db.Text(), nullable=False) + added = db.Column(db.DateTime(), nullable=False) + + def __init__(self, **kwargs): + if type(kwargs["activity_type"]) != str or len(kwargs["activity_type"]) > 20 or kwargs["activity_type"] == "": + raise TypeError("expected string for activity type between 1 and 20 characters") + if type(kwargs["text"]) != str: + raise TypeError("expected string for text") + if "added" not in kwargs: + kwargs["added"] = datetime.datetime.utcnow() + super().__init__(**kwargs) + + def notify(self) -> int: + count = 0 + hooks = Webhook.query.filter( + Webhook.destroyed == None + ) + for hook in hooks: + hook.send(self.text) + count += 1 + return count + + +class Webhook(AbstractConfiguration): + format = db.Column(db.String(20)) + url = db.Column(db.String(255)) + + def send(self, text: str): + if self.format == "telegram": + data = {"text": text} + else: + # Matrix as default + data = {"body": text} + r = requests.post(self.url, json=data) diff --git a/app/portal/__init__.py b/app/portal/__init__.py index ac2987a..432219b 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone from flask import Blueprint, render_template, request from sqlalchemy import desc, or_ +from app.models.activity import Activity from app.models.alarms import Alarm from app.models.mirrors import Origin, Proxy from app.models.base import Group @@ -16,6 +17,7 @@ from app.portal.list import bp as list_ from app.portal.origin import bp as origin from app.portal.onion import bp as onion from app.portal.proxy import bp as proxy +from app.portal.webhook import bp as webhook portal = Blueprint("portal", __name__, template_folder="templates", static_folder="static") portal.register_blueprint(automation, url_prefix="/automation") @@ -27,6 +29,7 @@ portal.register_blueprint(list_, url_prefix="/list") portal.register_blueprint(origin, url_prefix="/origin") portal.register_blueprint(onion, url_prefix="/onion") portal.register_blueprint(proxy, url_prefix="/proxy") +portal.register_blueprint(webhook, url_prefix="/webhook") @portal.app_template_filter("mirror_expiry") @@ -57,8 +60,9 @@ def portal_home(): s: len(Alarm.query.filter(Alarm.alarm_state == s.upper(), Alarm.last_updated > (now - timedelta(days=1))).all()) for s in ["critical", "warning", "ok", "unknown"] } + activity = Activity.query.filter(Activity.added > (now - timedelta(days=2))).order_by(desc(Activity.added)).all() return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72, - lastweek=lastweek, proxies=proxies, **alarms) + lastweek=lastweek, proxies=proxies, **alarms, activity=activity) @portal.route("/search") diff --git a/app/portal/templates/base.html.j2 b/app/portal/templates/base.html.j2 index c763d4f..76c2065 100644 --- a/app/portal/templates/base.html.j2 +++ b/app/portal/templates/base.html.j2 @@ -114,6 +114,12 @@ {{ icon("file-earmark-excel") }} Block Lists +
  • + + {{ icon("activity") }} Webhooks + +
    • Infrastructure diff --git a/app/portal/templates/home.html.j2 b/app/portal/templates/home.html.j2 index 4f629c9..99f7de8 100644 --- a/app/portal/templates/home.html.j2 +++ b/app/portal/templates/home.html.j2 @@ -44,4 +44,21 @@ +
    +
    +
    +

    Activity

    +
    + + {% for a in activity %} + + + + + {% endfor %} +
    {{ a.text }}{{ a.added | format_datetime }}
    +
    +
    +
    +
    {% endblock %} \ No newline at end of file diff --git a/app/portal/templates/icons.html.j2 b/app/portal/templates/icons.html.j2 index 79a23f3..ceb2b0b 100644 --- a/app/portal/templates/icons.html.j2 +++ b/app/portal/templates/icons.html.j2 @@ -1,5 +1,11 @@ {% macro icon(i) %} - {% if i == "arrow-down-up" %} + {% if i == "activity" %} + + + + {% elif i == "arrow-down-up" %} {{ title }}
    @@ -34,5 +35,7 @@ {{ origins_table(items) }} {% elif item == "proxy" %} {{ proxies_table(items) }} + {% elif item == "webhook" %} + {{ webhook_table(items) }} {% endif %} {% endblock %} diff --git a/app/portal/templates/tables.html.j2 b/app/portal/templates/tables.html.j2 index 235c930..32473c8 100644 --- a/app/portal/templates/tables.html.j2 +++ b/app/portal/templates/tables.html.j2 @@ -529,4 +529,33 @@ +{% endmacro %} + +{% macro webhook_table(webhooks) %} +
    + + + + + + + + + + {% for webhook in webhooks %} + {% if not webhook.destroyed %} + + + + + + + {% endif %} + {% endfor %} + +
    DescriptionFormatURL
    {{ webhook.description }}{{ webhook.format | webhook_format_name }}{{ webhook.url }} + Destroy +
    +
    {% endmacro %} \ No newline at end of file diff --git a/app/portal/util.py b/app/portal/util.py index bc0b839..affe139 100644 --- a/app/portal/util.py +++ b/app/portal/util.py @@ -4,6 +4,7 @@ from wtforms import SubmitField from app import db from app.models import AbstractResource +from app.models.activity import Activity def response_404(message: str): @@ -37,7 +38,13 @@ def view_lifecycle(*, else: flash("Unknown action") return redirect(url_for("portal.portal_home")) + activity = Activity( + activity_type="lifecycle", + text=f"Portal action: {message}. {success_message}" + ) + db.session.add(activity) db.session.commit() + activity.notify() flash(success_message, "success") return redirect(url_for(success_view)) return render_template("lifecycle.html.j2", diff --git a/app/portal/webhook.py b/app/portal/webhook.py new file mode 100644 index 0000000..ad8c8ce --- /dev/null +++ b/app/portal/webhook.py @@ -0,0 +1,105 @@ +from datetime import datetime + +from flask import Blueprint, flash, Response, render_template, redirect, url_for +from flask_wtf import FlaskForm +from sqlalchemy import exc +from wtforms import StringField, SelectField, SubmitField +from wtforms.validators import DataRequired + +from app.extensions import db +from app.models.activity import Webhook +from app.portal.util import response_404, view_lifecycle + +bp = Blueprint("webhook", __name__) + +@bp.app_template_filter("webhook_format_name") +def webhook_format_name(s: str) -> str: + if s == "telegram": + return "Telegram" + if s == "matrix": + return "Matrix" + + +class NewWebhookForm(FlaskForm): + description = StringField('Description', validators=[DataRequired()]) + format = SelectField('Format', choices=[ + ("telegram", "Telegram"), + ("matrix", "Matrix") + ], validators=[DataRequired()]) + url = StringField('URL', validators=[DataRequired()]) + submit = SubmitField('Save Changes') + + +@bp.route("/new", methods=['GET', 'POST']) +def webhook_new(): + form = NewWebhookForm() + if form.validate_on_submit(): + webhook = Webhook( + description=form.description.data, + format=form.format.data, + url=form.url.data + ) + try: + db.session.add(webhook) + db.session.commit() + flash(f"Created new webhook {webhook.url}.", "success") + return redirect(url_for("portal.webhook.webhook_edit", webhook_id=webhook.id)) + except exc.SQLAlchemyError as e: + flash("Failed to create new webhook.", "danger") + return redirect(url_for("portal.webhook.webhook_list")) + return render_template("new.html.j2", section="webhook", form=form) + + +@bp.route('/edit/', methods=['GET', 'POST']) +def webhook_edit(webhook_id): + webhook = Webhook.query.filter(Webhook.id == webhook_id).first() + if webhook is None: + return Response(render_template("error.html.j2", + section="webhook", + header="404 Webhook Not Found", + message="The requested webhook could not be found."), + status=404) + form = NewWebhookForm(description=webhook.description, + format=webhook.format, + url=webhook.url) + if form.validate_on_submit(): + webhook.description = form.description.data + webhook.format = form.description.data + webhook.url = form.description.data + webhook.updated = datetime.utcnow() + try: + db.session.commit() + flash("Saved changes to webhook.", "success") + except exc.SQLAlchemyError: + flash("An error occurred saving the changes to the webhook.", "danger") + return render_template("edit.html.j2", + section="webhook", + title="Edit Webhook", + item=webhook, form=form) + + +@bp.route("/list") +def webhook_list(): + webhooks = Webhook.query.all() + return render_template("list.html.j2", + section="webhook", + title="Webhooks", + item="webhook", + new_link=url_for("portal.webhook.webhook_new"), + items=webhooks) + + +@bp.route("/destroy/", methods=['GET', 'POST']) +def webhook_destroy(webhook_id: int): + webhook = Webhook.query.filter(Webhook.id == webhook_id, Webhook.destroyed == None).first() + if webhook is None: + return response_404("The requested webhook could not be found.") + return view_lifecycle( + header=f"Destroy webhook {webhook.url}", + message=webhook.description, + success_message="Webhook destroyed.", + success_view="portal.webhook.webhook_list", + section="webhook", + resource=webhook, + action="destroy" + ) diff --git a/migrations/versions/6f3e327e3b87_activities.py b/migrations/versions/6f3e327e3b87_activities.py new file mode 100644 index 0000000..e3ca832 --- /dev/null +++ b/migrations/versions/6f3e327e3b87_activities.py @@ -0,0 +1,61 @@ +"""activities + +Revision ID: 6f3e327e3b87 +Revises: 7ecfb305d243 +Create Date: 2022-05-14 09:10:57.320077 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '6f3e327e3b87' +down_revision = '7ecfb305d243' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('activity', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('group_id', sa.Integer(), nullable=True), + sa.Column('activity_type', sa.String(length=20), nullable=False), + sa.Column('text', sa.Text(), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_activity')) + ) + op.create_table('webhook', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.String(length=255), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('destroyed', sa.DateTime(), nullable=True), + sa.Column('format', sa.String(length=20), nullable=True), + sa.Column('url', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_webhook')) + ) + op.drop_table('eotk_instance') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('eotk_instance', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('added', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('updated', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('deprecated', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('deprecation_reason', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('destroyed', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('group_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('provider', sa.VARCHAR(length=20), autoincrement=False, nullable=False), + sa.Column('region', sa.VARCHAR(length=20), autoincrement=False, nullable=False), + sa.Column('instance_id', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['group_id'], ['group.id'], name='fk_eotk_instance_group_id_group'), + sa.PrimaryKeyConstraint('id', name='pk_eotk_instance') + ) + op.drop_table('webhook') + op.drop_table('activity') + # ### end Alembic commands ###