From 51779b6cc3df88a8e11e6c6baf5bcdfdc0a0a018 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Wed, 31 May 2023 13:59:45 +0100 Subject: [PATCH] feat(static): create web origins for the static mirrors --- app/cli/automate.py | 2 + app/models/mirrors.py | 3 +- app/portal/static.py | 35 +++++++++------- app/portal/templates/static.html.j2 | 7 +++- app/terraform/static/meta.py | 62 +++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 app/terraform/static/meta.py diff --git a/app/cli/automate.py b/app/cli/automate.py index 96884f6..bd22db6 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -40,6 +40,7 @@ from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation from app.terraform.proxy.fastly import ProxyFastlyAutomation from app.terraform.static.aws import StaticAWSAutomation +from app.terraform.static.meta import StaticMetaAutomation jobs = { x.short_name: x # type: ignore[attr-defined] @@ -56,6 +57,7 @@ jobs = { # Create new resources BridgeMetaAutomation, + StaticMetaAutomation, ProxyMetaAutomation, # Terraform diff --git a/app/models/mirrors.py b/app/models/mirrors.py index 438cf09..c35e58a 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -81,9 +81,8 @@ class StaticOrigin(AbstractConfiguration): foreign_keys=[source_cloud_account_id]) def destroy(self) -> None: + # TODO: The StaticMetaAutomation will clean up for now, but it should probably happen here for consistency super().destroy() - for proxy in self.proxies: - proxy.destroy() def update( self, diff --git a/app/portal/static.py b/app/portal/static.py index c725ca5..dd82c35 100644 --- a/app/portal/static.py +++ b/app/portal/static.py @@ -1,6 +1,7 @@ import logging from typing import Optional, List, Any +import sqlalchemy.exc from flask import flash, redirect, url_for, render_template, Response, Blueprint, current_app from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm @@ -11,7 +12,7 @@ from wtforms.validators import DataRequired from app.brm.static import create_static_origin from app.models.base import Group from app.models.cloud import CloudAccount, CloudProvider -from app.models.mirrors import StaticOrigin +from app.models.mirrors import StaticOrigin, Origin from app.portal.util import response_404, view_lifecycle bp = Blueprint("static", __name__) @@ -134,28 +135,28 @@ def static_new(group_id: Optional[int] = None) -> ResponseReturnValue: @bp.route('/edit/', methods=['GET', 'POST']) def static_edit(static_id: int) -> ResponseReturnValue: - static: Optional[StaticOrigin] = StaticOrigin.query.filter(StaticOrigin.id == static_id).first() - if static is None: + static_origin: Optional[StaticOrigin] = StaticOrigin.query.filter(StaticOrigin.id == static_id).first() + if static_origin is None: return Response(render_template("error.html.j2", section="static", header="404 Origin Not Found", message="The requested static origin could not be found."), status=404) - form = StaticOriginForm(description=static.description, - group=static.group_id, - storage_cloud_account=static.storage_cloud_account_id, - source_cloud_account=static.source_cloud_account_id, - source_project=static.source_project, - matrix_homeserver=static.matrix_homeserver, - keanu_convene_path=static.keanu_convene_path, - auto_rotate=static.auto_rotate, - enable_clean_insights=bool(static.clean_insights_backend)) + form = StaticOriginForm(description=static_origin.description, + group=static_origin.group_id, + storage_cloud_account=static_origin.storage_cloud_account_id, + source_cloud_account=static_origin.source_cloud_account_id, + source_project=static_origin.source_project, + matrix_homeserver=static_origin.matrix_homeserver, + keanu_convene_path=static_origin.keanu_convene_path, + auto_rotate=static_origin.auto_rotate, + enable_clean_insights=bool(static_origin.clean_insights_backend)) form.group.render_kw = {"disabled": ""} form.storage_cloud_account.render_kw = {"disabled": ""} form.source_cloud_account.render_kw = {"disabled": ""} if form.validate_on_submit(): try: - static.update( + static_origin.update( form.source_project.data, form.description.data, form.auto_rotate.data, @@ -173,9 +174,15 @@ def static_edit(static_id: int) -> ResponseReturnValue: except exc.SQLAlchemyError as e: logging.warning(e) flash("An error occurred saving the changes to the static origin due to a database error.", "danger") + try: + origin = Origin.query.filter_by(domain_name=static_origin.origin_domain_name).one() + proxies = origin.proxies + except sqlalchemy.exc.NoResultFound: + proxies = [] return render_template("static.html.j2", section="static", - static=static, form=form) + static=static_origin, form=form, + proxies=proxies) @bp.route("/list") diff --git a/app/portal/templates/static.html.j2 b/app/portal/templates/static.html.j2 index b8683a1..cbde9d0 100644 --- a/app/portal/templates/static.html.j2 +++ b/app/portal/templates/static.html.j2 @@ -1,6 +1,6 @@ {% extends "base.html.j2" %} {% from 'bootstrap5/form.html' import render_form %} -{% from "tables.html.j2" import static_table %} +{% from "tables.html.j2" import proxies_table, static_table %} {% block content %}

Static Origins

@@ -12,4 +12,9 @@ {{ render_form(form) }} + {% if proxies %} +

Proxies

+ {{ proxies_table(proxies) }} + {% endif %} + {% endblock %} diff --git a/app/terraform/static/meta.py b/app/terraform/static/meta.py new file mode 100644 index 0000000..1efe61f --- /dev/null +++ b/app/terraform/static/meta.py @@ -0,0 +1,62 @@ +import logging +from typing import Tuple + +from sqlalchemy.orm.exc import NoResultFound + +from app.extensions import db +from app.models.mirrors import Origin, StaticOrigin +from app.terraform import BaseAutomation + + +class StaticMetaAutomation(BaseAutomation): + short_name = "static_meta" + description = "Housekeeping for static origins" + frequency = 1 + + def automate(self, full: bool = False) -> Tuple[bool, str]: + """ + Create Origins for each StaticOrigin where an Origin does not already exist. + Set origin_domain_name of the StaticOrigin to the domain_name of the Origin. + Update Origin's auto_rotation attribute to match StaticOrigin's setting. + Remove Origins for StaticOrigins with non-null destroy value. + """ + + # Step 1: Create Origins for StaticOrigins without existing Origins + static_origins = StaticOrigin.query.all() + for static_origin in static_origins: + if static_origin.origin_domain_name is not None: + try: + # Check if an Origin with the same domain name already exists + origin = Origin.query.filter_by(domain_name=static_origin.origin_domain_name).one() + # Keep auto rotation value in sync + origin.auto_rotation = static_origin.auto_rotate + except NoResultFound: + # Create a new Origin since it doesn't exist + origin = Origin( + group_id=static_origin.group_id, + description=f"PORTAL !! DO NOT DELETE !! Automatically created web origin for static origin " + f"#{static_origin.id}", + domain_name=static_origin.origin_domain_name, + auto_rotation=static_origin.auto_rotate, + smart=False, + assets=False, + ) + db.session.add(origin) + logging.debug(f"Created Origin with domain name {origin.domain_name}") + + # Step 2: Remove Origins for StaticOrigins with non-null destroy value + static_origins_with_destroyed = StaticOrigin.query.filter(StaticOrigin.destroyed.isnot(None)).all() + for static_origin in static_origins_with_destroyed: + try: + origin = Origin.query.filter_by( + domain_name=static_origin.origin_domain_name, + destroyed=None, + ).one() + origin.destroy() + logging.debug(f"Destroyed Origin with domain name {origin.domain_name}") + except NoResultFound: + continue + + db.session.commit() + + return True, ""