feat(static): create web origins for the static mirrors

app/cli/automate.py

@@ -40,6 +40,7 @@ from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
 from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
 from app.terraform.proxy.fastly import ProxyFastlyAutomation
 from app.terraform.static.aws import StaticAWSAutomation
+from app.terraform.static.meta import StaticMetaAutomation
 
 jobs = {
     x.short_name: x  # type: ignore[attr-defined]
@@ -56,6 +57,7 @@ jobs = {
 
         # Create new resources
         BridgeMetaAutomation,
+        StaticMetaAutomation,
         ProxyMetaAutomation,
 
         # Terraform

app/models/mirrors.py

@@ -81,9 +81,8 @@ class StaticOrigin(AbstractConfiguration):
                                            foreign_keys=[source_cloud_account_id])
 
     def destroy(self) -> None:
+        # TODO: The StaticMetaAutomation will clean up for now, but it should probably happen here for consistency
         super().destroy()
-        for proxy in self.proxies:
-            proxy.destroy()
 
     def update(
             self,

app/portal/static.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Optional, List, Any
 
+import sqlalchemy.exc
 from flask import flash, redirect, url_for, render_template, Response, Blueprint, current_app
 from flask.typing import ResponseReturnValue
 from flask_wtf import FlaskForm
@@ -11,7 +12,7 @@ from wtforms.validators import DataRequired
 from app.brm.static import create_static_origin
 from app.models.base import Group
 from app.models.cloud import CloudAccount, CloudProvider
-from app.models.mirrors import StaticOrigin
+from app.models.mirrors import StaticOrigin, Origin
 from app.portal.util import response_404, view_lifecycle
 
 bp = Blueprint("static", __name__)
@@ -134,28 +135,28 @@ def static_new(group_id: Optional[int] = None) -> ResponseReturnValue:
 
 @bp.route('/edit/<static_id>', methods=['GET', 'POST'])
 def static_edit(static_id: int) -> ResponseReturnValue:
-    static: Optional[StaticOrigin] = StaticOrigin.query.filter(StaticOrigin.id == static_id).first()
+    static_origin: Optional[StaticOrigin] = StaticOrigin.query.filter(StaticOrigin.id == static_id).first()
-    if static is None:
+    if static_origin is None:
         return Response(render_template("error.html.j2",
                                         section="static",
                                         header="404 Origin Not Found",
                                         message="The requested static origin could not be found."),
                         status=404)
-    form = StaticOriginForm(description=static.description,
+    form = StaticOriginForm(description=static_origin.description,
-                            group=static.group_id,
+                            group=static_origin.group_id,
-                            storage_cloud_account=static.storage_cloud_account_id,
+                            storage_cloud_account=static_origin.storage_cloud_account_id,
-                            source_cloud_account=static.source_cloud_account_id,
+                            source_cloud_account=static_origin.source_cloud_account_id,
-                            source_project=static.source_project,
+                            source_project=static_origin.source_project,
-                            matrix_homeserver=static.matrix_homeserver,
+                            matrix_homeserver=static_origin.matrix_homeserver,
-                            keanu_convene_path=static.keanu_convene_path,
+                            keanu_convene_path=static_origin.keanu_convene_path,
-                            auto_rotate=static.auto_rotate,
+                            auto_rotate=static_origin.auto_rotate,
-                            enable_clean_insights=bool(static.clean_insights_backend))
+                            enable_clean_insights=bool(static_origin.clean_insights_backend))
     form.group.render_kw = {"disabled": ""}
     form.storage_cloud_account.render_kw = {"disabled": ""}
     form.source_cloud_account.render_kw = {"disabled": ""}
     if form.validate_on_submit():
         try:
-            static.update(
+            static_origin.update(
                 form.source_project.data,
                 form.description.data,
                 form.auto_rotate.data,
@@ -173,9 +174,15 @@ def static_edit(static_id: int) -> ResponseReturnValue:
         except exc.SQLAlchemyError as e:
             logging.warning(e)
             flash("An error occurred saving the changes to the static origin due to a database error.", "danger")
+    try:
+        origin = Origin.query.filter_by(domain_name=static_origin.origin_domain_name).one()
+        proxies = origin.proxies
+    except sqlalchemy.exc.NoResultFound:
+        proxies = []
     return render_template("static.html.j2",
                            section="static",
-                           static=static, form=form)
+                           static=static_origin, form=form,
+                           proxies=proxies)
 
 
 @bp.route("/list")

app/portal/templates/static.html.j2

@@ -1,6 +1,6 @@
 {% extends "base.html.j2" %}
 {% from 'bootstrap5/form.html' import render_form %}
-{% from "tables.html.j2" import static_table %}
+{% from "tables.html.j2" import proxies_table, static_table %}
 
 {% block content %}
     <h1 class="h2 mt-3">Static Origins</h1>
@@ -12,4 +12,9 @@
         {{ render_form(form) }}
     </div>
 
+    {% if proxies %}
+    <h3>Proxies</h3>
+    {{ proxies_table(proxies) }}
+    {% endif %}
+
 {% endblock %}

app/terraform/static/meta.py

@@ -0,0 +1,62 @@
+import logging
+from typing import Tuple
+
+from sqlalchemy.orm.exc import NoResultFound
+
+from app.extensions import db
+from app.models.mirrors import Origin, StaticOrigin
+from app.terraform import BaseAutomation
+
+
+class StaticMetaAutomation(BaseAutomation):
+    short_name = "static_meta"
+    description = "Housekeeping for static origins"
+    frequency = 1
+
+    def automate(self, full: bool = False) -> Tuple[bool, str]:
+        """
+        Create Origins for each StaticOrigin where an Origin does not already exist.
+        Set origin_domain_name of the StaticOrigin to the domain_name of the Origin.
+        Update Origin's auto_rotation attribute to match StaticOrigin's setting.
+        Remove Origins for StaticOrigins with non-null destroy value.
+        """
+
+        # Step 1: Create Origins for StaticOrigins without existing Origins
+        static_origins = StaticOrigin.query.all()
+        for static_origin in static_origins:
+            if static_origin.origin_domain_name is not None:
+                try:
+                    # Check if an Origin with the same domain name already exists
+                    origin = Origin.query.filter_by(domain_name=static_origin.origin_domain_name).one()
+                    # Keep auto rotation value in sync
+                    origin.auto_rotation = static_origin.auto_rotate
+                except NoResultFound:
+                    # Create a new Origin since it doesn't exist
+                    origin = Origin(
+                        group_id=static_origin.group_id,
+                        description=f"PORTAL !! DO NOT DELETE !! Automatically created web origin for static origin "
+                                    f"#{static_origin.id}",
+                        domain_name=static_origin.origin_domain_name,
+                        auto_rotation=static_origin.auto_rotate,
+                        smart=False,
+                        assets=False,
+                    )
+                    db.session.add(origin)
+                    logging.debug(f"Created Origin with domain name {origin.domain_name}")
+
+        # Step 2: Remove Origins for StaticOrigins with non-null destroy value
+        static_origins_with_destroyed = StaticOrigin.query.filter(StaticOrigin.destroyed.isnot(None)).all()
+        for static_origin in static_origins_with_destroyed:
+            try:
+                origin = Origin.query.filter_by(
+                    domain_name=static_origin.origin_domain_name,
+                    destroyed=None,
+                ).one()
+                origin.destroy()
+                logging.debug(f"Destroyed Origin with domain name {origin.domain_name}")
+            except NoResultFound:
+                continue
+
+        db.session.commit()
+
+        return True, ""