feat(static): adds new static origins feature

app/portal/__init__.py

@@ -26,6 +26,7 @@ from app.portal.onion import bp as onion
 from app.portal.pool import bp as pool
 from app.portal.proxy import bp as proxy
 from app.portal.smart_proxy import bp as smart_proxy
+from app.portal.static import bp as static
 from app.portal.storage import bp as storage
 from app.portal.webhook import bp as webhook
 
@@ -42,6 +43,7 @@ portal.register_blueprint(onion, url_prefix="/onion")
 portal.register_blueprint(pool, url_prefix="/pool")
 portal.register_blueprint(proxy, url_prefix="/proxy")
 portal.register_blueprint(smart_proxy, url_prefix="/smart")
+portal.register_blueprint(static, url_prefix="/static")
 portal.register_blueprint(storage, url_prefix="/state")
 portal.register_blueprint(webhook, url_prefix="/webhook")
 

app/portal/cloud.py

@@ -53,6 +53,16 @@ class HcloudAccountForm(FlaskForm):  # type: ignore
     submit = SubmitField('Save Changes')
 
 
+class GitlabAccountForm(FlaskForm):  # type: ignore
+    provider = StringField('Platform', render_kw={"disabled": ""})
+    description = StringField('Description', validators=[InputRequired()])
+    gitlab_token = StringField('GitLab Access Token', validators=[InputRequired()])
+    enabled = BooleanField('Enable this account', default=True,
+                           description="New resources will not be deployed to disabled accounts, however existing "
+                                       "resources will persist until destroyed at the end of their lifecycle.")
+    submit = SubmitField('Save Changes')
+
+
 class OvhHorizonForm(Form):  # type: ignore[misc]
     ovh_openstack_user = StringField("User")
     ovh_openstack_password = StringField("Password")
@@ -102,6 +112,7 @@ provider_forms: Dict[str, Type[CloudAccountForm]] = {
     CloudProvider.AWS.name: AWSAccountForm,
     CloudProvider.HCLOUD.name: HcloudAccountForm,
     CloudProvider.GANDI.name: GandiAccountForm,
+    CloudProvider.GITLAB.name: GitlabAccountForm,
     CloudProvider.OVH.name: OvhAccountForm,
 }
 
@@ -140,6 +151,10 @@ def cloud_account_save(account: Optional[CloudAccount], provider: CloudProvider,
         account.max_distributions = 0
         account.max_sub_distributions = 0
         account.max_instances = form.max_instances.data
+    elif provider == CloudProvider.GITLAB and isinstance(form, GitlabAccountForm):
+        account.credentials = {
+            "gitlab_token": form.gitlab_token.data,
+        }
     elif provider == CloudProvider.OVH and isinstance(form, OvhAccountForm):
         account.credentials = {
             "ovh_openstack_user": form.horizon.data["ovh_openstack_user"],
@@ -174,6 +189,8 @@ def cloud_account_populate(form: CloudAccountForm, account: CloudAccount) -> Non
         form.horizon.form.gandi_openstack_password.data = account.credentials["gandi_openstack_password"]
         form.horizon.form.gandi_openstack_tenant_id.data = account.credentials["gandi_openstack_tenant_id"]
         form.max_instances.data = account.max_instances
+    elif account.provider == CloudProvider.GITLAB and isinstance(form, GitlabAccountForm):
+        form.gitlab_token.data = account.credentials["gitlab_token"]
     elif account.provider == CloudProvider.OVH and isinstance(form, OvhAccountForm):
         form.horizon.form.ovh_openstack_user.data = account.credentials["ovh_openstack_user"]
         form.horizon.form.ovh_openstack_password.data = account.credentials["ovh_openstack_password"]

app/portal/static.py

@@ -0,0 +1,207 @@
+import logging
+from typing import Optional, List, Any
+
+from flask import flash, redirect, url_for, render_template, Response, Blueprint, current_app
+from flask.typing import ResponseReturnValue
+from flask_wtf import FlaskForm
+from sqlalchemy import exc
+from wtforms import StringField, SelectField, SubmitField, BooleanField, FileField
+from wtforms.validators import DataRequired
+
+from app.brm.static import create_static_origin
+from app.models.base import Group
+from app.models.cloud import CloudAccount, CloudProvider
+from app.models.mirrors import StaticOrigin
+from app.portal.util import response_404, view_lifecycle
+
+bp = Blueprint("static", __name__)
+
+
+class StaticOriginForm(FlaskForm):  # type: ignore
+    description = StringField(
+        'Description',
+        validators=[DataRequired()],
+        description='Enter a brief description of the static website that you are creating in this field. This is '
+                    'also a required field.'
+    )
+    group = SelectField(
+        'Group',
+        validators=[DataRequired()],
+        description='Select the group that you want the origin to belong to from the drop-down menu in this field. '
+                    'This is a required field.'
+    )
+    storage_cloud_account = SelectField(
+        'Storage Cloud Account',
+        validators=[DataRequired()],
+        description='Select the cloud account that you want the origin to be deployed to from the drop-down menu in '
+                    'this field. This is a required field.'
+    )
+    source_cloud_account = SelectField(
+        'Source Cloud Account',
+        validators=[DataRequired()],
+        description='Select the cloud account that will be used to modify the source repository for the web content '
+                    'for this static origin. This is a required field.'
+    )
+    source_project = StringField(
+        'Source Project',
+        validators=[DataRequired()],
+        description='GitLab project path.'
+    )
+    auto_rotate = BooleanField(
+        'Auto-Rotate',
+        default=True,
+        description='Select this field if you want to enable auto-rotation for the mirror. This means that the mirror '
+                    'will automatically redeploy with a new domain name if it is detected to be blocked. This field '
+                    'is optional and is enabled by default.'
+    )
+    matrix_homeserver = SelectField(
+        'Matrix Homeserver',
+        description='Select the Matrix homeserver from the drop-down box to enable Keanu Convene on mirrors of this '
+                    'static origin.'
+    )
+    keanu_convene_path = StringField(
+        'Keanu Convene Path',
+        default='talk',
+        description='Enter the subdirectory to present the Keanu Convene application at on the mirror. This defaults '
+                    'to "talk".'
+    )
+    keanu_convene_logo = FileField(
+        'Keanu Convene Logo',
+        description='Logo to use for Keanu Convene'
+    )
+    keanu_convene_color = StringField(
+        'Keanu Convene Accent Color',
+        default='#0047ab',
+        description='Accent color to use for Keanu Convene (HTML hex code)'
+    )
+    enable_clean_insights = BooleanField(
+        'Enable Clean Insights',
+        description='When enabled, a Clean Insights Measurement Proxy endpoint is deployed on the mirror to allow for '
+                    'submission of results from any of the supported Clean Insights SDKs.'
+    )
+    submit = SubmitField('Save Changes')
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.group.choices = [(x.id, x.group_name) for x in Group.query.all()]
+        self.storage_cloud_account.choices = [(x.id, f"{x.provider.description} - {x.description}") for x in
+                                              CloudAccount.query.filter(
+                                                  CloudAccount.provider == CloudProvider.AWS).all()]
+        self.source_cloud_account.choices = [(x.id, f"{x.provider.description} - {x.description}") for x in
+                                             CloudAccount.query.filter(
+                                                 CloudAccount.provider == CloudProvider.GITLAB).all()]
+        self.matrix_homeserver.choices = [(x, x) for x in current_app.config['MATRIX_HOMESERVERS']]
+
+
+@bp.route("/new", methods=['GET', 'POST'])
+@bp.route("/new/<group_id>", methods=['GET', 'POST'])
+def static_new(group_id: Optional[int] = None) -> ResponseReturnValue:
+    form = StaticOriginForm()
+    if len(form.source_cloud_account.choices) == 0 or len(form.storage_cloud_account.choices) == 0:
+        flash("You must add at least one AWS account and at least one GitLab account before creating static origins.",
+              "warning")
+        return redirect(url_for("portal.cloud.cloud_account_list"))
+    if form.validate_on_submit():
+        try:
+            static = create_static_origin(
+                form.description.data,
+                int(form.group.data),
+                int(form.storage_cloud_account.data),
+                int(form.source_cloud_account.data),
+                form.source_project.data,
+                form.auto_rotate.data,
+                form.matrix_homeserver.data,
+                form.keanu_convene_path.data,
+                form.keanu_convene_logo.data,
+                form.keanu_convene_color.data,
+                form.enable_clean_insights.data,
+                True
+            )
+            flash(f"Created new static origin #{static.id}.", "success")
+            return redirect(url_for("portal.static.static_edit", static_id=static.id))
+        except ValueError as e:  # may be returned by create_static_origin and from the int conversion
+            logging.warning(e)
+            flash("Failed to create new static origin due to an invalid input.", "danger")
+            return redirect(url_for("portal.static.static_list"))
+        except exc.SQLAlchemyError as e:
+            flash("Failed to create new static origin due to a database error.", "danger")
+            logging.warning(e)
+            return redirect(url_for("portal.static.static_list"))
+    if group_id:
+        form.group.data = group_id
+    return render_template("new.html.j2", section="static", form=form)
+
+
+@bp.route('/edit/<static_id>', methods=['GET', 'POST'])
+def static_edit(static_id: int) -> ResponseReturnValue:
+    static: Optional[StaticOrigin] = StaticOrigin.query.filter(StaticOrigin.id == static_id).first()
+    if static is None:
+        return Response(render_template("error.html.j2",
+                                        section="static",
+                                        header="404 Origin Not Found",
+                                        message="The requested static origin could not be found."),
+                        status=404)
+    form = StaticOriginForm(description=static.description,
+                            group=static.group_id,
+                            storage_cloud_account=static.storage_cloud_account_id,
+                            source_cloud_account=static.source_cloud_account_id,
+                            source_project=static.source_project,
+                            matrix_homeserver=static.matrix_homeserver,
+                            keanu_convene_path=static.keanu_convene_path,
+                            auto_rotate=static.auto_rotate,
+                            enable_clean_insights=bool(static.clean_insights_backend))
+    form.group.render_kw = {"disabled": ""}
+    form.storage_cloud_account.render_kw = {"disabled": ""}
+    form.source_cloud_account.render_kw = {"disabled": ""}
+    if form.validate_on_submit():
+        try:
+            static.update(
+                form.source_project.data,
+                form.description.data,
+                form.auto_rotate.data,
+                form.matrix_homeserver.data,
+                form.keanu_convene_path.data,
+                form.keanu_convene_logo.data,
+                form.keanu_convene_color.data,
+                form.enable_clean_insights.data,
+                True
+            )
+            flash("Saved changes to group.", "success")
+        except ValueError as e:  # may be returned by create_static_origin and from the int conversion
+            logging.warning(e)
+            flash("An error occurred saving the changes to the static origin due to an invalid input.", "danger")
+        except exc.SQLAlchemyError as e:
+            logging.warning(e)
+            flash("An error occurred saving the changes to the static origin due to a database error.", "danger")
+    return render_template("static.html.j2",
+                           section="static",
+                           static=static, form=form)
+
+
+@bp.route("/list")
+def static_list() -> ResponseReturnValue:
+    statics: List[StaticOrigin] = StaticOrigin.query.order_by(StaticOrigin.description).all()
+    return render_template("list.html.j2",
+                           section="static",
+                           title="Static Origins",
+                           item="static",
+                           new_link=url_for("portal.static.static_new"),
+                           items=statics
+                           )
+
+
+@bp.route("/destroy/<static_id>", methods=['GET', 'POST'])
+def static_destroy(static_id: int) -> ResponseReturnValue:
+    static = StaticOrigin.query.filter(StaticOrigin.id == static_id, StaticOrigin.destroyed.is_(None)).first()
+    if static is None:
+        return response_404("The requested static origin could not be found.")
+    return view_lifecycle(
+        header=f"Destroy static origin {static.description}",
+        message=static.description,
+        success_message="All proxies from the destroyed static origin will shortly be destroyed at their providers, "
+                        "and the static content will be removed from the cloud provider.",
+        success_view="portal.static.static_list",
+        section="static",
+        resource=static,
+        action="destroy"
+    )

app/portal/templates/base.html.j2

@@ -82,7 +82,7 @@
                 </h6>
                 <ul class="nav flex-column">
                     <li class="nav-item">
-                        <a class="nav-link{% if section == "cloud" %} active{% else %} disabled text-secondary{% endif %}"
+                        <a class="nav-link{% if section == "cloud" %} active{% endif %}"
                            href="{{ url_for("portal.cloud.cloud_account_list") }}">
                             {{ icon("cloud") }} Cloud Accounts
                         </a>
@@ -106,8 +106,8 @@
                         </a>
                     </li>
                     <li class="nav-item">
-                        <a class="nav-link{% if section == "origin" %} active{% endif %} disabled text-secondary"
-                           href="{{ url_for("portal.origin.origin_list") }}">
+                        <a class="nav-link{% if section == "static" %} active{% endif %}"
+                           href="{{ url_for("portal.static.static_list") }}">
                             {{ icon("hdd") }} Static Origins
                         </a>
                     </li>

app/portal/templates/list.html.j2

@@ -1,7 +1,7 @@
 {% extends "base.html.j2" %}
 {% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, cloud_accounts_table,
         eotk_table, groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table,
-        pools_table, proxies_table, webhook_table %}
+        pools_table, proxies_table, static_table, webhook_table %}
 
 {% block content %}
     <h1 class="h2 mt-3">{{ title }}</h1>
@@ -35,6 +35,8 @@
         {% endif %}
     {% elif item == "origin" %}
         {{ origins_table(items) }}
+    {% elif item == "static" %}
+        {{ static_table(items) }}
     {% elif item == "pool" %}
         {{ pools_table(items) }}
     {% elif item == "proxy" %}

app/portal/templates/static.html.j2

@@ -0,0 +1,15 @@
+{% extends "base.html.j2" %}
+{% from 'bootstrap5/form.html' import render_form %}
+{% from "tables.html.j2" import static_table %}
+
+{% block content %}
+    <h1 class="h2 mt-3">Static Origins</h1>
+    <h2 class="h3">
+        {{ static.group.group_name }}: {{ static.description }} (#{{ static.id }})
+    </h2>
+
+    <div style="border: 1px solid #666;" class="p-3">
+        {{ render_form(form) }}
+    </div>
+
+{% endblock %}

app/portal/templates/tables.html.j2

@@ -633,6 +633,57 @@
     </div>
 {% endmacro %}
 
+{% macro static_table(statics) %}
+    <div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Description</th>
+                <th scope="col">Origin URL</th>
+                <th scope="col">Keanu Convene</th>
+                <th scope="col">Auto-Rotation</th>
+                <th scope="col">Group</th>
+                <th scope="col">Actions</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for static in statics %}
+                {% if not static.destroyed %}
+                    <tr>
+                        <td>{{ static.description }}</td>
+                        <td>{% if static.origin_domain_name %}
+                            <a href="https://{{ static.origin_domain_name }}" class="btn btn-secondary btn-sm"
+                               target="_bypass"
+                               rel="noopener noreferer">⎋</a>
+                               {{ static.origin_domain_name }}
+                            {% else %}
+                                <em>Still deploying</em>
+                            {% endif %}</td>
+                        <td>{% if static.origin_domain_name and static.keanu_convene_path %}
+                            <a href="https://{{ static.origin_domain_name }}/{{ static.keanu_convene_path }}/" class="btn btn-secondary btn-sm"
+                               target="_bypass"
+                               rel="noopener noreferer">⎋</a>
+                            {% else %}
+                                <em>Still deploying</em>
+                            {% endif %}</td>
+                        <td>{% if static.auto_rotate %}✅{% else %}❌{% endif %}</td>
+                        <td>
+                            <a href="{{ url_for("portal.group.group_edit", group_id=static.group.id) }}">{{ static.group.group_name }}</a>
+                        </td>
+                        <td>
+                            <a href="{{ url_for("portal.static.static_edit", static_id=static.id) }}"
+                               class="btn btn-primary btn-sm">View/Edit</a>
+                            <a href="{{ url_for("portal.static.static_destroy", static_id=static.id) }}"
+                               class="btn btn-danger btn-sm">Destroy</a>
+                        </td>
+                    </tr>
+                {% endif %}
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endmacro %}
+
 {% macro webhook_table(webhooks) %}
     <div class="table-responsive">
         <table class="table table-striped table-sm">