feat(static): adds new static origins feature

.gitmodules

@@ -25,3 +25,6 @@
 [submodule "terraform-modules/terraform-aws-bc-eotk"]
 	path = terraform-modules/terraform-aws-bc-eotk
 	url = https://github.com/sr2c/terraform-aws-bc-eotk.git
+[submodule "terraform-modules/terraform-aws-bc-static-origin"]
+	path = terraform-modules/terraform-aws-bc-static-origin
+	url = https://gitlab.com/sr2c/terraform-aws-bc-static-origin.git

app/brm/static.py

@@ -0,0 +1,77 @@
+from typing import Optional, Union
+
+from werkzeug.datastructures import FileStorage
+
+from app.extensions import db
+from app.models.base import Group
+from app.models.cloud import CloudAccount
+from app.models.mirrors import StaticOrigin
+
+
+def create_static_origin(
+        description: str,
+        group_id: int,
+        storage_cloud_account_id: int,
+        source_cloud_account_id: int,
+        source_project: str,
+        auto_rotate: bool,
+        matrix_homeserver: Optional[str],
+        keanu_convene_path: Optional[str],
+        keanu_convene_logo: Optional[FileStorage],
+        keanu_convene_color: Optional[str],
+        clean_insights_backend: Optional[Union[str, bool]],
+        db_session_commit: bool = False,
+) -> StaticOrigin:
+    """
+    Create a new static origin.
+
+    :param description: The description of the new static origin.
+    :param group_id: The ID for the origin group to add this new static origin to.
+    :param storage_cloud_account_id: The ID for the cloud account to deploy this new static origin to.
+    :param source_cloud_account_id: The ID for the cloud account used to interact with the web sources.
+    :param source_project: The path for the source project, e.g. the GitLab project path.
+    :param auto_rotate: Whether to automatically rotate this domain when it is detected to be blocked.
+    :param matrix_homeserver: The domain name for the Matrix homeserver to proxy to.
+    :param keanu_convene_path: The path to serve the Keanu Convene application from.
+    :param clean_insights_backend: The domain name for the Clean Insights backend to proxy to.
+    :param db_session_commit: Whether to add the new StaticOrigin to the database session and commit it.
+    :returns: StaticOrigin -- the newly created StaticOrigin
+    :raises: ValueError, sqlalchemy.exc.SQLAlchemyError
+    """
+    static_origin = StaticOrigin()
+    if isinstance(group_id, int):
+        group = Group.query.filter(Group.id == group_id).first()
+        if group is None:
+            raise ValueError("group_id must match an existing group")
+        static_origin.group_id = group_id
+    else:
+        raise ValueError("group_id must be an int")
+    if isinstance(storage_cloud_account_id, int):
+        cloud_account = CloudAccount.query.filter(CloudAccount.id == storage_cloud_account_id).first()
+        if cloud_account is None:
+            raise ValueError("storage_cloud_account_id must match an existing provider")
+        static_origin.storage_cloud_account_id = storage_cloud_account_id
+    else:
+        raise ValueError("storage_cloud_account_id must be an int")
+    if isinstance(source_cloud_account_id, int):
+        cloud_account = CloudAccount.query.filter(CloudAccount.id == source_cloud_account_id).first()
+        if cloud_account is None:
+            raise ValueError("source_cloud_account_id must match an existing provider")
+        static_origin.source_cloud_account_id = source_cloud_account_id
+    else:
+        raise ValueError("source_cloud_account_id must be an int")
+    static_origin.update(
+        source_project,
+        description,
+        auto_rotate,
+        matrix_homeserver,
+        keanu_convene_path,
+        keanu_convene_logo,
+        keanu_convene_color,
+        clean_insights_backend,
+        False
+    )
+    if db_session_commit:
+        db.session.add(static_origin)
+        db.session.commit()
+    return static_origin

app/brm/utils.py

@@ -1,6 +1,12 @@
 from __future__ import annotations
 
-from typing import Any
+import base64
+from io import BytesIO
+from typing import Any, Tuple
+
+import webcolors
+from PIL import Image
+from werkzeug.datastructures import FileStorage
 
 
 def is_integer(contender: Any) -> bool:
@@ -18,3 +24,68 @@ def is_integer(contender: Any) -> bool:
         return False
     else:
         return float(contender).is_integer()
+
+
+def thumbnail_uploaded_image(file: FileStorage, max_size: Tuple[int, int] = (256, 256)) -> bytes:
+    """
+    Process an uploaded image file into a resized image of a specific size.
+
+    :param file: An uploaded image file.
+    :param max_size: A tuple containing the maximum width and height for the thumbnail image. Default is (256, 256).
+    :return: The byte data of the thumbnail image.
+    """
+    if file.filename is None:
+        raise ValueError("No file was uploaded")
+    img = Image.open(file)
+    img.thumbnail(max_size)
+    byte_arr = BytesIO()
+    img.save(byte_arr, format='PNG' if file.filename.lower().endswith('.png') else 'JPEG')
+    return byte_arr.getvalue()
+
+
+def create_data_uri(bytes_data: bytes, file_extension: str) -> str:
+    """
+    Create a data URI from binary data and a file extension.
+
+    :param bytes_data: The binary data of an image.
+    :param file_extension: The file extension of the image.
+    :return: A data URI representing the image.
+    """
+    # base64 encode
+    encoded = base64.b64encode(bytes_data).decode('ascii')
+    # create data URI
+    data_uri = "data:image/{};base64,{}".format('jpeg' if file_extension == 'jpg' else file_extension, encoded)
+    return data_uri
+
+
+def normalize_color(color: str) -> str:
+    """
+    Normalize a string representing a color to its hexadecimal representation.
+
+    This function accepts a string representing a color in one of the following formats:
+
+    - A CSS color name, such as 'red', 'green', or 'blue'.
+    - A 6-digit hexadecimal color code, such as '#FF0000' for red.
+    - A 3-digit hexadecimal color code, such as '#F00' for red.
+
+    The function returns a string with the color in 6-digit or 3-digit hexadecimal format,
+    with lowercase letters. If the color is given as a CSS color name, the function
+    returns its equivalent 6-digit hexadecimal format.
+
+    :param color: A string representing a color.
+    :return: The color in 6-digit or 3-digit hexadecimal format.
+    :raises: ValueError: If the input string does not represent a valid color in any of the accepted formats.
+    """
+    try:
+        return webcolors.name_to_hex(color)  # type: ignore[no-any-return]
+    except ValueError:
+        pass
+    if color.startswith('#'):
+        color = color[1:].lower()
+    if len(color) in [3, 6]:
+        try:
+            _ = int(color, 16)
+            return f"#{color}"
+        except ValueError:
+            pass
+    raise ValueError(f"color must be a valid HTML color, got: {repr(color)}")

app/cli/automate.py

@@ -39,6 +39,7 @@ from app.terraform.proxy.meta import ProxyMetaAutomation
 from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
 from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
 from app.terraform.proxy.fastly import ProxyFastlyAutomation
+from app.terraform.static.aws import StaticAWSAutomation
 
 jobs = {
     x.short_name: x  # type: ignore[attr-defined]
@@ -62,6 +63,7 @@ jobs = {
         BridgeGandiAutomation,
         BridgeHcloudAutomation,
         BridgeOvhAutomation,
+        StaticAWSAutomation,
         EotkAWSAutomation,
         ProxyAzureCdnAutomation,
         ProxyCloudfrontAutomation,

app/models/base.py

@@ -11,6 +11,7 @@ class Group(AbstractConfiguration):
     eotk = db.Column(db.Boolean())
 
     origins = db.relationship("Origin", back_populates="group")
+    statics = db.relationship("StaticOrigin", back_populates="group")
     eotks = db.relationship("Eotk", back_populates="group")
     onions = db.relationship("Onion", back_populates="group")
     smart_proxies = db.relationship("SmartProxy", back_populates="group")

app/models/cloud.py

@@ -3,6 +3,7 @@ import enum
 from app.brm.brn import BRN
 from app.extensions import db
 from app.models import AbstractConfiguration
+from app.models.mirrors import StaticOrigin
 
 
 class CloudProvider(enum.Enum):
@@ -36,6 +37,8 @@ class CloudAccount(AbstractConfiguration):
     max_instances = db.Column(db.Integer())
 
     bridges = db.relationship("Bridge", back_populates="cloud_account")
+    statics = db.relationship("StaticOrigin", back_populates="storage_cloud_account", foreign_keys=[
+        StaticOrigin.storage_cloud_account_id])
 
     @property
     def brn(self) -> BRN:

app/models/mirrors.py

@@ -1,8 +1,12 @@
-from typing import Optional, List
+import json
+from datetime import datetime
+from typing import Optional, List, Union, Any, Dict
 
 from tldextract import extract
+from werkzeug.datastructures import FileStorage
 
 from app.brm.brn import BRN
+from app.brm.utils import thumbnail_uploaded_image, create_data_uri, normalize_color
 from app.extensions import db
 from app.models import AbstractConfiguration, AbstractResource
 from app.models.onions import Onion
@@ -25,7 +29,7 @@ class Origin(AbstractConfiguration):
             product="mirror",
             provider="conf",
             resource_type="origin",
-            resource_id="self.domain_name"
+            resource_id=self.domain_name
         )
 
     @classmethod
@@ -48,6 +52,104 @@ class Origin(AbstractConfiguration):
         return f"https://{domain_name.replace(tld, onion.onion_name)}.onion"
 
 
+class StaticOrigin(AbstractConfiguration):
+    group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
+    storage_cloud_account_id = db.Column(db.Integer(), db.ForeignKey("cloud_account.id"), nullable=False)
+    source_cloud_account_id = db.Column(db.Integer(), db.ForeignKey("cloud_account.id"), nullable=False)
+    source_project = db.Column(db.String(255), nullable=False)
+    auto_rotate = db.Column(db.Boolean, nullable=False)
+    matrix_homeserver = db.Column(db.String(255), nullable=True)
+    keanu_convene_path = db.Column(db.String(255), nullable=True)
+    keanu_convene_config = db.Column(db.String(), nullable=True)
+    clean_insights_backend = db.Column(db.String(255), nullable=True)
+    origin_domain_name = db.Column(db.String(255), nullable=True)
+
+    @property
+    def brn(self) -> BRN:
+        return BRN(
+            group_id=self.group_id,
+            product="mirror",
+            provider="aws",
+            resource_type="static",
+            resource_id=self.domain_name
+        )
+
+    group = db.relationship("Group", back_populates="statics")
+    storage_cloud_account = db.relationship("CloudAccount", back_populates="statics",
+                                            foreign_keys=[storage_cloud_account_id])
+    source_cloud_account = db.relationship("CloudAccount", back_populates="statics",
+                                           foreign_keys=[source_cloud_account_id])
+
+    def destroy(self) -> None:
+        super().destroy()
+        for proxy in self.proxies:
+            proxy.destroy()
+
+    def update(
+            self,
+            source_project: str,
+            description: str,
+            auto_rotate: bool,
+            matrix_homeserver: Optional[str],
+            keanu_convene_path: Optional[str],
+            keanu_convene_logo: Optional[FileStorage],
+            keanu_convene_color: Optional[str],
+            clean_insights_backend: Optional[Union[str, bool]],
+            db_session_commit: bool,
+    ) -> None:
+        if isinstance(source_project, str):
+            self.source_project = source_project
+        else:
+            raise ValueError("source project must be a str")
+        if isinstance(description, str):
+            self.description = description
+        else:
+            raise ValueError("description must be a str")
+        if isinstance(auto_rotate, bool):
+            self.auto_rotate = auto_rotate
+        else:
+            raise ValueError("auto_rotate must be a bool")
+        if isinstance(matrix_homeserver, str):
+            self.matrix_homeserver = matrix_homeserver
+        else:
+            raise ValueError("matrix_homeserver must be a str")
+        if isinstance(keanu_convene_path, str):
+            self.keanu_convene_path = keanu_convene_path
+        else:
+            raise ValueError("keanu_convene_path must be a str")
+        if self.keanu_convene_config is None:
+            self.keanu_convene_config = "{}"
+        keanu_convene_config: Dict[str, Any] = json.loads(self.keanu_convene_config)
+        if keanu_convene_logo is None:
+            pass
+        elif isinstance(keanu_convene_logo, FileStorage):
+            if keanu_convene_logo.filename:  # if False, no file was uploaded
+                keanu_convene_config["logo"] = create_data_uri(
+                    thumbnail_uploaded_image(keanu_convene_logo), keanu_convene_logo.filename)
+        else:
+            raise ValueError("keanu_convene_logo must be a FileStorage")
+        try:
+            if isinstance(keanu_convene_color, str):
+                keanu_convene_config["color"] = normalize_color(keanu_convene_color)  # can raise ValueError
+            else:
+                raise ValueError()  # re-raised below with message
+        except ValueError:
+            raise ValueError("keanu_convene_path must be a str containing an HTML color (CSS name or hex)")
+        self.keanu_convene_config = json.dumps(keanu_convene_config, separators=(',', ':'))
+        del keanu_convene_config  # done with this temporary variable
+        if clean_insights_backend is None or (isinstance(clean_insights_backend, bool) and not clean_insights_backend):
+            self.clean_insights_backend = None
+        elif isinstance(clean_insights_backend, bool) and clean_insights_backend:
+            self.clean_insights_backend = "metrics.cleaninsights.org"
+        elif isinstance(clean_insights_backend, str):
+            self.clean_insights_backend = clean_insights_backend
+        else:
+            raise ValueError("clean_insights_backend must be a str, bool, or None")
+        if db_session_commit:
+            db.session.commit()
+        self.updated = datetime.utcnow()
+
+
 class Proxy(AbstractResource):
     origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)
     pool_id = db.Column(db.Integer, db.ForeignKey("pool.id"))

app/portal/__init__.py

@@ -26,6 +26,7 @@ from app.portal.onion import bp as onion
 from app.portal.pool import bp as pool
 from app.portal.proxy import bp as proxy
 from app.portal.smart_proxy import bp as smart_proxy
+from app.portal.static import bp as static
 from app.portal.storage import bp as storage
 from app.portal.webhook import bp as webhook
 
@@ -42,6 +43,7 @@ portal.register_blueprint(onion, url_prefix="/onion")
 portal.register_blueprint(pool, url_prefix="/pool")
 portal.register_blueprint(proxy, url_prefix="/proxy")
 portal.register_blueprint(smart_proxy, url_prefix="/smart")
+portal.register_blueprint(static, url_prefix="/static")
 portal.register_blueprint(storage, url_prefix="/state")
 portal.register_blueprint(webhook, url_prefix="/webhook")
 

app/portal/cloud.py

@@ -53,6 +53,16 @@ class HcloudAccountForm(FlaskForm):  # type: ignore
     submit = SubmitField('Save Changes')
 
 
+class GitlabAccountForm(FlaskForm):  # type: ignore
+    provider = StringField('Platform', render_kw={"disabled": ""})
+    description = StringField('Description', validators=[InputRequired()])
+    gitlab_token = StringField('GitLab Access Token', validators=[InputRequired()])
+    enabled = BooleanField('Enable this account', default=True,
+                           description="New resources will not be deployed to disabled accounts, however existing "
+                                       "resources will persist until destroyed at the end of their lifecycle.")
+    submit = SubmitField('Save Changes')
+
+
 class OvhHorizonForm(Form):  # type: ignore[misc]
     ovh_openstack_user = StringField("User")
     ovh_openstack_password = StringField("Password")
@@ -102,6 +112,7 @@ provider_forms: Dict[str, Type[CloudAccountForm]] = {
     CloudProvider.AWS.name: AWSAccountForm,
     CloudProvider.HCLOUD.name: HcloudAccountForm,
     CloudProvider.GANDI.name: GandiAccountForm,
+    CloudProvider.GITLAB.name: GitlabAccountForm,
     CloudProvider.OVH.name: OvhAccountForm,
 }
 
@@ -140,6 +151,10 @@ def cloud_account_save(account: Optional[CloudAccount], provider: CloudProvider,
         account.max_distributions = 0
         account.max_sub_distributions = 0
         account.max_instances = form.max_instances.data
+    elif provider == CloudProvider.GITLAB and isinstance(form, GitlabAccountForm):
+        account.credentials = {
+            "gitlab_token": form.gitlab_token.data,
+        }
     elif provider == CloudProvider.OVH and isinstance(form, OvhAccountForm):
         account.credentials = {
             "ovh_openstack_user": form.horizon.data["ovh_openstack_user"],
@@ -174,6 +189,8 @@ def cloud_account_populate(form: CloudAccountForm, account: CloudAccount) -> Non
         form.horizon.form.gandi_openstack_password.data = account.credentials["gandi_openstack_password"]
         form.horizon.form.gandi_openstack_tenant_id.data = account.credentials["gandi_openstack_tenant_id"]
         form.max_instances.data = account.max_instances
+    elif account.provider == CloudProvider.GITLAB and isinstance(form, GitlabAccountForm):
+        form.gitlab_token.data = account.credentials["gitlab_token"]
     elif account.provider == CloudProvider.OVH and isinstance(form, OvhAccountForm):
         form.horizon.form.ovh_openstack_user.data = account.credentials["ovh_openstack_user"]
         form.horizon.form.ovh_openstack_password.data = account.credentials["ovh_openstack_password"]

app/portal/static.py

@@ -0,0 +1,207 @@
+import logging
+from typing import Optional, List, Any
+
+from flask import flash, redirect, url_for, render_template, Response, Blueprint, current_app
+from flask.typing import ResponseReturnValue
+from flask_wtf import FlaskForm
+from sqlalchemy import exc
+from wtforms import StringField, SelectField, SubmitField, BooleanField, FileField
+from wtforms.validators import DataRequired
+
+from app.brm.static import create_static_origin
+from app.models.base import Group
+from app.models.cloud import CloudAccount, CloudProvider
+from app.models.mirrors import StaticOrigin
+from app.portal.util import response_404, view_lifecycle
+
+bp = Blueprint("static", __name__)
+
+
+class StaticOriginForm(FlaskForm):  # type: ignore
+    description = StringField(
+        'Description',
+        validators=[DataRequired()],
+        description='Enter a brief description of the static website that you are creating in this field. This is '
+                    'also a required field.'
+    )
+    group = SelectField(
+        'Group',
+        validators=[DataRequired()],
+        description='Select the group that you want the origin to belong to from the drop-down menu in this field. '
+                    'This is a required field.'
+    )
+    storage_cloud_account = SelectField(
+        'Storage Cloud Account',
+        validators=[DataRequired()],
+        description='Select the cloud account that you want the origin to be deployed to from the drop-down menu in '
+                    'this field. This is a required field.'
+    )
+    source_cloud_account = SelectField(
+        'Source Cloud Account',
+        validators=[DataRequired()],
+        description='Select the cloud account that will be used to modify the source repository for the web content '
+                    'for this static origin. This is a required field.'
+    )
+    source_project = StringField(
+        'Source Project',
+        validators=[DataRequired()],
+        description='GitLab project path.'
+    )
+    auto_rotate = BooleanField(
+        'Auto-Rotate',
+        default=True,
+        description='Select this field if you want to enable auto-rotation for the mirror. This means that the mirror '
+                    'will automatically redeploy with a new domain name if it is detected to be blocked. This field '
+                    'is optional and is enabled by default.'
+    )
+    matrix_homeserver = SelectField(
+        'Matrix Homeserver',
+        description='Select the Matrix homeserver from the drop-down box to enable Keanu Convene on mirrors of this '
+                    'static origin.'
+    )
+    keanu_convene_path = StringField(
+        'Keanu Convene Path',
+        default='talk',
+        description='Enter the subdirectory to present the Keanu Convene application at on the mirror. This defaults '
+                    'to "talk".'
+    )
+    keanu_convene_logo = FileField(
+        'Keanu Convene Logo',
+        description='Logo to use for Keanu Convene'
+    )
+    keanu_convene_color = StringField(
+        'Keanu Convene Accent Color',
+        default='#0047ab',
+        description='Accent color to use for Keanu Convene (HTML hex code)'
+    )
+    enable_clean_insights = BooleanField(
+        'Enable Clean Insights',
+        description='When enabled, a Clean Insights Measurement Proxy endpoint is deployed on the mirror to allow for '
+                    'submission of results from any of the supported Clean Insights SDKs.'
+    )
+    submit = SubmitField('Save Changes')
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.group.choices = [(x.id, x.group_name) for x in Group.query.all()]
+        self.storage_cloud_account.choices = [(x.id, f"{x.provider.description} - {x.description}") for x in
+                                              CloudAccount.query.filter(
+                                                  CloudAccount.provider == CloudProvider.AWS).all()]
+        self.source_cloud_account.choices = [(x.id, f"{x.provider.description} - {x.description}") for x in
+                                             CloudAccount.query.filter(
+                                                 CloudAccount.provider == CloudProvider.GITLAB).all()]
+        self.matrix_homeserver.choices = [(x, x) for x in current_app.config['MATRIX_HOMESERVERS']]
+
+
+@bp.route("/new", methods=['GET', 'POST'])
+@bp.route("/new/<group_id>", methods=['GET', 'POST'])
+def static_new(group_id: Optional[int] = None) -> ResponseReturnValue:
+    form = StaticOriginForm()
+    if len(form.source_cloud_account.choices) == 0 or len(form.storage_cloud_account.choices) == 0:
+        flash("You must add at least one AWS account and at least one GitLab account before creating static origins.",
+              "warning")
+        return redirect(url_for("portal.cloud.cloud_account_list"))
+    if form.validate_on_submit():
+        try:
+            static = create_static_origin(
+                form.description.data,
+                int(form.group.data),
+                int(form.storage_cloud_account.data),
+                int(form.source_cloud_account.data),
+                form.source_project.data,
+                form.auto_rotate.data,
+                form.matrix_homeserver.data,
+                form.keanu_convene_path.data,
+                form.keanu_convene_logo.data,
+                form.keanu_convene_color.data,
+                form.enable_clean_insights.data,
+                True
+            )
+            flash(f"Created new static origin #{static.id}.", "success")
+            return redirect(url_for("portal.static.static_edit", static_id=static.id))
+        except ValueError as e:  # may be returned by create_static_origin and from the int conversion
+            logging.warning(e)
+            flash("Failed to create new static origin due to an invalid input.", "danger")
+            return redirect(url_for("portal.static.static_list"))
+        except exc.SQLAlchemyError as e:
+            flash("Failed to create new static origin due to a database error.", "danger")
+            logging.warning(e)
+            return redirect(url_for("portal.static.static_list"))
+    if group_id:
+        form.group.data = group_id
+    return render_template("new.html.j2", section="static", form=form)
+
+
+@bp.route('/edit/<static_id>', methods=['GET', 'POST'])
+def static_edit(static_id: int) -> ResponseReturnValue:
+    static: Optional[StaticOrigin] = StaticOrigin.query.filter(StaticOrigin.id == static_id).first()
+    if static is None:
+        return Response(render_template("error.html.j2",
+                                        section="static",
+                                        header="404 Origin Not Found",
+                                        message="The requested static origin could not be found."),
+                        status=404)
+    form = StaticOriginForm(description=static.description,
+                            group=static.group_id,
+                            storage_cloud_account=static.storage_cloud_account_id,
+                            source_cloud_account=static.source_cloud_account_id,
+                            source_project=static.source_project,
+                            matrix_homeserver=static.matrix_homeserver,
+                            keanu_convene_path=static.keanu_convene_path,
+                            auto_rotate=static.auto_rotate,
+                            enable_clean_insights=bool(static.clean_insights_backend))
+    form.group.render_kw = {"disabled": ""}
+    form.storage_cloud_account.render_kw = {"disabled": ""}
+    form.source_cloud_account.render_kw = {"disabled": ""}
+    if form.validate_on_submit():
+        try:
+            static.update(
+                form.source_project.data,
+                form.description.data,
+                form.auto_rotate.data,
+                form.matrix_homeserver.data,
+                form.keanu_convene_path.data,
+                form.keanu_convene_logo.data,
+                form.keanu_convene_color.data,
+                form.enable_clean_insights.data,
+                True
+            )
+            flash("Saved changes to group.", "success")
+        except ValueError as e:  # may be returned by create_static_origin and from the int conversion
+            logging.warning(e)
+            flash("An error occurred saving the changes to the static origin due to an invalid input.", "danger")
+        except exc.SQLAlchemyError as e:
+            logging.warning(e)
+            flash("An error occurred saving the changes to the static origin due to a database error.", "danger")
+    return render_template("static.html.j2",
+                           section="static",
+                           static=static, form=form)
+
+
+@bp.route("/list")
+def static_list() -> ResponseReturnValue:
+    statics: List[StaticOrigin] = StaticOrigin.query.order_by(StaticOrigin.description).all()
+    return render_template("list.html.j2",
+                           section="static",
+                           title="Static Origins",
+                           item="static",
+                           new_link=url_for("portal.static.static_new"),
+                           items=statics
+                           )
+
+
+@bp.route("/destroy/<static_id>", methods=['GET', 'POST'])
+def static_destroy(static_id: int) -> ResponseReturnValue:
+    static = StaticOrigin.query.filter(StaticOrigin.id == static_id, StaticOrigin.destroyed.is_(None)).first()
+    if static is None:
+        return response_404("The requested static origin could not be found.")
+    return view_lifecycle(
+        header=f"Destroy static origin {static.description}",
+        message=static.description,
+        success_message="All proxies from the destroyed static origin will shortly be destroyed at their providers, "
+                        "and the static content will be removed from the cloud provider.",
+        success_view="portal.static.static_list",
+        section="static",
+        resource=static,
+        action="destroy"
+    )

app/portal/templates/base.html.j2

@@ -82,7 +82,7 @@
                 </h6>
                 <ul class="nav flex-column">
                     <li class="nav-item">
-                        <a class="nav-link{% if section == "cloud" %} active{% else %} disabled text-secondary{% endif %}"
+                        <a class="nav-link{% if section == "cloud" %} active{% endif %}"
                            href="{{ url_for("portal.cloud.cloud_account_list") }}">
                             {{ icon("cloud") }} Cloud Accounts
                         </a>
@@ -106,8 +106,8 @@
                         </a>
                     </li>
                     <li class="nav-item">
-                        <a class="nav-link{% if section == "origin" %} active{% endif %} disabled text-secondary"
+                        <a class="nav-link{% if section == "static" %} active{% endif %}"
-                           href="{{ url_for("portal.origin.origin_list") }}">
+                           href="{{ url_for("portal.static.static_list") }}">
                             {{ icon("hdd") }} Static Origins
                         </a>
                     </li>

app/portal/templates/list.html.j2

@@ -1,7 +1,7 @@
 {% extends "base.html.j2" %}
 {% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, cloud_accounts_table,
         eotk_table, groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table,
-        pools_table, proxies_table, webhook_table %}
+        pools_table, proxies_table, static_table, webhook_table %}
 
 {% block content %}
     <h1 class="h2 mt-3">{{ title }}</h1>
@@ -35,6 +35,8 @@
         {% endif %}
     {% elif item == "origin" %}
         {{ origins_table(items) }}
+    {% elif item == "static" %}
+        {{ static_table(items) }}
     {% elif item == "pool" %}
         {{ pools_table(items) }}
     {% elif item == "proxy" %}

app/portal/templates/static.html.j2

@@ -0,0 +1,15 @@
+{% extends "base.html.j2" %}
+{% from 'bootstrap5/form.html' import render_form %}
+{% from "tables.html.j2" import static_table %}
+
+{% block content %}
+    <h1 class="h2 mt-3">Static Origins</h1>
+    <h2 class="h3">
+        {{ static.group.group_name }}: {{ static.description }} (#{{ static.id }})
+    </h2>
+
+    <div style="border: 1px solid #666;" class="p-3">
+        {{ render_form(form) }}
+    </div>
+
+{% endblock %}

app/portal/templates/tables.html.j2

@@ -633,6 +633,57 @@
     </div>
 {% endmacro %}
 
+{% macro static_table(statics) %}
+    <div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Description</th>
+                <th scope="col">Origin URL</th>
+                <th scope="col">Keanu Convene</th>
+                <th scope="col">Auto-Rotation</th>
+                <th scope="col">Group</th>
+                <th scope="col">Actions</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for static in statics %}
+                {% if not static.destroyed %}
+                    <tr>
+                        <td>{{ static.description }}</td>
+                        <td>{% if static.origin_domain_name %}
+                            <a href="https://{{ static.origin_domain_name }}" class="btn btn-secondary btn-sm"
+                               target="_bypass"
+                               rel="noopener noreferer">⎋</a>
+                               {{ static.origin_domain_name }}
+                            {% else %}
+                                <em>Still deploying</em>
+                            {% endif %}</td>
+                        <td>{% if static.origin_domain_name and static.keanu_convene_path %}
+                            <a href="https://{{ static.origin_domain_name }}/{{ static.keanu_convene_path }}/" class="btn btn-secondary btn-sm"
+                               target="_bypass"
+                               rel="noopener noreferer">⎋</a>
+                            {% else %}
+                                <em>Still deploying</em>
+                            {% endif %}</td>
+                        <td>{% if static.auto_rotate %}✅{% else %}❌{% endif %}</td>
+                        <td>
+                            <a href="{{ url_for("portal.group.group_edit", group_id=static.group.id) }}">{{ static.group.group_name }}</a>
+                        </td>
+                        <td>
+                            <a href="{{ url_for("portal.static.static_edit", static_id=static.id) }}"
+                               class="btn btn-primary btn-sm">View/Edit</a>
+                            <a href="{{ url_for("portal.static.static_destroy", static_id=static.id) }}"
+                               class="btn btn-danger btn-sm">Destroy</a>
+                        </td>
+                    </tr>
+                {% endif %}
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endmacro %}
+
 {% macro webhook_table(webhooks) %}
     <div class="table-responsive">
         <table class="table table-striped table-sm">

app/terraform/static/aws.py

@@ -0,0 +1,141 @@
+import logging
+import os
+from datetime import datetime
+from typing import List, Any
+
+from flask import current_app
+
+from app.extensions import db
+from app.models.base import Group
+from app.models.cloud import CloudProvider, CloudAccount
+from app.models.mirrors import StaticOrigin
+from app.terraform.terraform import TerraformAutomation
+
+
+def import_state(state: Any) -> None:
+    if not isinstance(state, dict):
+        raise RuntimeError("The Terraform state object returned was not a dict.")
+    if "child_modules" not in state['values']['root_module']:
+        # There are no CloudFront origins deployed to import state for
+        return
+    # CloudFront distributions (origins)
+    for mod in state['values']['root_module']['child_modules']:
+        if mod['address'].startswith('module.static_'):
+            static_id = mod['address'][len('module.static_'):]
+            logging.debug("Found static module in state: %s", static_id)
+            for res in mod['resources']:
+                if res['address'].endswith('aws_cloudfront_distribution.this'):
+                    logging.debug("and found related cloudfront distribution")
+                    static = StaticOrigin.query.filter(StaticOrigin.id == static_id).first()
+                    static.origin_domain_name = res['values']['domain_name']
+                    logging.debug("and found static origin: %s to update with domain name: %s", static.id,
+                                  static.origin_domain_name)
+                    static.terraform_updated = datetime.utcnow()
+                    break
+    db.session.commit()
+
+
+class StaticAWSAutomation(TerraformAutomation):
+    short_name = "static_aws"
+    description = "Deploy static origins to AWS"
+    provider = CloudProvider.AWS
+    cloud_name = "aws"
+
+    template_parameters: List[str] = []
+
+    template = """
+    terraform {
+      {{ backend_config }}
+      required_providers {
+        aws = {
+          version = "~> 4.67.0"
+        }
+        gitlab = {
+          source = "gitlabhq/gitlab"
+          version = "~> 15.11.0"
+        }
+      }
+    }
+
+    {% for group in groups %}
+    module "label_{{ group.id }}" {
+      source  = "cloudposse/label/null"
+      version = "0.25.0"
+      namespace = "{{ global_namespace }}"
+      tenant = "{{ group.group_name }}"
+      label_order = ["namespace", "tenant", "name", "attributes"]
+    }
+    {% endfor %}
+
+    {% for account in source_cloud_accounts -%}
+    provider "gitlab" {
+      token = "{{ account.credentials['gitlab_token'] }}"
+      alias = "account_{{ account.id }}"
+    }
+    {% endfor -%}
+
+    {% for account in storage_cloud_accounts -%}
+    provider "aws" {
+      access_key = "{{ account.credentials['aws_access_key'] }}"
+      secret_key = "{{ account.credentials['aws_secret_key'] }}"
+      region     = "{{ account.credentials['aws_region'] }}"
+      alias      = "account_{{ account.id }}"
+    }
+
+    provider "aws" {
+      access_key = "{{ account.credentials['aws_access_key'] }}"
+      secret_key = "{{ account.credentials['aws_secret_key'] }}"
+      region     = "us-east-1"
+      alias      = "account_{{ account.id }}_us_east_1"
+    }
+
+    {% for static in account.statics | selectattr("destroyed", "none") %}
+    module "static_{{ static.id }}" {
+      providers = {
+        aws           = aws.account_{{ account.id }}
+        aws.us_east_1 = aws.account_{{ account.id }}_us_east_1
+        gitlab        = gitlab.account_{{ static.source_cloud_account_id }}
+      }
+      source = "{{ terraform_modules_path }}/terraform-aws-bc-static-origin"
+      name = "static"
+      context = module.label_{{ static.group.id }}.context
+      {% if static.keanu_convene_path -%}
+      keanu_convene_path = "{{ static.keanu_convene_path }}"
+      {%- endif %}
+      {% if static.keanu_convene_config -%}
+      keanu_convene_config = "{{ static.keanu_convene_config  | replace('"', '\\\\"')  }}"
+      {%- endif %}
+      {% if static.matrix_homeserver -%}
+      matrix_homeserver = "{{ static.matrix_homeserver }}"
+      {%- endif %}
+      gitlab_project = "{{ static.source_project }}"
+      attributes = ["{{ static.id }}"]
+    }
+    {% endfor -%}
+    {%- endfor %}
+    """
+
+    def tf_generate(self) -> None:
+        groups = Group.query.all()
+        storage_cloud_accounts = CloudAccount.query.filter(
+            CloudAccount.provider == CloudProvider.AWS
+        ).all()
+        source_cloud_accounts = CloudAccount.query.filter(
+            CloudAccount.provider == CloudProvider.GITLAB
+        ).all()
+        self.tf_write(
+            self.template,
+            groups=groups,
+            storage_cloud_accounts=storage_cloud_accounts,
+            source_cloud_accounts=source_cloud_accounts,
+            global_namespace=current_app.config['GLOBAL_NAMESPACE'], bypass_token=current_app.config['BYPASS_TOKEN'],
+            terraform_modules_path=os.path.join(*list(os.path.split(current_app.root_path))[:-1], 'terraform-modules'),
+            backend_config=f"""backend "http" {{
+              lock_address = "{current_app.config['TFSTATE_BACKEND']}/{self.short_name}"
+              unlock_address = "{current_app.config['TFSTATE_BACKEND']}/{self.short_name}"
+              address = "{current_app.config['TFSTATE_BACKEND']}/{self.short_name}"
+            }}""",
+            **{k: current_app.config[k.upper()] for k in self.template_parameters})
+
+    def tf_posthook(self, *, prehook_result: Any = None) -> None:
+        import_state(self.tf_show())

migrations/versions/2d747ffb9928_add_static_origins.py

@@ -0,0 +1,48 @@
+"""add static origins
+
+Revision ID: 2d747ffb9928
+Revises: 431704bac57a
+Create Date: 2023-05-25 14:23:30.568572
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2d747ffb9928'
+down_revision = '431704bac57a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('static_origin',
+    sa.Column('group_id', sa.Integer(), nullable=False),
+    sa.Column('storage_cloud_account_id', sa.Integer(), nullable=False),
+    sa.Column('source_cloud_account_id', sa.Integer(), nullable=False),
+    sa.Column('source_project', sa.String(length=255), nullable=False),
+    sa.Column('auto_rotate', sa.Boolean(), nullable=False),
+    sa.Column('matrix_homeserver', sa.String(length=255), nullable=True),
+    sa.Column('keanu_convene_path', sa.String(length=255), nullable=True),
+    sa.Column('keanu_convene_config', sa.String(), nullable=True),
+    sa.Column('clean_insights_backend', sa.String(length=255), nullable=True),
+    sa.Column('origin_domain_name', sa.String(length=255), nullable=True),
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('description', sa.String(length=255), nullable=False),
+    sa.Column('added', sa.DateTime(), nullable=False),
+    sa.Column('updated', sa.DateTime(), nullable=False),
+    sa.Column('destroyed', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_static_origin_group_id_group')),
+    sa.ForeignKeyConstraint(['source_cloud_account_id'], ['cloud_account.id'], name=op.f('fk_static_origin_source_cloud_account_id_cloud_account')),
+    sa.ForeignKeyConstraint(['storage_cloud_account_id'], ['cloud_account.id'], name=op.f('fk_static_origin_storage_cloud_account_id_cloud_account')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_static_origin'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('static_origin')
+    # ### end Alembic commands ###

requirements.txt

@@ -1,3 +1,4 @@
+Pillow
 PyGithub
 alembic
 azure-identity
@@ -10,14 +11,19 @@ flask
 flask-migrate
 flask-wtf
 jinja2
+lxml
 nose
 openpyxl
 prometheus_client
 pydantic
+pytest
 python-dateutil
 python-gitlab
 pyyaml
 requests
 sqlalchemy
+tftest
 tldextract
+webcolors
+werkzeug
 wtforms

terraform-modules/terraform-aws-bc-static-origin

@@ -0,0 +1 @@
+Subproject commit bbe16800b18815fbae1ff9e8965cf026f461e1c1

tests/utils/test_color.py

@@ -0,0 +1,28 @@
+import pytest
+
+from app.brm.utils import normalize_color
+
+
+def test_normalize_color():
+    # Test valid CSS color names
+    assert normalize_color('red') == '#ff0000'
+    assert normalize_color('white') == '#ffffff'
+    assert normalize_color('black') == '#000000'
+    # Test valid 6-digit hex color codes
+    assert normalize_color('#ffffff') == '#ffffff'
+    assert normalize_color('#000000') == '#000000'
+    assert normalize_color('#00ff00') == '#00ff00'
+    # Test valid 3-digit hex color codes
+    assert normalize_color('#fff') == '#fff'
+    assert normalize_color('#000') == '#000'
+    assert normalize_color('#0f0') == '#0f0'
+    # Test case insensitivity
+    assert normalize_color('#FFFFFF') == '#ffffff'
+    assert normalize_color('#ABCDEF') == '#abcdef'
+    # Test invalid color values
+    with pytest.raises(ValueError):
+        normalize_color('invalid')
+    with pytest.raises(ValueError):
+        normalize_color('#1234567')
+    with pytest.raises(ValueError):
+        normalize_color('#xyz')

tests/utils/test_images.py

@@ -0,0 +1,59 @@
+import pytest
+from PIL import Image, ImageDraw
+from app.brm.utils import thumbnail_uploaded_image, create_data_uri
+from werkzeug.datastructures import FileStorage
+from io import BytesIO
+import base64
+
+
+def create_test_image(image_format: str, extension: str):
+    """
+    Creates a test image with the data stored in a BytesIO buffer.
+
+    :return: A test image.
+    """
+    img_byte = BytesIO()
+    img = Image.new('RGB', (500, 500), color=(73, 109, 137))
+    d = ImageDraw.Draw(img)
+    d.text((10, 10), "Hello World", fill=(255, 255, 0))
+    img.save(img_byte, format="jpeg")
+    img_byte.seek(0)
+    file = FileStorage(stream=img_byte, filename='test.jpg')
+    return file
+
+
+@pytest.mark.parametrize("image_format, extension", [
+    ("jpeg", "jpg"),
+    ("png", "png"),
+    ("webp", "webp"),
+])
+def test_thumbnail_uploaded_image(image_format: str, extension: str):
+    file = create_test_image(image_format, extension)
+    output = thumbnail_uploaded_image(file)
+    image = Image.open(BytesIO(output))
+
+    # Check if the processed image size is as expected
+    assert image.size[0] <= 256
+    assert image.size[1] <= 256
+
+
+@pytest.mark.parametrize("image_format, extension", [
+    ("jpeg", "jpg"),
+    ("png", "png"),
+    ("webp", "webp"),
+])
+def test_create_data_uri(image_format: str, extension: str):
+    file = create_test_image(image_format, extension)
+    img_bytes = thumbnail_uploaded_image(file)
+    file_extension = 'jpg'
+    data_uri = create_data_uri(img_bytes, file_extension)
+
+    # Check if the data URI starts as expected
+    assert data_uri.startswith('data:image/jpeg;base64,')
+
+    # Check if the base64 image can be decoded
+    try:
+        base64_data = data_uri.split(',')[1]
+        base64.b64decode(base64_data)
+    except Exception as e:
+        pytest.fail(f"Decoding base64 data URI failed with error {e}")