From 15a85b1efecc8b9818ea5a482c07da729b2d4c76 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Thu, 25 May 2023 15:32:31 +0100 Subject: [PATCH] feat(static): adds new static origins feature --- .gitmodules | 3 + app/brm/static.py | 77 +++++++ app/brm/utils.py | 73 +++++- app/cli/automate.py | 2 + app/models/base.py | 1 + app/models/cloud.py | 3 + app/models/mirrors.py | 106 ++++++++- app/portal/__init__.py | 2 + app/portal/cloud.py | 17 ++ app/portal/static.py | 207 ++++++++++++++++++ app/portal/templates/base.html.j2 | 6 +- app/portal/templates/list.html.j2 | 4 +- app/portal/templates/static.html.j2 | 15 ++ app/portal/templates/tables.html.j2 | 51 +++++ app/terraform/static/aws.py | 141 ++++++++++++ .../2d747ffb9928_add_static_origins.py | 48 ++++ requirements.txt | 6 + .../terraform-aws-bc-static-origin | 1 + tests/utils/test_color.py | 28 +++ tests/utils/test_images.py | 59 +++++ 20 files changed, 843 insertions(+), 7 deletions(-) create mode 100644 app/brm/static.py create mode 100644 app/portal/static.py create mode 100644 app/portal/templates/static.html.j2 create mode 100644 app/terraform/static/aws.py create mode 100644 migrations/versions/2d747ffb9928_add_static_origins.py create mode 160000 terraform-modules/terraform-aws-bc-static-origin create mode 100644 tests/utils/test_color.py create mode 100644 tests/utils/test_images.py diff --git a/.gitmodules b/.gitmodules index a41077f..1bee740 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,3 +25,6 @@ [submodule "terraform-modules/terraform-aws-bc-eotk"] path = terraform-modules/terraform-aws-bc-eotk url = https://github.com/sr2c/terraform-aws-bc-eotk.git +[submodule "terraform-modules/terraform-aws-bc-static-origin"] + path = terraform-modules/terraform-aws-bc-static-origin + url = https://gitlab.com/sr2c/terraform-aws-bc-static-origin.git diff --git a/app/brm/static.py b/app/brm/static.py new file mode 100644 index 0000000..c622ec6 --- /dev/null +++ b/app/brm/static.py @@ -0,0 +1,77 @@ +from typing import Optional, Union + +from werkzeug.datastructures import FileStorage + +from app.extensions import db +from app.models.base import Group +from app.models.cloud import CloudAccount +from app.models.mirrors import StaticOrigin + + +def create_static_origin( + description: str, + group_id: int, + storage_cloud_account_id: int, + source_cloud_account_id: int, + source_project: str, + auto_rotate: bool, + matrix_homeserver: Optional[str], + keanu_convene_path: Optional[str], + keanu_convene_logo: Optional[FileStorage], + keanu_convene_color: Optional[str], + clean_insights_backend: Optional[Union[str, bool]], + db_session_commit: bool = False, +) -> StaticOrigin: + """ + Create a new static origin. + + :param description: The description of the new static origin. + :param group_id: The ID for the origin group to add this new static origin to. + :param storage_cloud_account_id: The ID for the cloud account to deploy this new static origin to. + :param source_cloud_account_id: The ID for the cloud account used to interact with the web sources. + :param source_project: The path for the source project, e.g. the GitLab project path. + :param auto_rotate: Whether to automatically rotate this domain when it is detected to be blocked. + :param matrix_homeserver: The domain name for the Matrix homeserver to proxy to. + :param keanu_convene_path: The path to serve the Keanu Convene application from. + :param clean_insights_backend: The domain name for the Clean Insights backend to proxy to. + :param db_session_commit: Whether to add the new StaticOrigin to the database session and commit it. + :returns: StaticOrigin -- the newly created StaticOrigin + :raises: ValueError, sqlalchemy.exc.SQLAlchemyError + """ + static_origin = StaticOrigin() + if isinstance(group_id, int): + group = Group.query.filter(Group.id == group_id).first() + if group is None: + raise ValueError("group_id must match an existing group") + static_origin.group_id = group_id + else: + raise ValueError("group_id must be an int") + if isinstance(storage_cloud_account_id, int): + cloud_account = CloudAccount.query.filter(CloudAccount.id == storage_cloud_account_id).first() + if cloud_account is None: + raise ValueError("storage_cloud_account_id must match an existing provider") + static_origin.storage_cloud_account_id = storage_cloud_account_id + else: + raise ValueError("storage_cloud_account_id must be an int") + if isinstance(source_cloud_account_id, int): + cloud_account = CloudAccount.query.filter(CloudAccount.id == source_cloud_account_id).first() + if cloud_account is None: + raise ValueError("source_cloud_account_id must match an existing provider") + static_origin.source_cloud_account_id = source_cloud_account_id + else: + raise ValueError("source_cloud_account_id must be an int") + static_origin.update( + source_project, + description, + auto_rotate, + matrix_homeserver, + keanu_convene_path, + keanu_convene_logo, + keanu_convene_color, + clean_insights_backend, + False + ) + if db_session_commit: + db.session.add(static_origin) + db.session.commit() + return static_origin diff --git a/app/brm/utils.py b/app/brm/utils.py index 03c7981..c87e3a1 100644 --- a/app/brm/utils.py +++ b/app/brm/utils.py @@ -1,6 +1,12 @@ from __future__ import annotations -from typing import Any +import base64 +from io import BytesIO +from typing import Any, Tuple + +import webcolors +from PIL import Image +from werkzeug.datastructures import FileStorage def is_integer(contender: Any) -> bool: @@ -18,3 +24,68 @@ def is_integer(contender: Any) -> bool: return False else: return float(contender).is_integer() + + +def thumbnail_uploaded_image(file: FileStorage, max_size: Tuple[int, int] = (256, 256)) -> bytes: + """ + Process an uploaded image file into a resized image of a specific size. + + :param file: An uploaded image file. + :param max_size: A tuple containing the maximum width and height for the thumbnail image. Default is (256, 256). + :return: The byte data of the thumbnail image. + """ + if file.filename is None: + raise ValueError("No file was uploaded") + img = Image.open(file) + img.thumbnail(max_size) + byte_arr = BytesIO() + img.save(byte_arr, format='PNG' if file.filename.lower().endswith('.png') else 'JPEG') + return byte_arr.getvalue() + + +def create_data_uri(bytes_data: bytes, file_extension: str) -> str: + """ + Create a data URI from binary data and a file extension. + + :param bytes_data: The binary data of an image. + :param file_extension: The file extension of the image. + :return: A data URI representing the image. + """ + # base64 encode + encoded = base64.b64encode(bytes_data).decode('ascii') + # create data URI + data_uri = "data:image/{};base64,{}".format('jpeg' if file_extension == 'jpg' else file_extension, encoded) + return data_uri + + +def normalize_color(color: str) -> str: + """ + Normalize a string representing a color to its hexadecimal representation. + + This function accepts a string representing a color in one of the following formats: + + - A CSS color name, such as 'red', 'green', or 'blue'. + - A 6-digit hexadecimal color code, such as '#FF0000' for red. + - A 3-digit hexadecimal color code, such as '#F00' for red. + + The function returns a string with the color in 6-digit or 3-digit hexadecimal format, + with lowercase letters. If the color is given as a CSS color name, the function + returns its equivalent 6-digit hexadecimal format. + + :param color: A string representing a color. + :return: The color in 6-digit or 3-digit hexadecimal format. + :raises: ValueError: If the input string does not represent a valid color in any of the accepted formats. + """ + try: + return webcolors.name_to_hex(color) # type: ignore[no-any-return] + except ValueError: + pass + if color.startswith('#'): + color = color[1:].lower() + if len(color) in [3, 6]: + try: + _ = int(color, 16) + return f"#{color}" + except ValueError: + pass + raise ValueError(f"color must be a valid HTML color, got: {repr(color)}") diff --git a/app/cli/automate.py b/app/cli/automate.py index ef4a04c..96884f6 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -39,6 +39,7 @@ from app.terraform.proxy.meta import ProxyMetaAutomation from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation from app.terraform.proxy.fastly import ProxyFastlyAutomation +from app.terraform.static.aws import StaticAWSAutomation jobs = { x.short_name: x # type: ignore[attr-defined] @@ -62,6 +63,7 @@ jobs = { BridgeGandiAutomation, BridgeHcloudAutomation, BridgeOvhAutomation, + StaticAWSAutomation, EotkAWSAutomation, ProxyAzureCdnAutomation, ProxyCloudfrontAutomation, diff --git a/app/models/base.py b/app/models/base.py index e33b68f..d230ac9 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -11,6 +11,7 @@ class Group(AbstractConfiguration): eotk = db.Column(db.Boolean()) origins = db.relationship("Origin", back_populates="group") + statics = db.relationship("StaticOrigin", back_populates="group") eotks = db.relationship("Eotk", back_populates="group") onions = db.relationship("Onion", back_populates="group") smart_proxies = db.relationship("SmartProxy", back_populates="group") diff --git a/app/models/cloud.py b/app/models/cloud.py index dc04e6c..fbbc802 100644 --- a/app/models/cloud.py +++ b/app/models/cloud.py @@ -3,6 +3,7 @@ import enum from app.brm.brn import BRN from app.extensions import db from app.models import AbstractConfiguration +from app.models.mirrors import StaticOrigin class CloudProvider(enum.Enum): @@ -36,6 +37,8 @@ class CloudAccount(AbstractConfiguration): max_instances = db.Column(db.Integer()) bridges = db.relationship("Bridge", back_populates="cloud_account") + statics = db.relationship("StaticOrigin", back_populates="storage_cloud_account", foreign_keys=[ + StaticOrigin.storage_cloud_account_id]) @property def brn(self) -> BRN: diff --git a/app/models/mirrors.py b/app/models/mirrors.py index e0b7cfa..438cf09 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -1,8 +1,12 @@ -from typing import Optional, List +import json +from datetime import datetime +from typing import Optional, List, Union, Any, Dict from tldextract import extract +from werkzeug.datastructures import FileStorage from app.brm.brn import BRN +from app.brm.utils import thumbnail_uploaded_image, create_data_uri, normalize_color from app.extensions import db from app.models import AbstractConfiguration, AbstractResource from app.models.onions import Onion @@ -25,7 +29,7 @@ class Origin(AbstractConfiguration): product="mirror", provider="conf", resource_type="origin", - resource_id="self.domain_name" + resource_id=self.domain_name ) @classmethod @@ -48,6 +52,104 @@ class Origin(AbstractConfiguration): return f"https://{domain_name.replace(tld, onion.onion_name)}.onion" +class StaticOrigin(AbstractConfiguration): + group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False) + storage_cloud_account_id = db.Column(db.Integer(), db.ForeignKey("cloud_account.id"), nullable=False) + source_cloud_account_id = db.Column(db.Integer(), db.ForeignKey("cloud_account.id"), nullable=False) + source_project = db.Column(db.String(255), nullable=False) + auto_rotate = db.Column(db.Boolean, nullable=False) + matrix_homeserver = db.Column(db.String(255), nullable=True) + keanu_convene_path = db.Column(db.String(255), nullable=True) + keanu_convene_config = db.Column(db.String(), nullable=True) + clean_insights_backend = db.Column(db.String(255), nullable=True) + origin_domain_name = db.Column(db.String(255), nullable=True) + + @property + def brn(self) -> BRN: + return BRN( + group_id=self.group_id, + product="mirror", + provider="aws", + resource_type="static", + resource_id=self.domain_name + ) + + group = db.relationship("Group", back_populates="statics") + storage_cloud_account = db.relationship("CloudAccount", back_populates="statics", + foreign_keys=[storage_cloud_account_id]) + source_cloud_account = db.relationship("CloudAccount", back_populates="statics", + foreign_keys=[source_cloud_account_id]) + + def destroy(self) -> None: + super().destroy() + for proxy in self.proxies: + proxy.destroy() + + def update( + self, + source_project: str, + description: str, + auto_rotate: bool, + matrix_homeserver: Optional[str], + keanu_convene_path: Optional[str], + keanu_convene_logo: Optional[FileStorage], + keanu_convene_color: Optional[str], + clean_insights_backend: Optional[Union[str, bool]], + db_session_commit: bool, + ) -> None: + if isinstance(source_project, str): + self.source_project = source_project + else: + raise ValueError("source project must be a str") + if isinstance(description, str): + self.description = description + else: + raise ValueError("description must be a str") + if isinstance(auto_rotate, bool): + self.auto_rotate = auto_rotate + else: + raise ValueError("auto_rotate must be a bool") + if isinstance(matrix_homeserver, str): + self.matrix_homeserver = matrix_homeserver + else: + raise ValueError("matrix_homeserver must be a str") + if isinstance(keanu_convene_path, str): + self.keanu_convene_path = keanu_convene_path + else: + raise ValueError("keanu_convene_path must be a str") + if self.keanu_convene_config is None: + self.keanu_convene_config = "{}" + keanu_convene_config: Dict[str, Any] = json.loads(self.keanu_convene_config) + if keanu_convene_logo is None: + pass + elif isinstance(keanu_convene_logo, FileStorage): + if keanu_convene_logo.filename: # if False, no file was uploaded + keanu_convene_config["logo"] = create_data_uri( + thumbnail_uploaded_image(keanu_convene_logo), keanu_convene_logo.filename) + else: + raise ValueError("keanu_convene_logo must be a FileStorage") + try: + if isinstance(keanu_convene_color, str): + keanu_convene_config["color"] = normalize_color(keanu_convene_color) # can raise ValueError + else: + raise ValueError() # re-raised below with message + except ValueError: + raise ValueError("keanu_convene_path must be a str containing an HTML color (CSS name or hex)") + self.keanu_convene_config = json.dumps(keanu_convene_config, separators=(',', ':')) + del keanu_convene_config # done with this temporary variable + if clean_insights_backend is None or (isinstance(clean_insights_backend, bool) and not clean_insights_backend): + self.clean_insights_backend = None + elif isinstance(clean_insights_backend, bool) and clean_insights_backend: + self.clean_insights_backend = "metrics.cleaninsights.org" + elif isinstance(clean_insights_backend, str): + self.clean_insights_backend = clean_insights_backend + else: + raise ValueError("clean_insights_backend must be a str, bool, or None") + if db_session_commit: + db.session.commit() + self.updated = datetime.utcnow() + + class Proxy(AbstractResource): origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False) pool_id = db.Column(db.Integer, db.ForeignKey("pool.id")) diff --git a/app/portal/__init__.py b/app/portal/__init__.py index d1ab73f..8c8dddc 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -26,6 +26,7 @@ from app.portal.onion import bp as onion from app.portal.pool import bp as pool from app.portal.proxy import bp as proxy from app.portal.smart_proxy import bp as smart_proxy +from app.portal.static import bp as static from app.portal.storage import bp as storage from app.portal.webhook import bp as webhook @@ -42,6 +43,7 @@ portal.register_blueprint(onion, url_prefix="/onion") portal.register_blueprint(pool, url_prefix="/pool") portal.register_blueprint(proxy, url_prefix="/proxy") portal.register_blueprint(smart_proxy, url_prefix="/smart") +portal.register_blueprint(static, url_prefix="/static") portal.register_blueprint(storage, url_prefix="/state") portal.register_blueprint(webhook, url_prefix="/webhook") diff --git a/app/portal/cloud.py b/app/portal/cloud.py index 19de7cb..aa6f6bd 100644 --- a/app/portal/cloud.py +++ b/app/portal/cloud.py @@ -53,6 +53,16 @@ class HcloudAccountForm(FlaskForm): # type: ignore submit = SubmitField('Save Changes') +class GitlabAccountForm(FlaskForm): # type: ignore + provider = StringField('Platform', render_kw={"disabled": ""}) + description = StringField('Description', validators=[InputRequired()]) + gitlab_token = StringField('GitLab Access Token', validators=[InputRequired()]) + enabled = BooleanField('Enable this account', default=True, + description="New resources will not be deployed to disabled accounts, however existing " + "resources will persist until destroyed at the end of their lifecycle.") + submit = SubmitField('Save Changes') + + class OvhHorizonForm(Form): # type: ignore[misc] ovh_openstack_user = StringField("User") ovh_openstack_password = StringField("Password") @@ -102,6 +112,7 @@ provider_forms: Dict[str, Type[CloudAccountForm]] = { CloudProvider.AWS.name: AWSAccountForm, CloudProvider.HCLOUD.name: HcloudAccountForm, CloudProvider.GANDI.name: GandiAccountForm, + CloudProvider.GITLAB.name: GitlabAccountForm, CloudProvider.OVH.name: OvhAccountForm, } @@ -140,6 +151,10 @@ def cloud_account_save(account: Optional[CloudAccount], provider: CloudProvider, account.max_distributions = 0 account.max_sub_distributions = 0 account.max_instances = form.max_instances.data + elif provider == CloudProvider.GITLAB and isinstance(form, GitlabAccountForm): + account.credentials = { + "gitlab_token": form.gitlab_token.data, + } elif provider == CloudProvider.OVH and isinstance(form, OvhAccountForm): account.credentials = { "ovh_openstack_user": form.horizon.data["ovh_openstack_user"], @@ -174,6 +189,8 @@ def cloud_account_populate(form: CloudAccountForm, account: CloudAccount) -> Non form.horizon.form.gandi_openstack_password.data = account.credentials["gandi_openstack_password"] form.horizon.form.gandi_openstack_tenant_id.data = account.credentials["gandi_openstack_tenant_id"] form.max_instances.data = account.max_instances + elif account.provider == CloudProvider.GITLAB and isinstance(form, GitlabAccountForm): + form.gitlab_token.data = account.credentials["gitlab_token"] elif account.provider == CloudProvider.OVH and isinstance(form, OvhAccountForm): form.horizon.form.ovh_openstack_user.data = account.credentials["ovh_openstack_user"] form.horizon.form.ovh_openstack_password.data = account.credentials["ovh_openstack_password"] diff --git a/app/portal/static.py b/app/portal/static.py new file mode 100644 index 0000000..c725ca5 --- /dev/null +++ b/app/portal/static.py @@ -0,0 +1,207 @@ +import logging +from typing import Optional, List, Any + +from flask import flash, redirect, url_for, render_template, Response, Blueprint, current_app +from flask.typing import ResponseReturnValue +from flask_wtf import FlaskForm +from sqlalchemy import exc +from wtforms import StringField, SelectField, SubmitField, BooleanField, FileField +from wtforms.validators import DataRequired + +from app.brm.static import create_static_origin +from app.models.base import Group +from app.models.cloud import CloudAccount, CloudProvider +from app.models.mirrors import StaticOrigin +from app.portal.util import response_404, view_lifecycle + +bp = Blueprint("static", __name__) + + +class StaticOriginForm(FlaskForm): # type: ignore + description = StringField( + 'Description', + validators=[DataRequired()], + description='Enter a brief description of the static website that you are creating in this field. This is ' + 'also a required field.' + ) + group = SelectField( + 'Group', + validators=[DataRequired()], + description='Select the group that you want the origin to belong to from the drop-down menu in this field. ' + 'This is a required field.' + ) + storage_cloud_account = SelectField( + 'Storage Cloud Account', + validators=[DataRequired()], + description='Select the cloud account that you want the origin to be deployed to from the drop-down menu in ' + 'this field. This is a required field.' + ) + source_cloud_account = SelectField( + 'Source Cloud Account', + validators=[DataRequired()], + description='Select the cloud account that will be used to modify the source repository for the web content ' + 'for this static origin. This is a required field.' + ) + source_project = StringField( + 'Source Project', + validators=[DataRequired()], + description='GitLab project path.' + ) + auto_rotate = BooleanField( + 'Auto-Rotate', + default=True, + description='Select this field if you want to enable auto-rotation for the mirror. This means that the mirror ' + 'will automatically redeploy with a new domain name if it is detected to be blocked. This field ' + 'is optional and is enabled by default.' + ) + matrix_homeserver = SelectField( + 'Matrix Homeserver', + description='Select the Matrix homeserver from the drop-down box to enable Keanu Convene on mirrors of this ' + 'static origin.' + ) + keanu_convene_path = StringField( + 'Keanu Convene Path', + default='talk', + description='Enter the subdirectory to present the Keanu Convene application at on the mirror. This defaults ' + 'to "talk".' + ) + keanu_convene_logo = FileField( + 'Keanu Convene Logo', + description='Logo to use for Keanu Convene' + ) + keanu_convene_color = StringField( + 'Keanu Convene Accent Color', + default='#0047ab', + description='Accent color to use for Keanu Convene (HTML hex code)' + ) + enable_clean_insights = BooleanField( + 'Enable Clean Insights', + description='When enabled, a Clean Insights Measurement Proxy endpoint is deployed on the mirror to allow for ' + 'submission of results from any of the supported Clean Insights SDKs.' + ) + submit = SubmitField('Save Changes') + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.group.choices = [(x.id, x.group_name) for x in Group.query.all()] + self.storage_cloud_account.choices = [(x.id, f"{x.provider.description} - {x.description}") for x in + CloudAccount.query.filter( + CloudAccount.provider == CloudProvider.AWS).all()] + self.source_cloud_account.choices = [(x.id, f"{x.provider.description} - {x.description}") for x in + CloudAccount.query.filter( + CloudAccount.provider == CloudProvider.GITLAB).all()] + self.matrix_homeserver.choices = [(x, x) for x in current_app.config['MATRIX_HOMESERVERS']] + + +@bp.route("/new", methods=['GET', 'POST']) +@bp.route("/new/", methods=['GET', 'POST']) +def static_new(group_id: Optional[int] = None) -> ResponseReturnValue: + form = StaticOriginForm() + if len(form.source_cloud_account.choices) == 0 or len(form.storage_cloud_account.choices) == 0: + flash("You must add at least one AWS account and at least one GitLab account before creating static origins.", + "warning") + return redirect(url_for("portal.cloud.cloud_account_list")) + if form.validate_on_submit(): + try: + static = create_static_origin( + form.description.data, + int(form.group.data), + int(form.storage_cloud_account.data), + int(form.source_cloud_account.data), + form.source_project.data, + form.auto_rotate.data, + form.matrix_homeserver.data, + form.keanu_convene_path.data, + form.keanu_convene_logo.data, + form.keanu_convene_color.data, + form.enable_clean_insights.data, + True + ) + flash(f"Created new static origin #{static.id}.", "success") + return redirect(url_for("portal.static.static_edit", static_id=static.id)) + except ValueError as e: # may be returned by create_static_origin and from the int conversion + logging.warning(e) + flash("Failed to create new static origin due to an invalid input.", "danger") + return redirect(url_for("portal.static.static_list")) + except exc.SQLAlchemyError as e: + flash("Failed to create new static origin due to a database error.", "danger") + logging.warning(e) + return redirect(url_for("portal.static.static_list")) + if group_id: + form.group.data = group_id + return render_template("new.html.j2", section="static", form=form) + + +@bp.route('/edit/', methods=['GET', 'POST']) +def static_edit(static_id: int) -> ResponseReturnValue: + static: Optional[StaticOrigin] = StaticOrigin.query.filter(StaticOrigin.id == static_id).first() + if static is None: + return Response(render_template("error.html.j2", + section="static", + header="404 Origin Not Found", + message="The requested static origin could not be found."), + status=404) + form = StaticOriginForm(description=static.description, + group=static.group_id, + storage_cloud_account=static.storage_cloud_account_id, + source_cloud_account=static.source_cloud_account_id, + source_project=static.source_project, + matrix_homeserver=static.matrix_homeserver, + keanu_convene_path=static.keanu_convene_path, + auto_rotate=static.auto_rotate, + enable_clean_insights=bool(static.clean_insights_backend)) + form.group.render_kw = {"disabled": ""} + form.storage_cloud_account.render_kw = {"disabled": ""} + form.source_cloud_account.render_kw = {"disabled": ""} + if form.validate_on_submit(): + try: + static.update( + form.source_project.data, + form.description.data, + form.auto_rotate.data, + form.matrix_homeserver.data, + form.keanu_convene_path.data, + form.keanu_convene_logo.data, + form.keanu_convene_color.data, + form.enable_clean_insights.data, + True + ) + flash("Saved changes to group.", "success") + except ValueError as e: # may be returned by create_static_origin and from the int conversion + logging.warning(e) + flash("An error occurred saving the changes to the static origin due to an invalid input.", "danger") + except exc.SQLAlchemyError as e: + logging.warning(e) + flash("An error occurred saving the changes to the static origin due to a database error.", "danger") + return render_template("static.html.j2", + section="static", + static=static, form=form) + + +@bp.route("/list") +def static_list() -> ResponseReturnValue: + statics: List[StaticOrigin] = StaticOrigin.query.order_by(StaticOrigin.description).all() + return render_template("list.html.j2", + section="static", + title="Static Origins", + item="static", + new_link=url_for("portal.static.static_new"), + items=statics + ) + + +@bp.route("/destroy/", methods=['GET', 'POST']) +def static_destroy(static_id: int) -> ResponseReturnValue: + static = StaticOrigin.query.filter(StaticOrigin.id == static_id, StaticOrigin.destroyed.is_(None)).first() + if static is None: + return response_404("The requested static origin could not be found.") + return view_lifecycle( + header=f"Destroy static origin {static.description}", + message=static.description, + success_message="All proxies from the destroyed static origin will shortly be destroyed at their providers, " + "and the static content will be removed from the cloud provider.", + success_view="portal.static.static_list", + section="static", + resource=static, + action="destroy" + ) diff --git a/app/portal/templates/base.html.j2 b/app/portal/templates/base.html.j2 index 0c3ace0..3625ee0 100644 --- a/app/portal/templates/base.html.j2 +++ b/app/portal/templates/base.html.j2 @@ -82,7 +82,7 @@
  • - {{ icon("cloud") }} Cloud Accounts @@ -106,8 +106,8 @@
  • - + {{ icon("hdd") }} Static Origins
    • diff --git a/app/portal/templates/list.html.j2 b/app/portal/templates/list.html.j2 index 13aed6c..745722a 100644 --- a/app/portal/templates/list.html.j2 +++ b/app/portal/templates/list.html.j2 @@ -1,7 +1,7 @@ {% extends "base.html.j2" %} {% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, cloud_accounts_table, eotk_table, groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, - pools_table, proxies_table, webhook_table %} + pools_table, proxies_table, static_table, webhook_table %} {% block content %}

    {{ title }}

    @@ -35,6 +35,8 @@ {% endif %} {% elif item == "origin" %} {{ origins_table(items) }} + {% elif item == "static" %} + {{ static_table(items) }} {% elif item == "pool" %} {{ pools_table(items) }} {% elif item == "proxy" %} diff --git a/app/portal/templates/static.html.j2 b/app/portal/templates/static.html.j2 new file mode 100644 index 0000000..b8683a1 --- /dev/null +++ b/app/portal/templates/static.html.j2 @@ -0,0 +1,15 @@ +{% extends "base.html.j2" %} +{% from 'bootstrap5/form.html' import render_form %} +{% from "tables.html.j2" import static_table %} + +{% block content %} +

    Static Origins

    +

    + {{ static.group.group_name }}: {{ static.description }} (#{{ static.id }}) +

    + +
    + {{ render_form(form) }} +
    + +{% endblock %} diff --git a/app/portal/templates/tables.html.j2 b/app/portal/templates/tables.html.j2 index bed167d..6d2238a 100644 --- a/app/portal/templates/tables.html.j2 +++ b/app/portal/templates/tables.html.j2 @@ -633,6 +633,57 @@ {% endmacro %} +{% macro static_table(statics) %} +
    + + + + + + + + + + + + + {% for static in statics %} + {% if not static.destroyed %} + + + + + + + + + {% endif %} + {% endfor %} + +
    DescriptionOrigin URLKeanu ConveneAuto-RotationGroupActions
    {{ static.description }}{% if static.origin_domain_name %} + ⎋ + {{ static.origin_domain_name }} + {% else %} + Still deploying + {% endif %}{% if static.origin_domain_name and static.keanu_convene_path %} + ⎋ + {% else %} + Still deploying + {% endif %}{% if static.auto_rotate %}✅{% else %}❌{% endif %} + {{ static.group.group_name }} + + View/Edit + Destroy +
    +
    +{% endmacro %} + {% macro webhook_table(webhooks) %}
    diff --git a/app/terraform/static/aws.py b/app/terraform/static/aws.py new file mode 100644 index 0000000..5aa2f17 --- /dev/null +++ b/app/terraform/static/aws.py @@ -0,0 +1,141 @@ +import logging +import os +from datetime import datetime +from typing import List, Any + +from flask import current_app + +from app.extensions import db +from app.models.base import Group +from app.models.cloud import CloudProvider, CloudAccount +from app.models.mirrors import StaticOrigin +from app.terraform.terraform import TerraformAutomation + + +def import_state(state: Any) -> None: + if not isinstance(state, dict): + raise RuntimeError("The Terraform state object returned was not a dict.") + if "child_modules" not in state['values']['root_module']: + # There are no CloudFront origins deployed to import state for + return + # CloudFront distributions (origins) + for mod in state['values']['root_module']['child_modules']: + if mod['address'].startswith('module.static_'): + static_id = mod['address'][len('module.static_'):] + logging.debug("Found static module in state: %s", static_id) + for res in mod['resources']: + if res['address'].endswith('aws_cloudfront_distribution.this'): + logging.debug("and found related cloudfront distribution") + static = StaticOrigin.query.filter(StaticOrigin.id == static_id).first() + static.origin_domain_name = res['values']['domain_name'] + logging.debug("and found static origin: %s to update with domain name: %s", static.id, + static.origin_domain_name) + static.terraform_updated = datetime.utcnow() + break + db.session.commit() + + +class StaticAWSAutomation(TerraformAutomation): + short_name = "static_aws" + description = "Deploy static origins to AWS" + provider = CloudProvider.AWS + cloud_name = "aws" + + template_parameters: List[str] = [] + + template = """ + terraform { + {{ backend_config }} + required_providers { + aws = { + version = "~> 4.67.0" + } + gitlab = { + source = "gitlabhq/gitlab" + version = "~> 15.11.0" + } + } + } + + {% for group in groups %} + module "label_{{ group.id }}" { + source = "cloudposse/label/null" + version = "0.25.0" + namespace = "{{ global_namespace }}" + tenant = "{{ group.group_name }}" + label_order = ["namespace", "tenant", "name", "attributes"] + } + {% endfor %} + + {% for account in source_cloud_accounts -%} + provider "gitlab" { + token = "{{ account.credentials['gitlab_token'] }}" + alias = "account_{{ account.id }}" + } + {% endfor -%} + + {% for account in storage_cloud_accounts -%} + provider "aws" { + access_key = "{{ account.credentials['aws_access_key'] }}" + secret_key = "{{ account.credentials['aws_secret_key'] }}" + region = "{{ account.credentials['aws_region'] }}" + alias = "account_{{ account.id }}" + } + + provider "aws" { + access_key = "{{ account.credentials['aws_access_key'] }}" + secret_key = "{{ account.credentials['aws_secret_key'] }}" + region = "us-east-1" + alias = "account_{{ account.id }}_us_east_1" + } + + {% for static in account.statics | selectattr("destroyed", "none") %} + module "static_{{ static.id }}" { + providers = { + aws = aws.account_{{ account.id }} + aws.us_east_1 = aws.account_{{ account.id }}_us_east_1 + gitlab = gitlab.account_{{ static.source_cloud_account_id }} + } + source = "{{ terraform_modules_path }}/terraform-aws-bc-static-origin" + name = "static" + context = module.label_{{ static.group.id }}.context + {% if static.keanu_convene_path -%} + keanu_convene_path = "{{ static.keanu_convene_path }}" + {%- endif %} + {% if static.keanu_convene_config -%} + keanu_convene_config = "{{ static.keanu_convene_config | replace('"', '\\\\"') }}" + {%- endif %} + {% if static.matrix_homeserver -%} + matrix_homeserver = "{{ static.matrix_homeserver }}" + {%- endif %} + gitlab_project = "{{ static.source_project }}" + attributes = ["{{ static.id }}"] + } + {% endfor -%} + {%- endfor %} + """ + + def tf_generate(self) -> None: + groups = Group.query.all() + storage_cloud_accounts = CloudAccount.query.filter( + CloudAccount.provider == CloudProvider.AWS + ).all() + source_cloud_accounts = CloudAccount.query.filter( + CloudAccount.provider == CloudProvider.GITLAB + ).all() + self.tf_write( + self.template, + groups=groups, + storage_cloud_accounts=storage_cloud_accounts, + source_cloud_accounts=source_cloud_accounts, + global_namespace=current_app.config['GLOBAL_NAMESPACE'], bypass_token=current_app.config['BYPASS_TOKEN'], + terraform_modules_path=os.path.join(*list(os.path.split(current_app.root_path))[:-1], 'terraform-modules'), + backend_config=f"""backend "http" {{ + lock_address = "{current_app.config['TFSTATE_BACKEND']}/{self.short_name}" + unlock_address = "{current_app.config['TFSTATE_BACKEND']}/{self.short_name}" + address = "{current_app.config['TFSTATE_BACKEND']}/{self.short_name}" + }}""", + **{k: current_app.config[k.upper()] for k in self.template_parameters}) + + def tf_posthook(self, *, prehook_result: Any = None) -> None: + import_state(self.tf_show()) diff --git a/migrations/versions/2d747ffb9928_add_static_origins.py b/migrations/versions/2d747ffb9928_add_static_origins.py new file mode 100644 index 0000000..0e601b0 --- /dev/null +++ b/migrations/versions/2d747ffb9928_add_static_origins.py @@ -0,0 +1,48 @@ +"""add static origins + +Revision ID: 2d747ffb9928 +Revises: 431704bac57a +Create Date: 2023-05-25 14:23:30.568572 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '2d747ffb9928' +down_revision = '431704bac57a' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('static_origin', + sa.Column('group_id', sa.Integer(), nullable=False), + sa.Column('storage_cloud_account_id', sa.Integer(), nullable=False), + sa.Column('source_cloud_account_id', sa.Integer(), nullable=False), + sa.Column('source_project', sa.String(length=255), nullable=False), + sa.Column('auto_rotate', sa.Boolean(), nullable=False), + sa.Column('matrix_homeserver', sa.String(length=255), nullable=True), + sa.Column('keanu_convene_path', sa.String(length=255), nullable=True), + sa.Column('keanu_convene_config', sa.String(), nullable=True), + sa.Column('clean_insights_backend', sa.String(length=255), nullable=True), + sa.Column('origin_domain_name', sa.String(length=255), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.String(length=255), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('destroyed', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_static_origin_group_id_group')), + sa.ForeignKeyConstraint(['source_cloud_account_id'], ['cloud_account.id'], name=op.f('fk_static_origin_source_cloud_account_id_cloud_account')), + sa.ForeignKeyConstraint(['storage_cloud_account_id'], ['cloud_account.id'], name=op.f('fk_static_origin_storage_cloud_account_id_cloud_account')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_static_origin')) + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('static_origin') + # ### end Alembic commands ### diff --git a/requirements.txt b/requirements.txt index 049c456..8e446d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +Pillow PyGithub alembic azure-identity @@ -10,14 +11,19 @@ flask flask-migrate flask-wtf jinja2 +lxml nose openpyxl prometheus_client pydantic +pytest python-dateutil python-gitlab pyyaml requests sqlalchemy +tftest tldextract +webcolors +werkzeug wtforms diff --git a/terraform-modules/terraform-aws-bc-static-origin b/terraform-modules/terraform-aws-bc-static-origin new file mode 160000 index 0000000..bbe1680 --- /dev/null +++ b/terraform-modules/terraform-aws-bc-static-origin @@ -0,0 +1 @@ +Subproject commit bbe16800b18815fbae1ff9e8965cf026f461e1c1 diff --git a/tests/utils/test_color.py b/tests/utils/test_color.py new file mode 100644 index 0000000..59466a5 --- /dev/null +++ b/tests/utils/test_color.py @@ -0,0 +1,28 @@ +import pytest + +from app.brm.utils import normalize_color + + +def test_normalize_color(): + # Test valid CSS color names + assert normalize_color('red') == '#ff0000' + assert normalize_color('white') == '#ffffff' + assert normalize_color('black') == '#000000' + # Test valid 6-digit hex color codes + assert normalize_color('#ffffff') == '#ffffff' + assert normalize_color('#000000') == '#000000' + assert normalize_color('#00ff00') == '#00ff00' + # Test valid 3-digit hex color codes + assert normalize_color('#fff') == '#fff' + assert normalize_color('#000') == '#000' + assert normalize_color('#0f0') == '#0f0' + # Test case insensitivity + assert normalize_color('#FFFFFF') == '#ffffff' + assert normalize_color('#ABCDEF') == '#abcdef' + # Test invalid color values + with pytest.raises(ValueError): + normalize_color('invalid') + with pytest.raises(ValueError): + normalize_color('#1234567') + with pytest.raises(ValueError): + normalize_color('#xyz') \ No newline at end of file diff --git a/tests/utils/test_images.py b/tests/utils/test_images.py new file mode 100644 index 0000000..3355233 --- /dev/null +++ b/tests/utils/test_images.py @@ -0,0 +1,59 @@ +import pytest +from PIL import Image, ImageDraw +from app.brm.utils import thumbnail_uploaded_image, create_data_uri +from werkzeug.datastructures import FileStorage +from io import BytesIO +import base64 + + +def create_test_image(image_format: str, extension: str): + """ + Creates a test image with the data stored in a BytesIO buffer. + + :return: A test image. + """ + img_byte = BytesIO() + img = Image.new('RGB', (500, 500), color=(73, 109, 137)) + d = ImageDraw.Draw(img) + d.text((10, 10), "Hello World", fill=(255, 255, 0)) + img.save(img_byte, format="jpeg") + img_byte.seek(0) + file = FileStorage(stream=img_byte, filename='test.jpg') + return file + + +@pytest.mark.parametrize("image_format, extension", [ + ("jpeg", "jpg"), + ("png", "png"), + ("webp", "webp"), +]) +def test_thumbnail_uploaded_image(image_format: str, extension: str): + file = create_test_image(image_format, extension) + output = thumbnail_uploaded_image(file) + image = Image.open(BytesIO(output)) + + # Check if the processed image size is as expected + assert image.size[0] <= 256 + assert image.size[1] <= 256 + + +@pytest.mark.parametrize("image_format, extension", [ + ("jpeg", "jpg"), + ("png", "png"), + ("webp", "webp"), +]) +def test_create_data_uri(image_format: str, extension: str): + file = create_test_image(image_format, extension) + img_bytes = thumbnail_uploaded_image(file) + file_extension = 'jpg' + data_uri = create_data_uri(img_bytes, file_extension) + + # Check if the data URI starts as expected + assert data_uri.startswith('data:image/jpeg;base64,') + + # Check if the base64 image can be decoded + try: + base64_data = data_uri.split(',')[1] + base64.b64decode(base64_data) + except Exception as e: + pytest.fail(f"Decoding base64 data URI failed with error {e}")