feat: abstracting cloud providers

app/portal/__init__.py

@@ -14,6 +14,7 @@ from app.models.bridges import Bridge
 from app.models.mirrors import Origin, Proxy
 from app.models.base import Group
 from app.models.onions import Eotk
+from app.portal.cloud import bp as cloud
 from app.portal.automation import bp as automation
 from app.portal.bridgeconf import bp as bridgeconf
 from app.portal.bridge import bp as bridge
@@ -32,6 +33,7 @@ portal = Blueprint("portal", __name__, template_folder="templates", static_folde
 portal.register_blueprint(automation, url_prefix="/automation")
 portal.register_blueprint(bridgeconf, url_prefix="/bridgeconf")
 portal.register_blueprint(bridge, url_prefix="/bridge")
+portal.register_blueprint(cloud, url_prefix="/cloud")
 portal.register_blueprint(eotk, url_prefix="/eotk")
 portal.register_blueprint(group, url_prefix="/group")
 portal.register_blueprint(list_, url_prefix="/list")

app/portal/cloud.py

@@ -0,0 +1,243 @@
+from typing import List, Union, Optional, Dict, Type
+
+from flask import render_template, url_for, redirect, Blueprint
+from flask.typing import ResponseReturnValue
+from flask_wtf import FlaskForm
+from wtforms import SelectField, StringField, SubmitField, IntegerField, BooleanField, Form, FormField
+from wtforms.validators import InputRequired
+
+from app.extensions import db
+from app.models.cloud import CloudAccount, CloudProvider
+
+bp = Blueprint("cloud", __name__)
+
+_SECTION_TEMPLATE_VARS = {
+    "section": "cloud",
+    "help_url": "https://bypass.censorship.guide/user/cloud.html"
+}
+
+
+class NewCloudAccountForm(FlaskForm):  # type: ignore
+    provider = SelectField('Cloud Provider', validators=[InputRequired()])
+    submit = SubmitField('Next')
+
+
+class AWSAccountForm(FlaskForm):  # type: ignore
+    provider = StringField('Platform', render_kw={"disabled": ""})
+    description = StringField('Description', validators=[InputRequired()])
+    aws_access_key = StringField('AWS Access Key', validators=[InputRequired()])
+    aws_secret_key = StringField('AWS Secret Key', validators=[InputRequired()])
+    aws_region = StringField('AWS Region', default='us-east-2', validators=[InputRequired()])
+    max_distributions = IntegerField('Cloudfront Distributions Quota', default=200,
+                                     description="This is the quota for number of distributions per account.",
+                                     validators=[InputRequired()])
+    max_instances = IntegerField('EC2 Instance Quota', default=2,
+                                 description="This can be impacted by a number of quotas including instance limits "
+                                             "and IP address limits.",
+                                 validators=[InputRequired()])
+    enabled = BooleanField('Enable this account', default=True,
+                           description="New resources will not be deployed to disabled accounts, however existing "
+                                       "resources will persist until destroyed at the end of their lifecycle.")
+    submit = SubmitField('Save Changes')
+
+
+class HcloudAccountForm(FlaskForm):  # type: ignore
+    provider = StringField('Platform', render_kw={"disabled": ""})
+    description = StringField('Description', validators=[InputRequired()])
+    hcloud_token = StringField('Hetzner Cloud Token', validators=[InputRequired()])
+    max_instances = IntegerField('Server Limit', default=10,
+                                 validators=[InputRequired()])
+    enabled = BooleanField('Enable this account', default=True,
+                           description="New resources will not be deployed to disabled accounts, however existing "
+                                       "resources will persist until destroyed at the end of their lifecycle.")
+    submit = SubmitField('Save Changes')
+
+
+class OvhHorizonForm(Form):  # type: ignore[misc]
+    ovh_openstack_user = StringField("User")
+    ovh_openstack_password = StringField("Password")
+    ovh_openstack_tenant_id = StringField("Tenant ID")
+
+
+class OvhApiForm(Form):  # type: ignore[misc]
+    ovh_cloud_application_key = StringField("Application Key")
+    ovh_cloud_application_secret = StringField("Application Secret")
+    ovh_cloud_consumer_key = StringField("Consumer Key")
+
+
+class OvhAccountForm(FlaskForm):  # type: ignore
+    provider = StringField('Platform', render_kw={"disabled": ""})
+    description = StringField('Description', validators=[InputRequired()])
+    horizon = FormField(OvhHorizonForm, 'OpenStack Horizon API')
+    ovh_api = FormField(OvhApiForm, 'OVH API')
+    max_instances = IntegerField('Server Limit', default=10,
+                                 validators=[InputRequired()])
+    enabled = BooleanField('Enable this account', default=True,
+                           description="New resources will not be deployed to disabled accounts, however existing "
+                                       "resources will persist until destroyed at the end of their lifecycle.")
+    submit = SubmitField('Save Changes')
+
+
+class GandiHorizonForm(Form):  # type: ignore[misc]
+    gandi_openstack_user = StringField("User")
+    gandi_openstack_password = StringField("Password")
+    gandi_openstack_tenant_id = StringField("Tenant ID")
+
+
+class GandiAccountForm(FlaskForm):  # type: ignore
+    provider = StringField('Platform', render_kw={"disabled": ""})
+    description = StringField('Description', validators=[InputRequired()])
+    horizon = FormField(GandiHorizonForm, 'OpenStack Horizon API')
+    max_instances = IntegerField('Server Limit', default=10,
+                                 validators=[InputRequired()])
+    enabled = BooleanField('Enable this account', default=True,
+                           description="New resources will not be deployed to disabled accounts, however existing "
+                                       "resources will persist until destroyed at the end of their lifecycle.")
+    submit = SubmitField('Save Changes')
+
+
+CloudAccountForm = Union[AWSAccountForm, HcloudAccountForm, GandiAccountForm, OvhAccountForm]
+
+provider_forms: Dict[str, Type[CloudAccountForm]] = {
+    CloudProvider.AWS.name: AWSAccountForm,
+    CloudProvider.HCLOUD.name: HcloudAccountForm,
+    CloudProvider.GANDI.name: GandiAccountForm,
+    CloudProvider.OVH.name: OvhAccountForm,
+}
+
+
+def cloud_account_save(account: Optional[CloudAccount], provider: CloudProvider, form: CloudAccountForm) -> None:
+    if not account:
+        account = CloudAccount()
+        account.provider = provider
+        db.session.add(account)
+    if account.provider != provider:
+        raise RuntimeError("Provider mismatch in saving cloud account.")
+    account.description = form.description.data
+    account.enabled = form.enabled.data
+    if provider == CloudProvider.AWS and isinstance(form, AWSAccountForm):
+        account.credentials = {
+            "aws_access_key": form.aws_access_key.data,
+            "aws_secret_key": form.aws_secret_key.data,
+            "aws_region": form.aws_region.data,
+        }
+        account.max_distributions = form.max_distributions.data
+        account.max_sub_distributions = 1
+        account.max_instances = form.max_instances.data
+    elif provider == CloudProvider.HCLOUD and isinstance(form, HcloudAccountForm):
+        account.credentials = {
+            "hcloud_token": form.hcloud_token.data,
+        }
+        account.max_distributions = 0
+        account.max_sub_distributions = 0
+        account.max_instances = form.max_instances.data
+    elif provider == CloudProvider.GANDI and isinstance(form, GandiAccountForm):
+        account.credentials = {
+            "gandi_openstack_user": form.horizon.data["gandi_openstack_user"],
+            "gandi_openstack_password": form.horizon.data["gandi_openstack_password"],
+            "gandi_openstack_tenant_id": form.horizon.data["gandi_openstack_tenant_id"],
+        }
+        account.max_distributions = 0
+        account.max_sub_distributions = 0
+        account.max_instances = form.max_instances.data
+    elif provider == CloudProvider.OVH and isinstance(form, OvhAccountForm):
+        account.credentials = {
+            "ovh_openstack_user": form.horizon.data["ovh_openstack_user"],
+            "ovh_openstack_password": form.horizon.data["ovh_openstack_password"],
+            "ovh_openstack_tenant_id": form.horizon.data["ovh_openstack_tenant_id"],
+            "ovh_cloud_application_key": form.ovh_api.data["ovh_cloud_application_key"],
+            "ovh_cloud_application_secret": form.ovh_api.data["ovh_cloud_application_secret"],
+            "ovh_cloud_consumer_key": form.ovh_api.data["ovh_cloud_consumer_key"],
+        }
+        account.max_distributions = 0
+        account.max_sub_distributions = 0
+        account.max_instances = form.max_instances.data
+    else:
+        raise RuntimeError("Unknown provider or form data did not match provider.")
+
+
+def cloud_account_populate(form: CloudAccountForm, account: CloudAccount) -> None:
+    form.provider.data = account.provider.description
+    form.description.data = account.description
+    form.enabled.data = account.enabled
+    if account.provider == CloudProvider.AWS and isinstance(form, AWSAccountForm):
+        form.aws_access_key.data = account.credentials["aws_access_key"]
+        form.aws_secret_key.data = account.credentials["aws_secret_key"]
+        form.aws_region.data = account.credentials["aws_region"]
+        form.max_distributions.data = account.max_distributions
+        form.max_instances.data = account.max_instances
+    elif account.provider == CloudProvider.HCLOUD and isinstance(form, HcloudAccountForm):
+        form.hcloud_token.data = account.credentials["hcloud_token"]
+        form.max_instances.data = account.max_instances
+    elif account.provider == CloudProvider.GANDI and isinstance(form, GandiAccountForm):
+        form.horizon.form.gandi_openstack_user.data = account.credentials["gandi_openstack_user"]
+        form.horizon.form.gandi_openstack_password.data = account.credentials["gandi_openstack_password"]
+        form.horizon.form.gandi_openstack_tenant_id.data = account.credentials["gandi_openstack_tenant_id"]
+        form.max_instances.data = account.max_instances
+    elif account.provider == CloudProvider.OVH and isinstance(form, OvhAccountForm):
+        form.horizon.form.ovh_openstack_user.data = account.credentials["ovh_openstack_user"]
+        form.horizon.form.ovh_openstack_password.data = account.credentials["ovh_openstack_password"]
+        form.horizon.form.ovh_openstack_tenant_id.data = account.credentials["ovh_openstack_tenant_id"]
+        form.ovh_api.form.ovh_cloud_application_key.data = account.credentials["ovh_cloud_application_key"]
+        form.ovh_api.form.ovh_cloud_application_secret.data = account.credentials["ovh_cloud_application_secret"]
+        form.ovh_api.form.ovh_cloud_consumer_key.data = account.credentials["ovh_cloud_consumer_key"]
+        form.max_instances.data = account.max_instances
+    else:
+        raise RuntimeError(f"Unknown provider {account.provider} or form data {type(form)} did not match provider.")
+
+
+@bp.route("/list")
+def cloud_account_list() -> ResponseReturnValue:
+    accounts: List[CloudAccount] = CloudAccount.query.filter(CloudAccount.destroyed.is_(None)).all()
+    return render_template("list.html.j2",
+                           title="Cloud Accounts",
+                           item="cloud account",
+                           items=accounts,
+                           new_link=url_for("portal.cloud.cloud_account_new"),
+                           **_SECTION_TEMPLATE_VARS)
+
+
+@bp.route("/new", methods=['GET', 'POST'])
+def cloud_account_new() -> ResponseReturnValue:
+    form = NewCloudAccountForm()
+    form.provider.choices = sorted([
+        (provider.name, provider.description) for provider in CloudProvider
+    ], key=lambda p: p[1].lower())  # type: ignore[no-any-return]
+    if form.validate_on_submit():
+        return redirect(url_for("portal.cloud.cloud_account_new_for", provider=form.provider.data))
+    return render_template("new.html.j2",
+                           form=form,
+                           **_SECTION_TEMPLATE_VARS)
+
+
+@bp.route("/new/<provider>", methods=['GET', 'POST'])
+def cloud_account_new_for(provider: str) -> ResponseReturnValue:
+    form = provider_forms[provider]()
+    form.provider.data = CloudProvider[provider].description
+    if form.validate_on_submit():
+        cloud_account_save(None, CloudProvider[provider], form)
+        db.session.commit()
+        return redirect(url_for("portal.cloud.cloud_account_list"))
+    return render_template("new.html.j2",
+                           form=form,
+                           **_SECTION_TEMPLATE_VARS)
+
+
+@bp.route("/edit/<account_id>", methods=['GET', 'POST'])
+def cloud_account_edit(account_id: int) -> ResponseReturnValue:
+    account = CloudAccount.query.filter(
+        CloudAccount.id == account_id,
+        CloudAccount.destroyed.is_(None),
+    ).first()
+    if not account:
+        return "Not found", 404
+    form = provider_forms[account.provider.name]()
+    if form.validate_on_submit():
+        cloud_account_save(account, account.provider, form)
+        print(account.description)
+        db.session.commit()
+        return redirect(url_for("portal.cloud.cloud_account_list"))
+    cloud_account_populate(form, account)
+    return render_template("new.html.j2",
+                           form=form,
+                           **_SECTION_TEMPLATE_VARS)

app/portal/templates/base.html.j2

@@ -81,6 +81,12 @@
                     <span>Configuration</span>
                 </h6>
                 <ul class="nav flex-column">
+                    <li class="nav-item">
+                        <a class="nav-link{% if section == "cloud" %} active{% else %} disabled text-secondary{% endif %}"
+                           href="{{ url_for("portal.cloud.cloud_account_list") }}">
+                            {{ icon("cloud") }} Cloud Accounts
+                        </a>
+                    </li>
                     <li class="nav-item">
                         <a class="nav-link{% if section == "pool" %} active{% endif %}"
                            href="{{ url_for("portal.pool.pool_list") }}">

app/portal/templates/icons.html.j2

@@ -16,6 +16,10 @@
              viewBox="0 0 16 16">
             <path d="M8 16a2 2 0 0 0 2-2H6a2 2 0 0 0 2 2zM8 1.918l-.797.161A4.002 4.002 0 0 0 4 6c0 .628-.134 2.197-.459 3.742-.16.767-.376 1.566-.663 2.258h10.244c-.287-.692-.502-1.49-.663-2.258C12.134 8.197 12 6.628 12 6a4.002 4.002 0 0 0-3.203-3.92L8 1.917zM14.22 12c.223.447.481.801.78 1H1c.299-.199.557-.553.78-1C2.68 10.2 3 6.88 3 6c0-2.42 1.72-4.44 4.005-4.901a1 1 0 1 1 1.99 0A5.002 5.002 0 0 1 13 6c0 .88.32 4.2 1.22 6z"/>
         </svg>
+    {% elif i == "cloud" %}
+        <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-cloud" viewBox="0 0 16 16">
+            <path d="M4.406 3.342A5.53 5.53 0 0 1 8 2c2.69 0 4.923 2 5.166 4.579C14.758 6.804 16 8.137 16 9.773 16 11.569 14.502 13 12.687 13H3.781C1.708 13 0 11.366 0 9.318c0-1.763 1.266-3.223 2.942-3.593.143-.863.698-1.723 1.464-2.383zm.653.757c-.757.653-1.153 1.44-1.153 2.056v.448l-.445.049C2.064 6.805 1 7.952 1 9.318 1 10.785 2.23 12 3.781 12h8.906C13.98 12 15 10.988 15 9.773c0-1.216-1.02-2.228-2.313-2.228h-.5v-.5C12.188 4.825 10.328 3 8 3a4.53 4.53 0 0 0-2.941 1.1z"/>
+        </svg>
     {% elif i == "collection" %}
         <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-collection"
              viewBox="0 0 16 16">

app/portal/templates/list.html.j2

@@ -1,7 +1,7 @@
 {% extends "base.html.j2" %}
-{% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, eotk_table,
-        groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, pools_table,
-        proxies_table, webhook_table %}
+{% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, cloud_accounts_table,
+        eotk_table, groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table,
+        pools_table, proxies_table, webhook_table %}
 
 {% block content %}
     <h1 class="h2 mt-3">{{ title }}</h1>
@@ -19,6 +19,8 @@
         {{ bridgeconfs_table(items) }}
     {% elif item == "bridge" %}
         {{ bridges_table(items) }}
+    {% elif item == "cloud account" %}
+        {{ cloud_accounts_table(items) }}
     {% elif item == "eotk" %}
         {{ eotk_table(items) }}
     {% elif item == "group" %}

app/portal/templates/tables.html.j2

@@ -185,6 +185,34 @@
     </div>
 {% endmacro %}
 
+{% macro cloud_accounts_table(accounts) %}
+    <div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Provider</th>
+                <th scope="col">Description</th>
+                <th scope="col">Enabled</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for account in accounts %}
+                {% if not account.destroyed %}
+                    <tr class="align-middle">
+                        <td>{{ account.provider.description }}</td>
+                        <td>{{ account.description }}</td>
+                        <td>{% if account.enabled %}✅{% else %}❌{% endif %}</td>
+                        <td>
+                            <a href="{{ url_for("portal.cloud.cloud_account_edit", account_id=account.id) }}" class="btn btn-primary btn-sm">View/Edit</a>
+                        </td>
+                    </tr>
+                {% endif %}
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endmacro %}
+
 {% macro groups_table(groups, pool=None) %}
 <div class="table-responsive">
         <table class="table table-striped table-sm">
@@ -491,10 +519,11 @@
         <table class="table table-striped table-sm">
             <thead>
             <tr>
-                <th scope="col">Pool</th>
-                <th scope="col">Configuration</th>
                 <th scope="col">Nickname</th>
                 <th scope="col">Hashed Fingerprint</th>
+                <th scope="col">Pool</th>
+                <th scope="col">Configuration</th>
+                <th scope="col">Cloud Account</th>
                 <th scope="col">Alarms</th>
                 <th scope="col">Actions</th>
             </tr>
@@ -503,10 +532,6 @@
             {% for bridge in bridges %}
                 {% if not bridge.destroyed %}
                     <tr class="align-middle{% if bridge.deprecated %} bg-warning{% endif %}">
-                        <td>
-                            <a href="{{ url_for("portal.pool.pool_edit", pool_id=bridge.conf.pool_id) }}">{{ bridge.conf.pool.pool_name }}</a>
-                        </td>
-                        <td>{{ bridge.conf.description }} ({{ bridge.provider }}/{{ bridge.conf.method }})</td>
                         <td>
                             <a href="https://metrics.torproject.org/rs.html#details/{{ bridge.hashed_fingerprint }}">
                                 {{ bridge.nickname }}
@@ -515,6 +540,15 @@
                         <td>
                             <code>{{ bridge.hashed_fingerprint }}</code>
                         </td>
+                        <td>
+                            <a href="{{ url_for("portal.pool.pool_edit", pool_id=bridge.conf.pool_id) }}">{{ bridge.conf.pool.pool_name }}</a>
+                        </td>
+                        <td>{{ bridge.conf.description }} ({{ bridge.conf.method }})</td>
+                        <td>
+                            <a href="{{ url_for("portal.cloud.cloud_account_edit", account_id=bridge.cloud_account_id) }}">
+                                {{ bridge.cloud_account.provider.description }} ({{ bridge.cloud_account.description }})
+                            </a>
+                        </td>
                         <td>
                             {% for alarm in bridge.alarms %}
                                 <span title="{{ alarm.alarm_type }}">