onions: add onion service management

app/models/base.py

@@ -10,6 +10,8 @@ class Group(AbstractConfiguration):
 
     origins = db.relationship("Origin", back_populates="group")
     bridgeconfs = db.relationship("BridgeConf", back_populates="group")
+    eotks = db.relationship("Eotk", back_populates="group")
+    onions = db.relationship("Onion", back_populates="group")
     alarms = db.relationship("Alarm", back_populates="group")
 
     @classmethod

app/models/mirrors.py

@@ -1,5 +1,10 @@
+from typing import Optional
+
+from tldextract import extract
+
 from app import db
 from app.models import AbstractConfiguration, AbstractResource
+from app.models.onions import Onion
 
 
 class Origin(AbstractConfiguration):
@@ -23,6 +28,13 @@ class Origin(AbstractConfiguration):
         for proxy in self.proxies:
             proxy.destroy()
 
+    def onion(self) -> Optional[str]:
+        tld = extract(self.domain_name).registered_domain
+        onion = Onion.query.filter(Onion.domain_name == tld).first()
+        if not onion:
+            return None
+        return self.domain_name.replace(tld, f"{onion.onion_name}")
+
 
 class Proxy(AbstractResource):
     origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)

app/models/onions.py

@@ -0,0 +1,18 @@
+from app.extensions import db
+from app.models import AbstractConfiguration, AbstractResource
+
+
+class Onion(AbstractConfiguration):
+    group_id = db.Column(db.Integer(), db.ForeignKey("group.id"), nullable=False)
+    domain_name = db.Column(db.String(255), nullable=False)
+    onion_name = db.Column(db.String(56), nullable=False, unique=True)
+
+    group = db.relationship("Group", back_populates="onions")
+
+
+class Eotk(AbstractResource):
+    group_id = db.Column(db.Integer(), db.ForeignKey("group.id"), nullable=False)
+    instance_id = db.Column(db.String(100), nullable=True)
+    region = db.Column(db.String(20), nullable=False)
+
+    group = db.relationship("Group", back_populates="eotks")

app/portal/__init__.py

@@ -12,6 +12,7 @@ from app.portal.bridgeconf import bp as bridgeconf
 from app.portal.bridge import bp as bridge
 from app.portal.group import bp as group
 from app.portal.origin import bp as origin
+from app.portal.onion import bp as onion
 from app.portal.proxy import bp as proxy
 from app.portal.util import response_404, view_lifecycle
 
@@ -21,6 +22,7 @@ portal.register_blueprint(bridgeconf, url_prefix="/bridgeconf")
 portal.register_blueprint(bridge, url_prefix="/bridge")
 portal.register_blueprint(group, url_prefix="/group")
 portal.register_blueprint(origin, url_prefix="/origin")
+portal.register_blueprint(onion, url_prefix="/onion")
 portal.register_blueprint(proxy, url_prefix="/proxy")
 
 

app/portal/onion.py

@@ -0,0 +1,109 @@
+from datetime import datetime
+
+from flask import flash, redirect, url_for, render_template, Response, Blueprint
+from flask_wtf import FlaskForm
+from sqlalchemy import exc
+from wtforms import StringField, SelectField, SubmitField
+from wtforms.validators import DataRequired, Length
+
+from app.extensions import db
+from app.models.base import Group
+from app.models.onions import Onion
+from app.portal.util import response_404, view_lifecycle
+
+bp = Blueprint("onion", __name__)
+
+
+class NewOnionForm(FlaskForm):
+    domain_name = StringField('Domain Name', validators=[DataRequired()])
+    onion_name = StringField('Onion Name', validators=[DataRequired(), Length(min=56, max=56)],
+                             description="Onion service hostname, excluding the .onion suffix")
+    description = StringField('Description', validators=[DataRequired()])
+    group = SelectField('Group', validators=[DataRequired()])
+    submit = SubmitField('Save Changes')
+
+
+class EditOnionForm(FlaskForm):
+    description = StringField('Description', validators=[DataRequired()])
+    group = SelectField('Group', validators=[DataRequired()])
+    submit = SubmitField('Save Changes')
+
+
+@bp.route("/new", methods=['GET', 'POST'])
+@bp.route("/new/<group_id>", methods=['GET', 'POST'])
+def onion_new(group_id=None):
+    form = NewOnionForm()
+    form.group.choices = [(x.id, x.group_name) for x in Group.query.all()]
+    if form.validate_on_submit():
+        onion = Onion()
+        onion.group_id = form.group.data
+        onion.domain_name = form.domain_name.data
+        onion.onion_name = form.onion_name.data
+        onion.description = form.description.data
+        onion.created = datetime.utcnow()
+        onion.updated = datetime.utcnow()
+        try:
+            db.session.add(onion)
+            db.session.commit()
+            flash(f"Created new onion {onion.onion_name}.", "success")
+            return redirect(url_for("portal.onion.onion_edit", onion_id=onion.id))
+        except exc.SQLAlchemyError as e:
+            print(e)
+            flash("Failed to create new onion.", "danger")
+            return redirect(url_for("portal.onion.onion_list"))
+    if group_id:
+        form.group.data = group_id
+    return render_template("new.html.j2", section="onion", form=form)
+
+
+@bp.route('/edit/<onion_id>', methods=['GET', 'POST'])
+def onion_edit(onion_id):
+    onion = Onion.query.filter(Onion.id == onion_id).first()
+    if onion is None:
+        return Response(render_template("error.html.j2",
+                                        section="onion",
+                                        header="404 Onion Not Found",
+                                        message="The requested onion service could not be found."),
+                        status=404)
+    form = EditOnionForm(group=onion.group_id,
+                         description=onion.description)
+    form.group.choices = [(x.id, x.group_name) for x in Group.query.all()]
+    if form.validate_on_submit():
+        onion.group_id = form.group.data
+        onion.description = form.description.data
+        onion.updated = datetime.utcnow()
+        try:
+            db.session.commit()
+            flash("Saved changes to group.", "success")
+        except exc.SQLAlchemyError:
+            flash("An error occurred saving the changes to the group.", "danger")
+    return render_template("onion.html.j2",
+                           section="onion",
+                           onion=onion, form=form)
+
+
+@bp.route("/list")
+def onion_list():
+    onions = Onion.query.order_by(Onion.domain_name).all()
+    return render_template("list.html.j2",
+                           section="onion",
+                           title="Onion Services",
+                           item="onion service",
+                           new_link=url_for("portal.onion.onion_new"),
+                           items=onions)
+
+
+@bp.route("/destroy/<onion_id>", methods=['GET', 'POST'])
+def onion_destroy(onion_id: int):
+    onion = Onion.query.filter(Onion.id == onion_id, Onion.destroyed == None).first()
+    if onion is None:
+        return response_404("The requested onion service could not be found.")
+    return view_lifecycle(
+        header=f"Destroy onion service {onion.onion_name}",
+        message=onion.description,
+        success_message="You will need to manually remove this from the EOTK configuration.",
+        success_view="portal.onion.onion_list",
+        section="onion",
+        resource=onion,
+        action="destroy"
+    )

app/portal/origin.py

@@ -95,6 +95,17 @@ def origin_list():
                            items=origins)
 
 
+@bp.route("/onion")
+def origin_onion():
+    origins = Origin.query.order_by(Origin.domain_name).all()
+    return render_template("list.html.j2",
+                           section="origin",
+                           title="Onion Sites",
+                           item="onion service",
+                           new_link=url_for("portal.onion.onion_new"),
+                           items=origins)
+
+
 @bp.route("/destroy/<origin_id>", methods=['GET', 'POST'])
 def origin_destroy(origin_id: int):
     origin = Origin.query.filter(Origin.id == origin_id, Origin.destroyed == None).first()
@@ -104,7 +115,7 @@ def origin_destroy(origin_id: int):
         header=f"Destroy origin {origin.domain_name}",
         message=origin.description,
         success_message="All proxies from the destroyed origin will shortly be destroyed at their providers.",
-        success_view="portal.view_origins",
+        success_view="portal.origin.origin_list",
         section="origin",
         resource=origin,
         action="destroy"

app/portal/templates/base.html.j2

@@ -82,6 +82,12 @@
                             Origins
                         </a>
                     </li>
+                    <li class="nav-item">
+                        <a class="nav-link{% if section == "onion" %} active{% endif %}"
+                           href="{{ url_for("portal.onion.onion_list") }}">
+                            Onion Services
+                        </a>
+                    </li>
                     <li class="nav-item">
                         <a class="nav-link{% if section == "list" %} active{% endif %}"
                            href="{{ url_for("portal.view_mirror_lists") }}">

app/portal/templates/list.html.j2

@@ -1,6 +1,7 @@
 {% extends "base.html.j2" %}
 {% from "tables.html.j2" import bridgeconfs_table, bridges_table,
-        groups_table, mirrorlists_table, origins_table, proxies_table %}
+        groups_table, mirrorlists_table, origins_table, origin_onion_table,
+        onions_table, proxies_table %}
 
 {% block content %}
     <h1 class="h2 mt-3">{{ title }}</h1>
@@ -17,6 +18,12 @@
         {{ groups_table(items) }}
     {% elif item == "mirror list" %}
         {{ mirrorlists_table(items) }}
+    {% elif item == "onion service" %}
+        {% if section == "onion" %}
+            {{ onions_table(items) }}
+        {% elif section == "origin" %}
+            {{ origin_onion_table(items) }}
+        {% endif %}
     {% elif item == "origin" %}
         {{ origins_table(items) }}
     {% elif item == "proxy" %}

app/portal/templates/onion.html.j2

@@ -0,0 +1,16 @@
+{% extends "base.html.j2" %}
+{% from 'bootstrap5/form.html' import render_form %}
+{% from "tables.html.j2" import proxies_table %}
+
+{% block content %}
+    <h1 class="h2 mt-3">Onion Services</h1>
+    <h2 class="h3">
+        {{ onion.group.group_name }}: {{ onion.onion_name }}.onion
+        <a href="https://{{ onion.onion_name }}.onion/" class="btn btn-secondary btn-sm" target="_bypass"
+           rel="noopener noreferer">⎋</a>
+    </h2>
+
+    <div style="border: 1px solid #666;" class="p-3">
+        {{ render_form(form) }}
+    </div>
+{% endblock %}

app/portal/templates/tables.html.j2

@@ -61,6 +61,7 @@
                 <th scope="col">Name</th>
                 <th scope="col">Description</th>
                 <th scope="col">Auto-rotation</th>
+                <th scope="col">Onion Service</th>
                 <th scope="col">Group</th>
                 <th scope="col">Actions</th>
             </tr>
@@ -76,6 +77,7 @@
                         </td>
                         <td>{{ origin.description }}</td>
                         <td>{% if origin.auto_rotation %}✅{% else %}❌{% endif %}</td>
+                        <td>{% if origin.onion() %}✅{% else %}❌{% endif %}</td>
                         <td>
                             <a href="{{ url_for("portal.group.group_edit", group_id=origin.group.id) }}">{{ origin.group.group_name }}</a>
                         </td>
@@ -91,6 +93,92 @@
     </div>
 {% endmacro %}
 
+{% macro origin_onion_table(origins) %}
+    <div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Name</th>
+                <th scope="col">Description</th>
+                <th scope="col">Onion Service</th>
+                <th scope="col">Group</th>
+                <th scope="col">Actions</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for origin in origins %}
+                {% if not origin.destroyed %}
+                    <tr>
+                        <td>
+                            <a href="https://{{ origin.domain_name }}" target="_bypass" rel="noopener noreferer"
+                               class="btn btn-secondary btn-sm">⎋</a>
+                            {{ origin.domain_name }}
+                        </td>
+                        <td>{{ origin.description }}</td>
+                        <td>
+                            {% if origin.onion() %}
+                            <a href="https://{{ origin.onion() }}.onion" target="_bypass" rel="noopener noreferer"
+                               class="btn btn-secondary btn-sm">⎋</a>
+                            {{ origin.onion() }}.onion
+                            {% endif %}
+                        </td>
+                        <td>
+                            <a href="{{ url_for("portal.group.group_edit", group_id=origin.group.id) }}">{{ origin.group.group_name }}</a>
+                        </td>
+                        <td>
+                            <a href="{{ url_for("portal.origin.origin_edit", origin_id=origin.id) }}"
+                               class="btn btn-primary btn-sm">View/Edit</a>
+                        </td>
+                    </tr>
+                {% endif %}
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endmacro %}
+
+{% macro onions_table(onions) %}
+    <div class="table-responsive">
+        <table class="table table-striped table-sm">
+            <thead>
+            <tr>
+                <th scope="col">Domain Name</th>
+                <th scope="col">Onion Name</th>
+                <th scope="col">Description</th>
+                <th scope="col">Group</th>
+                <th scope="col">Actions</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for onion in onions %}
+                {% if not onion.destroyed %}
+                    <tr>
+                        <td>
+                            <a href="https://{{ onion.domain_name }}" target="_bypass" rel="noopener noreferer"
+                               class="btn btn-secondary btn-sm">⎋</a>
+                            {{ onion.domain_name }}
+                        </td>
+                        <td>
+                            <a href="https://{{ onion.onion_name }}.onion" target="_bypass" rel="noopener noreferer"
+                               class="btn btn-secondary btn-sm">⎋</a>
+                            {{ onion.onion_name }}
+                        </td>
+                        <td>{{ onion.description }}</td>
+                        <td>
+                            <a href="{{ url_for("portal.group.group_edit", group_id=onion.group.id) }}">{{ onion.group.group_name }}</a>
+                        </td>
+                        <td>
+                            <a href="{{ url_for("portal.onion.onion_edit", onion_id=onion.id) }}"
+                               class="btn btn-primary btn-sm">View/Edit</a>
+                        </td>
+                    </tr>
+                {% endif %}
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endmacro %}
+
 {% macro proxies_table(proxies) %}
     <div class="table-responsive">
         <table class="table table-striped table-sm">

migrations/versions/c3d6e95caa79_onion_services.py

@@ -0,0 +1,54 @@
+"""onion services
+
+Revision ID: c3d6e95caa79
+Revises: 56fbcfa1138c
+Create Date: 2022-05-04 15:03:52.406674
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c3d6e95caa79'
+down_revision = '56fbcfa1138c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('eotk',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('added', sa.DateTime(), nullable=False),
+    sa.Column('updated', sa.DateTime(), nullable=False),
+    sa.Column('deprecated', sa.DateTime(), nullable=True),
+    sa.Column('deprecation_reason', sa.String(), nullable=True),
+    sa.Column('destroyed', sa.DateTime(), nullable=True),
+    sa.Column('group_id', sa.Integer(), nullable=False),
+    sa.Column('instance_id', sa.String(length=100), nullable=True),
+    sa.Column('region', sa.String(length=20), nullable=False),
+    sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_eotk_group_id_group')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_eotk'))
+    )
+    op.create_table('onion',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('description', sa.String(length=255), nullable=False),
+    sa.Column('added', sa.DateTime(), nullable=False),
+    sa.Column('updated', sa.DateTime(), nullable=False),
+    sa.Column('destroyed', sa.DateTime(), nullable=True),
+    sa.Column('group_id', sa.Integer(), nullable=False),
+    sa.Column('domain_name', sa.String(length=255), nullable=False),
+    sa.Column('onion_name', sa.String(length=56), nullable=False),
+    sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_onion_group_id_group')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_onion')),
+    sa.UniqueConstraint('onion_name', name=op.f('uq_onion_onion_name'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('onion')
+    op.drop_table('eotk')
+    # ### end Alembic commands ###