From 8efb7d918661486ae3a861fee5028cb31115a6c1 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Wed, 4 May 2022 15:36:36 +0100 Subject: [PATCH] onions: add onion service management --- app/models/base.py | 2 + app/models/mirrors.py | 12 ++ app/models/onions.py | 18 +++ app/portal/__init__.py | 2 + app/portal/onion.py | 109 ++++++++++++++++++ app/portal/origin.py | 13 ++- app/portal/templates/base.html.j2 | 6 + app/portal/templates/list.html.j2 | 9 +- app/portal/templates/onion.html.j2 | 16 +++ app/portal/templates/tables.html.j2 | 88 ++++++++++++++ .../versions/c3d6e95caa79_onion_services.py | 54 +++++++++ 11 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 app/models/onions.py create mode 100644 app/portal/onion.py create mode 100644 app/portal/templates/onion.html.j2 create mode 100644 migrations/versions/c3d6e95caa79_onion_services.py diff --git a/app/models/base.py b/app/models/base.py index ff8fcfe..4f3bc22 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -10,6 +10,8 @@ class Group(AbstractConfiguration): origins = db.relationship("Origin", back_populates="group") bridgeconfs = db.relationship("BridgeConf", back_populates="group") + eotks = db.relationship("Eotk", back_populates="group") + onions = db.relationship("Onion", back_populates="group") alarms = db.relationship("Alarm", back_populates="group") @classmethod diff --git a/app/models/mirrors.py b/app/models/mirrors.py index d7a9f3d..faa9e19 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -1,5 +1,10 @@ +from typing import Optional + +from tldextract import extract + from app import db from app.models import AbstractConfiguration, AbstractResource +from app.models.onions import Onion class Origin(AbstractConfiguration): @@ -23,6 +28,13 @@ class Origin(AbstractConfiguration): for proxy in self.proxies: proxy.destroy() + def onion(self) -> Optional[str]: + tld = extract(self.domain_name).registered_domain + onion = Onion.query.filter(Onion.domain_name == tld).first() + if not onion: + return None + return self.domain_name.replace(tld, f"{onion.onion_name}") + class Proxy(AbstractResource): origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False) diff --git a/app/models/onions.py b/app/models/onions.py new file mode 100644 index 0000000..faac6c6 --- /dev/null +++ b/app/models/onions.py @@ -0,0 +1,18 @@ +from app.extensions import db +from app.models import AbstractConfiguration, AbstractResource + + +class Onion(AbstractConfiguration): + group_id = db.Column(db.Integer(), db.ForeignKey("group.id"), nullable=False) + domain_name = db.Column(db.String(255), nullable=False) + onion_name = db.Column(db.String(56), nullable=False, unique=True) + + group = db.relationship("Group", back_populates="onions") + + +class Eotk(AbstractResource): + group_id = db.Column(db.Integer(), db.ForeignKey("group.id"), nullable=False) + instance_id = db.Column(db.String(100), nullable=True) + region = db.Column(db.String(20), nullable=False) + + group = db.relationship("Group", back_populates="eotks") diff --git a/app/portal/__init__.py b/app/portal/__init__.py index 65c8fe5..0154d15 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -12,6 +12,7 @@ from app.portal.bridgeconf import bp as bridgeconf from app.portal.bridge import bp as bridge from app.portal.group import bp as group from app.portal.origin import bp as origin +from app.portal.onion import bp as onion from app.portal.proxy import bp as proxy from app.portal.util import response_404, view_lifecycle @@ -21,6 +22,7 @@ portal.register_blueprint(bridgeconf, url_prefix="/bridgeconf") portal.register_blueprint(bridge, url_prefix="/bridge") portal.register_blueprint(group, url_prefix="/group") portal.register_blueprint(origin, url_prefix="/origin") +portal.register_blueprint(onion, url_prefix="/onion") portal.register_blueprint(proxy, url_prefix="/proxy") diff --git a/app/portal/onion.py b/app/portal/onion.py new file mode 100644 index 0000000..635f912 --- /dev/null +++ b/app/portal/onion.py @@ -0,0 +1,109 @@ +from datetime import datetime + +from flask import flash, redirect, url_for, render_template, Response, Blueprint +from flask_wtf import FlaskForm +from sqlalchemy import exc +from wtforms import StringField, SelectField, SubmitField +from wtforms.validators import DataRequired, Length + +from app.extensions import db +from app.models.base import Group +from app.models.onions import Onion +from app.portal.util import response_404, view_lifecycle + +bp = Blueprint("onion", __name__) + + +class NewOnionForm(FlaskForm): + domain_name = StringField('Domain Name', validators=[DataRequired()]) + onion_name = StringField('Onion Name', validators=[DataRequired(), Length(min=56, max=56)], + description="Onion service hostname, excluding the .onion suffix") + description = StringField('Description', validators=[DataRequired()]) + group = SelectField('Group', validators=[DataRequired()]) + submit = SubmitField('Save Changes') + + +class EditOnionForm(FlaskForm): + description = StringField('Description', validators=[DataRequired()]) + group = SelectField('Group', validators=[DataRequired()]) + submit = SubmitField('Save Changes') + + +@bp.route("/new", methods=['GET', 'POST']) +@bp.route("/new/", methods=['GET', 'POST']) +def onion_new(group_id=None): + form = NewOnionForm() + form.group.choices = [(x.id, x.group_name) for x in Group.query.all()] + if form.validate_on_submit(): + onion = Onion() + onion.group_id = form.group.data + onion.domain_name = form.domain_name.data + onion.onion_name = form.onion_name.data + onion.description = form.description.data + onion.created = datetime.utcnow() + onion.updated = datetime.utcnow() + try: + db.session.add(onion) + db.session.commit() + flash(f"Created new onion {onion.onion_name}.", "success") + return redirect(url_for("portal.onion.onion_edit", onion_id=onion.id)) + except exc.SQLAlchemyError as e: + print(e) + flash("Failed to create new onion.", "danger") + return redirect(url_for("portal.onion.onion_list")) + if group_id: + form.group.data = group_id + return render_template("new.html.j2", section="onion", form=form) + + +@bp.route('/edit/', methods=['GET', 'POST']) +def onion_edit(onion_id): + onion = Onion.query.filter(Onion.id == onion_id).first() + if onion is None: + return Response(render_template("error.html.j2", + section="onion", + header="404 Onion Not Found", + message="The requested onion service could not be found."), + status=404) + form = EditOnionForm(group=onion.group_id, + description=onion.description) + form.group.choices = [(x.id, x.group_name) for x in Group.query.all()] + if form.validate_on_submit(): + onion.group_id = form.group.data + onion.description = form.description.data + onion.updated = datetime.utcnow() + try: + db.session.commit() + flash("Saved changes to group.", "success") + except exc.SQLAlchemyError: + flash("An error occurred saving the changes to the group.", "danger") + return render_template("onion.html.j2", + section="onion", + onion=onion, form=form) + + +@bp.route("/list") +def onion_list(): + onions = Onion.query.order_by(Onion.domain_name).all() + return render_template("list.html.j2", + section="onion", + title="Onion Services", + item="onion service", + new_link=url_for("portal.onion.onion_new"), + items=onions) + + +@bp.route("/destroy/", methods=['GET', 'POST']) +def onion_destroy(onion_id: int): + onion = Onion.query.filter(Onion.id == onion_id, Onion.destroyed == None).first() + if onion is None: + return response_404("The requested onion service could not be found.") + return view_lifecycle( + header=f"Destroy onion service {onion.onion_name}", + message=onion.description, + success_message="You will need to manually remove this from the EOTK configuration.", + success_view="portal.onion.onion_list", + section="onion", + resource=onion, + action="destroy" + ) diff --git a/app/portal/origin.py b/app/portal/origin.py index 5c7d7d1..81e6b44 100644 --- a/app/portal/origin.py +++ b/app/portal/origin.py @@ -95,6 +95,17 @@ def origin_list(): items=origins) +@bp.route("/onion") +def origin_onion(): + origins = Origin.query.order_by(Origin.domain_name).all() + return render_template("list.html.j2", + section="origin", + title="Onion Sites", + item="onion service", + new_link=url_for("portal.onion.onion_new"), + items=origins) + + @bp.route("/destroy/", methods=['GET', 'POST']) def origin_destroy(origin_id: int): origin = Origin.query.filter(Origin.id == origin_id, Origin.destroyed == None).first() @@ -104,7 +115,7 @@ def origin_destroy(origin_id: int): header=f"Destroy origin {origin.domain_name}", message=origin.description, success_message="All proxies from the destroyed origin will shortly be destroyed at their providers.", - success_view="portal.view_origins", + success_view="portal.origin.origin_list", section="origin", resource=origin, action="destroy" diff --git a/app/portal/templates/base.html.j2 b/app/portal/templates/base.html.j2 index 6449ac0..52d9910 100644 --- a/app/portal/templates/base.html.j2 +++ b/app/portal/templates/base.html.j2 @@ -82,6 +82,12 @@ Origins +
  • + + Onion Services + +
  • diff --git a/app/portal/templates/list.html.j2 b/app/portal/templates/list.html.j2 index 8810ba4..b7ed4e4 100644 --- a/app/portal/templates/list.html.j2 +++ b/app/portal/templates/list.html.j2 @@ -1,6 +1,7 @@ {% extends "base.html.j2" %} {% from "tables.html.j2" import bridgeconfs_table, bridges_table, - groups_table, mirrorlists_table, origins_table, proxies_table %} + groups_table, mirrorlists_table, origins_table, origin_onion_table, + onions_table, proxies_table %} {% block content %}

    {{ title }}

    @@ -17,6 +18,12 @@ {{ groups_table(items) }} {% elif item == "mirror list" %} {{ mirrorlists_table(items) }} + {% elif item == "onion service" %} + {% if section == "onion" %} + {{ onions_table(items) }} + {% elif section == "origin" %} + {{ origin_onion_table(items) }} + {% endif %} {% elif item == "origin" %} {{ origins_table(items) }} {% elif item == "proxy" %} diff --git a/app/portal/templates/onion.html.j2 b/app/portal/templates/onion.html.j2 new file mode 100644 index 0000000..97629c4 --- /dev/null +++ b/app/portal/templates/onion.html.j2 @@ -0,0 +1,16 @@ +{% extends "base.html.j2" %} +{% from 'bootstrap5/form.html' import render_form %} +{% from "tables.html.j2" import proxies_table %} + +{% block content %} +

    Onion Services

    +

    + {{ onion.group.group_name }}: {{ onion.onion_name }}.onion + ⎋ +

    + +
    + {{ render_form(form) }} +
    +{% endblock %} diff --git a/app/portal/templates/tables.html.j2 b/app/portal/templates/tables.html.j2 index c1bb6b1..bdeb5ed 100644 --- a/app/portal/templates/tables.html.j2 +++ b/app/portal/templates/tables.html.j2 @@ -61,6 +61,7 @@ Name Description Auto-rotation + Onion Service Group Actions @@ -76,6 +77,7 @@ {{ origin.description }} {% if origin.auto_rotation %}✅{% else %}❌{% endif %} + {% if origin.onion() %}✅{% else %}❌{% endif %} {{ origin.group.group_name }} @@ -91,6 +93,92 @@ {% endmacro %} +{% macro origin_onion_table(origins) %} +
    + + + + + + + + + + + + {% for origin in origins %} + {% if not origin.destroyed %} + + + + + + + + {% endif %} + {% endfor %} + +
    NameDescriptionOnion ServiceGroupActions
    + ⎋ + {{ origin.domain_name }} + {{ origin.description }} + {% if origin.onion() %} + ⎋ + {{ origin.onion() }}.onion + {% endif %} + + {{ origin.group.group_name }} + + View/Edit +
    +
    +{% endmacro %} + +{% macro onions_table(onions) %} +
    + + + + + + + + + + + + {% for onion in onions %} + {% if not onion.destroyed %} + + + + + + + + {% endif %} + {% endfor %} + +
    Domain NameOnion NameDescriptionGroupActions
    + ⎋ + {{ onion.domain_name }} + + ⎋ + {{ onion.onion_name }} + {{ onion.description }} + {{ onion.group.group_name }} + + View/Edit +
    +
    +{% endmacro %} + {% macro proxies_table(proxies) %}
    diff --git a/migrations/versions/c3d6e95caa79_onion_services.py b/migrations/versions/c3d6e95caa79_onion_services.py new file mode 100644 index 0000000..4dbf286 --- /dev/null +++ b/migrations/versions/c3d6e95caa79_onion_services.py @@ -0,0 +1,54 @@ +"""onion services + +Revision ID: c3d6e95caa79 +Revises: 56fbcfa1138c +Create Date: 2022-05-04 15:03:52.406674 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c3d6e95caa79' +down_revision = '56fbcfa1138c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('eotk', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('deprecated', sa.DateTime(), nullable=True), + sa.Column('deprecation_reason', sa.String(), nullable=True), + sa.Column('destroyed', sa.DateTime(), nullable=True), + sa.Column('group_id', sa.Integer(), nullable=False), + sa.Column('instance_id', sa.String(length=100), nullable=True), + sa.Column('region', sa.String(length=20), nullable=False), + sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_eotk_group_id_group')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_eotk')) + ) + op.create_table('onion', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.String(length=255), nullable=False), + sa.Column('added', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('destroyed', sa.DateTime(), nullable=True), + sa.Column('group_id', sa.Integer(), nullable=False), + sa.Column('domain_name', sa.String(length=255), nullable=False), + sa.Column('onion_name', sa.String(length=56), nullable=False), + sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_onion_group_id_group')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_onion')), + sa.UniqueConstraint('onion_name', name=op.f('uq_onion_onion_name')) + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('onion') + op.drop_table('eotk') + # ### end Alembic commands ###