jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

proxies: add smart proxy support

Browse source 
still to do:

* document new configuration options
* add smart proxies to groups view
* import bandwidth and CPU alarms
This commit is contained in:
Iain Learmonth 2022-05-24 19:51:38 +01:00
parent 9b90101cf4
commit 66af6e6550
15 changed files with 275 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

2
app/cli/db.py
View file

@ -54,7 +54,7 @@ def impot(model: db.Model) -> None:
line[i] = None # type: ignore
else:
line[i] = datetime.datetime.strptime(line[i], "%Y-%m-%d %H:%M:%S.%f") # type: ignore
elif header[i] in ["eotk"]:
elif header[i] in ["eotk", "auto_rotation", "smart"]:
# boolean fields
line[i] = line[i] == "True" # type: ignore
elif header[i].endswith("_id") and line[i] == "":

1
app/models/base.py
View file

@ -13,6 +13,7 @@ class Group(AbstractConfiguration):
bridgeconfs = db.relationship("BridgeConf", back_populates="group")
eotks = db.relationship("Eotk", back_populates="group")
onions = db.relationship("Onion", back_populates="group")
smart_proxies = db.relationship("SmartProxy", back_populates="group")
@classmethod
def csv_header(cls) -> List[str]:

16
app/models/mirrors.py
View file

@ -12,6 +12,7 @@ class Origin(AbstractConfiguration):
group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
domain_name = db.Column(db.String(255), unique=True, nullable=False)
auto_rotation = db.Column(db.Boolean, nullable=False)
smart = db.Column(db.Boolean(), nullable=False)
group = db.relationship("Group", back_populates="origins")
proxies = db.relationship("Proxy", back_populates="origin")
@ -23,7 +24,7 @@ class Origin(AbstractConfiguration):
@classmethod
def csv_header(cls) -> List[str]:
return super().csv_header() + [
"group_id", "domain_name"
"group_id", "domain_name", "auto_rotation", "smart"
]
def destroy(self) -> None:
@ -59,3 +60,16 @@ class Proxy(AbstractResource):
return super().csv_header() + [
"origin_id", "provider", "psg", "slug", "terraform_updated", "url"
]
class SmartProxy(AbstractResource):
group_id = db.Column(db.Integer(), db.ForeignKey("group.id"), nullable=False)
instance_id = db.Column(db.String(100), nullable=True)
provider = db.Column(db.String(20), nullable=False)
region = db.Column(db.String(20), nullable=False)
group = db.relationship("Group", back_populates="smart_proxies")
@property
def brn(self) -> str:
return f"brn:{current_app.config['GLOBAL_NAMESPACE']}:{self.group_id}:mirror:{self.provider}:smart-proxy/1"

2
app/portal/__init__.py
View file

@ -21,6 +21,7 @@ from app.portal.list import bp as list_
from app.portal.origin import bp as origin
from app.portal.onion import bp as onion
from app.portal.proxy import bp as proxy
from app.portal.smart_proxy import bp as smart_proxy
from app.portal.webhook import bp as webhook
portal = Blueprint("portal", __name__, template_folder="templates", static_folder="static")
@ -33,6 +34,7 @@ portal.register_blueprint(list_, url_prefix="/list")
portal.register_blueprint(origin, url_prefix="/origin")
portal.register_blueprint(onion, url_prefix="/onion")
portal.register_blueprint(proxy, url_prefix="/proxy")
portal.register_blueprint(smart_proxy, url_prefix="/smart")
portal.register_blueprint(webhook, url_prefix="/webhook")

7
app/portal/origin.py
View file

@ -21,6 +21,7 @@ class NewOriginForm(FlaskForm): # type: ignore
description = StringField('Description', validators=[DataRequired()])
group = SelectField('Group', validators=[DataRequired()])
auto_rotate = BooleanField("Enable auto-rotation?", default=True)
smart_proxy = BooleanField("Requires smart proxy?", default=False)
submit = SubmitField('Save Changes')
@ -28,6 +29,7 @@ class EditOriginForm(FlaskForm): # type: ignore
description = StringField('Description', validators=[DataRequired()])
group = SelectField('Group', validators=[DataRequired()])
auto_rotate = BooleanField("Enable auto-rotation?")
smart_proxy = BooleanField("Requires smart proxy?")
submit = SubmitField('Save Changes')
@ -42,6 +44,7 @@ def origin_new(group_id: Optional[int] = None) -> ResponseReturnValue:
origin.domain_name = form.domain_name.data
origin.description = form.description.data
origin.auto_rotation = form.auto_rotate.data
origin.smart = form.smart_proxy.data
origin.created = datetime.utcnow()
origin.updated = datetime.utcnow()
try:
@ -69,12 +72,14 @@ def origin_edit(origin_id: int) -> ResponseReturnValue:
status=404)
form = EditOriginForm(group=origin.group_id,
description=origin.description,
auto_rotate=origin.auto_rotation)
auto_rotate=origin.auto_rotation,
smart_proxy=origin.smart)
form.group.choices = [(x.id, x.group_name) for x in Group.query.all()]
if form.validate_on_submit():
origin.group_id = form.group.data
origin.description = form.description.data
origin.auto_rotation = form.auto_rotate.data
origin.smart = form.smart_proxy.data
origin.updated = datetime.utcnow()
try:
db.session.commit()

17
app/portal/smart_proxy.py Normal file
View file

@ -0,0 +1,17 @@
from flask import render_template, Blueprint
from flask.typing import ResponseReturnValue
from sqlalchemy import desc
from app.models.mirrors import SmartProxy
bp = Blueprint("smart_proxy", __name__)
@bp.route("/list")
def smart_proxy_list() -> ResponseReturnValue:
instances = SmartProxy.query.filter(SmartProxy.destroyed.is_(None)).order_by(desc(SmartProxy.added)).all()
return render_template("list.html.j2",
section="smart_proxy",
title="Smart Proxy Instances",
item="smart proxy",
items=instances)

4
app/portal/templates/base.html.j2
View file

@ -132,8 +132,8 @@
</a>
</li>
<li class="nav-item">
<a class="nav-link{% if section == "smart_proxy" %} active{% endif %} disabled text-secondary"
href="#">
<a class="nav-link{% if section == "smart_proxy" %} active{% endif %}"
href="{{ url_for("portal.smart_proxy.smart_proxy_list") }}">
{{ icon("globe") }} Smart Proxy Instances
</a>
</li>

5
app/portal/templates/icons.html.j2
View file

@ -71,6 +71,11 @@
viewBox="0 0 16 16">
<path d="M8 4.5a7 7 0 0 0-7 7 .5.5 0 0 1-1 0 8 8 0 1 1 16 0 .5.5 0 0 1-1 0 7 7 0 0 0-7-7zm0 2a5 5 0 0 0-5 5 .5.5 0 0 1-1 0 6 6 0 1 1 12 0 .5.5 0 0 1-1 0 5 5 0 0 0-5-5zm0 2a3 3 0 0 0-3 3 .5.5 0 0 1-1 0 4 4 0 1 1 8 0 .5.5 0 0 1-1 0 3 3 0 0 0-3-3zm0 2a1 1 0 0 0-1 1 .5.5 0 0 1-1 0 2 2 0 1 1 4 0 .5.5 0 0 1-1 0 1 1 0 0 0-1-1z"/>
</svg>
{% elif i == "terminal" %}
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-terminal" viewBox="0 0 16 16">
<path d="M6 9a.5.5 0 0 1 .5-.5h3a.5.5 0 0 1 0 1h-3A.5.5 0 0 1 6 9zM3.854 4.146a.5.5 0 1 0-.708.708L4.793 6.5 3.146 8.146a.5.5 0 1 0 .708.708l2-2a.5.5 0 0 0 0-.708l-2-2z"/>
<path d="M2 1a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V3a2 2 0 0 0-2-2H2zm12 1a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V3a1 1 0 0 1 1-1h12z"/>
</svg>
{% elif i == "onion" %}
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-onion"
viewBox="0 2 24 24" style="margin-top: -3px;">

4
app/portal/templates/list.html.j2
View file

@ -1,6 +1,6 @@
{% extends "base.html.j2" %}
{% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, eotk_table,
groups_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, proxies_table,
groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, proxies_table,
webhook_table %}
{% block content %}
@ -35,6 +35,8 @@
{{ origins_table(items) }}
{% elif item == "proxy" %}
{{ proxies_table(items) }}
{% elif item == "smart proxy" %}
{{ instances_table("smart_proxy", items) }}
{% elif item == "webhook" %}
{{ webhook_table(items) }}
{% endif %}

23
app/portal/templates/tables.html.j2
View file

@ -1,3 +1,5 @@
{% from "icons.html.j2" import icon %}
{% macro alarm_ok() %}
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
class="bi bi-check-circle text-success" viewBox="0 0 16 16">
@ -49,7 +51,7 @@
</div>
{% endmacro %}
{% macro eotk_table(instances) %}
{% macro instances_table(application, instances) %}
<div class="table-responsive">
<table class="table table-striped table-sm">
<thead>
@ -92,8 +94,13 @@
{% endfor %}
</td>
<td>
<a href="{{ url_for("portal.eotk.eotk_conf", group_id=instance.group_id) }}"
{% if application in ["eotk"] %}
<a href="{{ url_for("portal." + application + "." + application + "_conf", group_id=instance.group_id) }}"
class="btn btn-primary btn-sm">Preview Configuration</a>
{% endif %}
<a href="https://{{ instance.region }}.console.aws.amazon.com/systems-manager/session-manager/{{ instance.instance_id }}?region={{ instance.region }}" class="btn btn-outline-secondary btn-sm" target="_ssm">
{{ icon("terminal") }}
</a>
</td>
</tr>
{% endif %}
@ -103,6 +110,10 @@
</div>
{% endmacro %}
{% macro eotk_table(instances) %}
{{ instances_table("eotk", instances) }}
{% endmacro %}
{% macro automations_table(automations) %}
<div class="table-responsive">
<table class="table table-striped table-sm">
@ -198,7 +209,8 @@
<tr>
<th scope="col">Name</th>
<th scope="col">Description</th>
<th scope="col">Auto-rotation</th>
<th scope="col">Auto-Rotation</th>
<th scope="col">Smart Proxy</th>
<th scope="col">Onion Service</th>
<th scope="col">Group</th>
<th scope="col">Actions</th>
@ -215,6 +227,7 @@
</td>
<td>{{ origin.description }}</td>
<td>{% if origin.auto_rotation %}✅{% else %}❌{% endif %}</td>
<td>{% if origin.smart %}✅{% else %}❌{% endif %}</td>
<td>{% if origin.onion() %}✅{% else %}❌{% endif %}</td>
<td>
<a href="{{ url_for("portal.group.group_edit", group_id=origin.group.id) }}">{{ origin.group.group_name }}</a>
@ -257,9 +270,9 @@
<td>{{ origin.description }}</td>
<td>
{% if origin.onion() %}
<a href="https://{{ origin.onion() }}.onion" target="_bypass" rel="noopener noreferer"
<a href="https://{{ origin.onion() }}" target="_bypass" rel="noopener noreferer"
class="btn btn-secondary btn-sm">⎋</a>
{{ origin.onion() }}.onion
{{ origin.onion() }}
{% endif %}
</td>
<td>

17
app/terraform/__init__.py
View file

@ -1,6 +1,8 @@
from abc import ABCMeta, abstractmethod
import os
from typing import Tuple, Optional
from typing import Tuple, Optional, Any
import jinja2
from app import app
@ -34,3 +36,16 @@ class BaseAutomation(metaclass=ABCMeta):
self.short_name or self.__class__.__name__.lower(),
filename or ""
)
def tmpl_write(self, filename: str, template: str, **kwargs: Any) -> None:
"""
Write a Jinja2 template to the working directory for use by an automation module.
:param filename: filename to write to
:param template: Jinja2 template
:param kwargs: variables for use with the template
:return: None
"""
tmpl = jinja2.Template(template)
with open(self.working_directory(filename), 'w') as tf:
tf.write(tmpl.render(**kwargs))

75
app/terraform/proxy/__init__.py
View file

@ -12,10 +12,32 @@ from tldextract import tldextract
from app import app
from app.extensions import db
from app.models.base import Group
from app.models.mirrors import Proxy
from app.models.mirrors import Proxy, Origin, SmartProxy
from app.terraform.terraform import TerraformAutomation
def update_smart_proxy_instance(group_id: int,
provider: str,
region: str,
instance_id: str) -> None:
print("SMART PROXY")
instance = SmartProxy.query.filter(
SmartProxy.group_id == group_id,
SmartProxy.region == region,
SmartProxy.provider == provider,
SmartProxy.destroyed.is_(None)
).first()
if instance is None:
instance = SmartProxy()
instance.added = datetime.datetime.utcnow()
instance.group_id = group_id
instance.provider = provider
instance.region = region
db.session.add(instance)
instance.updated = datetime.datetime.utcnow()
instance.instance_id = instance_id
class ProxyAutomation(TerraformAutomation):
subgroup_max = math.inf
"""
@ -35,6 +57,11 @@ class ProxyAutomation(TerraformAutomation):
in the templating of the Terraform configuration.
"""
smart_proxies = False
"""
Whether this provider supports "smart" proxies.
"""
def get_subgroups(self) -> Dict[int, Dict[int, int]]:
conn = db.engine.connect()
result = conn.execute(text("""
@ -118,18 +145,40 @@ class ProxyAutomation(TerraformAutomation):
self.import_state(self.tf_show())
def tf_generate(self) -> None:
groups = Group.query.all()
self.tf_write(
self.template,
groups=Group.query.all(),
groups=groups,
proxies=Proxy.query.filter(
Proxy.provider == self.provider,
Proxy.destroyed.is_(None)
).all(),
subgroups=self.get_subgroups(),
global_namespace=app.config['GLOBAL_NAMESPACE'],
bypass_token=app.config['BYPASS_TOKEN'],
**{
k: app.config[k.upper()]
for k in self.template_parameters
}
)
Proxy.provider == self.provider, Proxy.destroyed.is_(None)).all(), subgroups=self.get_subgroups(),
global_namespace=app.config['GLOBAL_NAMESPACE'], bypass_token=app.config['BYPASS_TOKEN'],
**{k: app.config[k.upper()] for k in self.template_parameters})
if self.smart_proxies:
for group in groups:
self.sp_config(group)
def sp_config(self, group: Group) -> None:
group_origins: List[Origin] = Origin.query.filter(
Origin.group_id == group.id,
Origin.destroyed.is_(None),
Origin.smart.is_(True)
).all()
self.tmpl_write(f"smart_proxy.{group.id}.conf", """
{% for origin in origins %}
server {
listen 443 ssl;
server_name origin-{{ origin.id }}.{{ provider }}.smart.censorship.guide;
location / {
proxy_set_header Accept-Encoding "";
proxy_ssl_server_name on;
proxy_pass https://{{ origin.domain_name }}/;
subs_filter_types text/html text/css text/xml;
subs_filter https://{{ origin.domain_name }}/ /;
}
ssl_certificate /etc/ssl/smart_proxy.crt;
ssl_certificate_key /etc/ssl/private/smart_proxy.key;
}
{% endfor %}
""",
provider=self.provider,
origins=group_origins)

78
app/terraform/proxy/cloudfront.py
View file

@ -3,34 +3,66 @@ from typing import Any
from app.extensions import db
from app.models.mirrors import Proxy
from app.terraform.proxy import ProxyAutomation
from app.terraform.proxy import ProxyAutomation, update_smart_proxy_instance
class ProxyCloudfrontAutomation(ProxyAutomation):
short_name = "proxy_cloudfront"
description = "Deploy proxies to AWS CloudFront"
provider = "cloudfront"
smart_proxies = True
template_parameters = [
"aws_access_key",
"aws_secret_key"
"aws_secret_key",
"rfc2136_nameserver",
"rfc2136_tsig_key",
"rfc2136_tsig_secret",
"smart_zone"
]
template = """
terraform {
required_providers {
acme = {
source = "vancluever/acme"
version = "~> 2.8.0"
}
aws = {
version = "~> 4.4.0"
}
dns = {
version = "~> 3.2.3"
}
}
}
provider "acme" {
server_url = "https://acme-v02.api.letsencrypt.org/directory"
}
provider "aws" {
access_key = "{{ aws_access_key }}"
secret_key = "{{ aws_secret_key }}"
region = "us-east-2"
}
provider "dns" {
update {
server = local.rfc2136_nameserver
key_name = local.rfc2136_tsig_key
key_secret = local.rfc2136_tsig_secret
key_algorithm = "hmac-sha512"
}
}
locals {
rfc2136_nameserver = "{{ rfc2136_nameserver }}"
rfc2136_tsig_key = "{{ rfc2136_tsig_key }}"
rfc2136_tsig_secret = "{{ rfc2136_tsig_secret }}"
smart_zone = "{{ smart_zone }}"
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
@ -55,13 +87,47 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
resource "aws_sns_topic" "alarms_{{ group.id }}" {
name = "${module.label_{{ group.id }}.id}-cloudfront-alarms"
}
{% for origin in group.origins | selectattr("destroyed", "none") | selectattr("smart") %}
{% if loop.first %}
module "smart_proxy_{{ group.id }}" {
source = "sr2c/bc-smart-proxy-instance/aws"
version = "0.0.1"
context = module.label_{{ group.id }}.context
name = "smart-proxy"
disable_api_termination = false
domain_name = "cloudfront.smart.${local.smart_zone}"
rfc2136_nameserver = local.rfc2136_nameserver
rfc2136_tsig_key = local.rfc2136_tsig_key
rfc2136_tsig_secret = local.rfc2136_tsig_secret
}
resource "aws_s3_object" "smart_config_{{ group.id }}" {
bucket = module.smart_proxy_{{ group.id }}.config_bucket_name
key = "default"
source = "smart_proxy.{{ group.id }}.conf"
etag = filemd5("smart_proxy.{{ group.id }}.conf")
}
{% endif %}
resource "dns_a_record_set" "smart_dns_{{ origin.id }}" {
zone = "{{ smart_zone }}"
name = "origin-{{ origin.id }}.cloudfront.smart"
addresses = module.smart_proxy_{{ origin.group.id }}.ip_addresses
ttl = 60
}
{% endfor %}
{% endfor %}
{% for proxy in proxies %}
module "cloudfront_{{ proxy.id }}" {
source = "sr2c/bc-proxy/aws"
version = "0.0.7"
{% if proxy.origin.smart %}
origin_domain = "origin-{{ proxy.origin.id }}.cloudfront.smart.{{ smart_zone[:-1] }}"
{% else %}
origin_domain = "{{ proxy.origin.domain_name }}"
{% endif %}
logging_bucket = module.log_bucket_{{ proxy.origin.group.id }}.bucket_domain_name
sns_topic_arn = aws_sns_topic.alarms_{{ proxy.origin.group.id }}.arn
low_bandwidth_alarm = false
@ -88,4 +154,12 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
proxy.slug = res['values']['id']
proxy.terraform_updated = datetime.datetime.utcnow()
break
for g in state["values"]["root_module"]["child_modules"]:
if g["address"].startswith("module.smart_proxy_"):
group_id = int(g["address"][len("module.smart_proxy_"):])
for s in g["child_modules"]:
if s["address"].endswith(".module.instance"):
for x in s["resources"]:
if x["address"].endswith(".module.instance.aws_instance.default[0]"):
update_smart_proxy_instance(group_id, self.provider, "us-east-2", x['values']['id'])
db.session.commit()

6
app/terraform/terraform.py
View file

@ -3,8 +3,6 @@ import subprocess # nosec
from abc import abstractmethod
from typing import Any, Optional, Tuple
import jinja2
from app.terraform import BaseAutomation
@ -151,6 +149,4 @@ class TerraformAutomation(BaseAutomation):
return json.loads(terraform.stdout)
def tf_write(self, template: str, **kwargs: Any) -> None:
tmpl = jinja2.Template(template)
with open(self.working_directory("main.tf"), 'w') as tf:
tf.write(tmpl.render(**kwargs))
self.tmpl_write("main.tf", template, **kwargs)

50
migrations/versions/133961a48525_add_smart_proxies.py Normal file
View file

@ -0,0 +1,50 @@
"""add smart proxies
Revision ID: 133961a48525
Revises: 31aec2f86c40
Create Date: 2022-05-24 14:56:43.071054
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '133961a48525'
down_revision = '31aec2f86c40'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('smart_proxy',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('added', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('deprecated', sa.DateTime(), nullable=True),
sa.Column('deprecation_reason', sa.String(), nullable=True),
sa.Column('destroyed', sa.DateTime(), nullable=True),
sa.Column('group_id', sa.Integer(), nullable=False),
sa.Column('instance_id', sa.String(length=100), nullable=True),
sa.Column('provider', sa.String(length=20), nullable=False),
sa.Column('region', sa.String(length=20), nullable=False),
sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_smart_proxy_group_id_group')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_smart_proxy'))
)
with op.batch_alter_table('origin', schema=None) as batch_op:
batch_op.add_column(sa.Column('smart', sa.Boolean(), nullable=True))
with op.batch_alter_table('origin', schema=None) as batch_op:
batch_op.execute("UPDATE origin SET smart=FALSE")
batch_op.alter_column(sa.Column('smart', sa.Boolean(), nullable=False))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('origin', schema=None) as batch_op:
batch_op.drop_column('smart')
op.drop_table('smart_proxy')
# ### end Alembic commands ###