proxies: add smart proxy support
still to do: * document new configuration options * add smart proxies to groups view * import bandwidth and CPU alarms
This commit is contained in:
parent
9b90101cf4
commit
66af6e6550
15 changed files with 275 additions and 32 deletions
|
@ -54,7 +54,7 @@ def impot(model: db.Model) -> None:
|
||||||
line[i] = None # type: ignore
|
line[i] = None # type: ignore
|
||||||
else:
|
else:
|
||||||
line[i] = datetime.datetime.strptime(line[i], "%Y-%m-%d %H:%M:%S.%f") # type: ignore
|
line[i] = datetime.datetime.strptime(line[i], "%Y-%m-%d %H:%M:%S.%f") # type: ignore
|
||||||
elif header[i] in ["eotk"]:
|
elif header[i] in ["eotk", "auto_rotation", "smart"]:
|
||||||
# boolean fields
|
# boolean fields
|
||||||
line[i] = line[i] == "True" # type: ignore
|
line[i] = line[i] == "True" # type: ignore
|
||||||
elif header[i].endswith("_id") and line[i] == "":
|
elif header[i].endswith("_id") and line[i] == "":
|
||||||
|
|
|
@ -13,6 +13,7 @@ class Group(AbstractConfiguration):
|
||||||
bridgeconfs = db.relationship("BridgeConf", back_populates="group")
|
bridgeconfs = db.relationship("BridgeConf", back_populates="group")
|
||||||
eotks = db.relationship("Eotk", back_populates="group")
|
eotks = db.relationship("Eotk", back_populates="group")
|
||||||
onions = db.relationship("Onion", back_populates="group")
|
onions = db.relationship("Onion", back_populates="group")
|
||||||
|
smart_proxies = db.relationship("SmartProxy", back_populates="group")
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def csv_header(cls) -> List[str]:
|
def csv_header(cls) -> List[str]:
|
||||||
|
|
|
@ -12,6 +12,7 @@ class Origin(AbstractConfiguration):
|
||||||
group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
|
group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
|
||||||
domain_name = db.Column(db.String(255), unique=True, nullable=False)
|
domain_name = db.Column(db.String(255), unique=True, nullable=False)
|
||||||
auto_rotation = db.Column(db.Boolean, nullable=False)
|
auto_rotation = db.Column(db.Boolean, nullable=False)
|
||||||
|
smart = db.Column(db.Boolean(), nullable=False)
|
||||||
|
|
||||||
group = db.relationship("Group", back_populates="origins")
|
group = db.relationship("Group", back_populates="origins")
|
||||||
proxies = db.relationship("Proxy", back_populates="origin")
|
proxies = db.relationship("Proxy", back_populates="origin")
|
||||||
|
@ -23,7 +24,7 @@ class Origin(AbstractConfiguration):
|
||||||
@classmethod
|
@classmethod
|
||||||
def csv_header(cls) -> List[str]:
|
def csv_header(cls) -> List[str]:
|
||||||
return super().csv_header() + [
|
return super().csv_header() + [
|
||||||
"group_id", "domain_name"
|
"group_id", "domain_name", "auto_rotation", "smart"
|
||||||
]
|
]
|
||||||
|
|
||||||
def destroy(self) -> None:
|
def destroy(self) -> None:
|
||||||
|
@ -59,3 +60,16 @@ class Proxy(AbstractResource):
|
||||||
return super().csv_header() + [
|
return super().csv_header() + [
|
||||||
"origin_id", "provider", "psg", "slug", "terraform_updated", "url"
|
"origin_id", "provider", "psg", "slug", "terraform_updated", "url"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class SmartProxy(AbstractResource):
|
||||||
|
group_id = db.Column(db.Integer(), db.ForeignKey("group.id"), nullable=False)
|
||||||
|
instance_id = db.Column(db.String(100), nullable=True)
|
||||||
|
provider = db.Column(db.String(20), nullable=False)
|
||||||
|
region = db.Column(db.String(20), nullable=False)
|
||||||
|
|
||||||
|
group = db.relationship("Group", back_populates="smart_proxies")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def brn(self) -> str:
|
||||||
|
return f"brn:{current_app.config['GLOBAL_NAMESPACE']}:{self.group_id}:mirror:{self.provider}:smart-proxy/1"
|
||||||
|
|
|
@ -21,6 +21,7 @@ from app.portal.list import bp as list_
|
||||||
from app.portal.origin import bp as origin
|
from app.portal.origin import bp as origin
|
||||||
from app.portal.onion import bp as onion
|
from app.portal.onion import bp as onion
|
||||||
from app.portal.proxy import bp as proxy
|
from app.portal.proxy import bp as proxy
|
||||||
|
from app.portal.smart_proxy import bp as smart_proxy
|
||||||
from app.portal.webhook import bp as webhook
|
from app.portal.webhook import bp as webhook
|
||||||
|
|
||||||
portal = Blueprint("portal", __name__, template_folder="templates", static_folder="static")
|
portal = Blueprint("portal", __name__, template_folder="templates", static_folder="static")
|
||||||
|
@ -33,6 +34,7 @@ portal.register_blueprint(list_, url_prefix="/list")
|
||||||
portal.register_blueprint(origin, url_prefix="/origin")
|
portal.register_blueprint(origin, url_prefix="/origin")
|
||||||
portal.register_blueprint(onion, url_prefix="/onion")
|
portal.register_blueprint(onion, url_prefix="/onion")
|
||||||
portal.register_blueprint(proxy, url_prefix="/proxy")
|
portal.register_blueprint(proxy, url_prefix="/proxy")
|
||||||
|
portal.register_blueprint(smart_proxy, url_prefix="/smart")
|
||||||
portal.register_blueprint(webhook, url_prefix="/webhook")
|
portal.register_blueprint(webhook, url_prefix="/webhook")
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -21,6 +21,7 @@ class NewOriginForm(FlaskForm): # type: ignore
|
||||||
description = StringField('Description', validators=[DataRequired()])
|
description = StringField('Description', validators=[DataRequired()])
|
||||||
group = SelectField('Group', validators=[DataRequired()])
|
group = SelectField('Group', validators=[DataRequired()])
|
||||||
auto_rotate = BooleanField("Enable auto-rotation?", default=True)
|
auto_rotate = BooleanField("Enable auto-rotation?", default=True)
|
||||||
|
smart_proxy = BooleanField("Requires smart proxy?", default=False)
|
||||||
submit = SubmitField('Save Changes')
|
submit = SubmitField('Save Changes')
|
||||||
|
|
||||||
|
|
||||||
|
@ -28,6 +29,7 @@ class EditOriginForm(FlaskForm): # type: ignore
|
||||||
description = StringField('Description', validators=[DataRequired()])
|
description = StringField('Description', validators=[DataRequired()])
|
||||||
group = SelectField('Group', validators=[DataRequired()])
|
group = SelectField('Group', validators=[DataRequired()])
|
||||||
auto_rotate = BooleanField("Enable auto-rotation?")
|
auto_rotate = BooleanField("Enable auto-rotation?")
|
||||||
|
smart_proxy = BooleanField("Requires smart proxy?")
|
||||||
submit = SubmitField('Save Changes')
|
submit = SubmitField('Save Changes')
|
||||||
|
|
||||||
|
|
||||||
|
@ -42,6 +44,7 @@ def origin_new(group_id: Optional[int] = None) -> ResponseReturnValue:
|
||||||
origin.domain_name = form.domain_name.data
|
origin.domain_name = form.domain_name.data
|
||||||
origin.description = form.description.data
|
origin.description = form.description.data
|
||||||
origin.auto_rotation = form.auto_rotate.data
|
origin.auto_rotation = form.auto_rotate.data
|
||||||
|
origin.smart = form.smart_proxy.data
|
||||||
origin.created = datetime.utcnow()
|
origin.created = datetime.utcnow()
|
||||||
origin.updated = datetime.utcnow()
|
origin.updated = datetime.utcnow()
|
||||||
try:
|
try:
|
||||||
|
@ -69,12 +72,14 @@ def origin_edit(origin_id: int) -> ResponseReturnValue:
|
||||||
status=404)
|
status=404)
|
||||||
form = EditOriginForm(group=origin.group_id,
|
form = EditOriginForm(group=origin.group_id,
|
||||||
description=origin.description,
|
description=origin.description,
|
||||||
auto_rotate=origin.auto_rotation)
|
auto_rotate=origin.auto_rotation,
|
||||||
|
smart_proxy=origin.smart)
|
||||||
form.group.choices = [(x.id, x.group_name) for x in Group.query.all()]
|
form.group.choices = [(x.id, x.group_name) for x in Group.query.all()]
|
||||||
if form.validate_on_submit():
|
if form.validate_on_submit():
|
||||||
origin.group_id = form.group.data
|
origin.group_id = form.group.data
|
||||||
origin.description = form.description.data
|
origin.description = form.description.data
|
||||||
origin.auto_rotation = form.auto_rotate.data
|
origin.auto_rotation = form.auto_rotate.data
|
||||||
|
origin.smart = form.smart_proxy.data
|
||||||
origin.updated = datetime.utcnow()
|
origin.updated = datetime.utcnow()
|
||||||
try:
|
try:
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
17
app/portal/smart_proxy.py
Normal file
17
app/portal/smart_proxy.py
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
from flask import render_template, Blueprint
|
||||||
|
from flask.typing import ResponseReturnValue
|
||||||
|
from sqlalchemy import desc
|
||||||
|
|
||||||
|
from app.models.mirrors import SmartProxy
|
||||||
|
|
||||||
|
bp = Blueprint("smart_proxy", __name__)
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route("/list")
|
||||||
|
def smart_proxy_list() -> ResponseReturnValue:
|
||||||
|
instances = SmartProxy.query.filter(SmartProxy.destroyed.is_(None)).order_by(desc(SmartProxy.added)).all()
|
||||||
|
return render_template("list.html.j2",
|
||||||
|
section="smart_proxy",
|
||||||
|
title="Smart Proxy Instances",
|
||||||
|
item="smart proxy",
|
||||||
|
items=instances)
|
|
@ -132,8 +132,8 @@
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
<li class="nav-item">
|
<li class="nav-item">
|
||||||
<a class="nav-link{% if section == "smart_proxy" %} active{% endif %} disabled text-secondary"
|
<a class="nav-link{% if section == "smart_proxy" %} active{% endif %}"
|
||||||
href="#">
|
href="{{ url_for("portal.smart_proxy.smart_proxy_list") }}">
|
||||||
{{ icon("globe") }} Smart Proxy Instances
|
{{ icon("globe") }} Smart Proxy Instances
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
|
|
|
@ -71,6 +71,11 @@
|
||||||
viewBox="0 0 16 16">
|
viewBox="0 0 16 16">
|
||||||
<path d="M8 4.5a7 7 0 0 0-7 7 .5.5 0 0 1-1 0 8 8 0 1 1 16 0 .5.5 0 0 1-1 0 7 7 0 0 0-7-7zm0 2a5 5 0 0 0-5 5 .5.5 0 0 1-1 0 6 6 0 1 1 12 0 .5.5 0 0 1-1 0 5 5 0 0 0-5-5zm0 2a3 3 0 0 0-3 3 .5.5 0 0 1-1 0 4 4 0 1 1 8 0 .5.5 0 0 1-1 0 3 3 0 0 0-3-3zm0 2a1 1 0 0 0-1 1 .5.5 0 0 1-1 0 2 2 0 1 1 4 0 .5.5 0 0 1-1 0 1 1 0 0 0-1-1z"/>
|
<path d="M8 4.5a7 7 0 0 0-7 7 .5.5 0 0 1-1 0 8 8 0 1 1 16 0 .5.5 0 0 1-1 0 7 7 0 0 0-7-7zm0 2a5 5 0 0 0-5 5 .5.5 0 0 1-1 0 6 6 0 1 1 12 0 .5.5 0 0 1-1 0 5 5 0 0 0-5-5zm0 2a3 3 0 0 0-3 3 .5.5 0 0 1-1 0 4 4 0 1 1 8 0 .5.5 0 0 1-1 0 3 3 0 0 0-3-3zm0 2a1 1 0 0 0-1 1 .5.5 0 0 1-1 0 2 2 0 1 1 4 0 .5.5 0 0 1-1 0 1 1 0 0 0-1-1z"/>
|
||||||
</svg>
|
</svg>
|
||||||
|
{% elif i == "terminal" %}
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-terminal" viewBox="0 0 16 16">
|
||||||
|
<path d="M6 9a.5.5 0 0 1 .5-.5h3a.5.5 0 0 1 0 1h-3A.5.5 0 0 1 6 9zM3.854 4.146a.5.5 0 1 0-.708.708L4.793 6.5 3.146 8.146a.5.5 0 1 0 .708.708l2-2a.5.5 0 0 0 0-.708l-2-2z"/>
|
||||||
|
<path d="M2 1a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V3a2 2 0 0 0-2-2H2zm12 1a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V3a1 1 0 0 1 1-1h12z"/>
|
||||||
|
</svg>
|
||||||
{% elif i == "onion" %}
|
{% elif i == "onion" %}
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-onion"
|
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-onion"
|
||||||
viewBox="0 2 24 24" style="margin-top: -3px;">
|
viewBox="0 2 24 24" style="margin-top: -3px;">
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
{% extends "base.html.j2" %}
|
{% extends "base.html.j2" %}
|
||||||
{% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, eotk_table,
|
{% from "tables.html.j2" import alarms_table, automations_table, bridgeconfs_table, bridges_table, eotk_table,
|
||||||
groups_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, proxies_table,
|
groups_table, instances_table, mirrorlists_table, origins_table, origin_onion_table, onions_table, proxies_table,
|
||||||
webhook_table %}
|
webhook_table %}
|
||||||
|
|
||||||
{% block content %}
|
{% block content %}
|
||||||
|
@ -35,6 +35,8 @@
|
||||||
{{ origins_table(items) }}
|
{{ origins_table(items) }}
|
||||||
{% elif item == "proxy" %}
|
{% elif item == "proxy" %}
|
||||||
{{ proxies_table(items) }}
|
{{ proxies_table(items) }}
|
||||||
|
{% elif item == "smart proxy" %}
|
||||||
|
{{ instances_table("smart_proxy", items) }}
|
||||||
{% elif item == "webhook" %}
|
{% elif item == "webhook" %}
|
||||||
{{ webhook_table(items) }}
|
{{ webhook_table(items) }}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
{% from "icons.html.j2" import icon %}
|
||||||
|
|
||||||
{% macro alarm_ok() %}
|
{% macro alarm_ok() %}
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
|
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
|
||||||
class="bi bi-check-circle text-success" viewBox="0 0 16 16">
|
class="bi bi-check-circle text-success" viewBox="0 0 16 16">
|
||||||
|
@ -49,7 +51,7 @@
|
||||||
</div>
|
</div>
|
||||||
{% endmacro %}
|
{% endmacro %}
|
||||||
|
|
||||||
{% macro eotk_table(instances) %}
|
{% macro instances_table(application, instances) %}
|
||||||
<div class="table-responsive">
|
<div class="table-responsive">
|
||||||
<table class="table table-striped table-sm">
|
<table class="table table-striped table-sm">
|
||||||
<thead>
|
<thead>
|
||||||
|
@ -92,8 +94,13 @@
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
<a href="{{ url_for("portal.eotk.eotk_conf", group_id=instance.group_id) }}"
|
{% if application in ["eotk"] %}
|
||||||
|
<a href="{{ url_for("portal." + application + "." + application + "_conf", group_id=instance.group_id) }}"
|
||||||
class="btn btn-primary btn-sm">Preview Configuration</a>
|
class="btn btn-primary btn-sm">Preview Configuration</a>
|
||||||
|
{% endif %}
|
||||||
|
<a href="https://{{ instance.region }}.console.aws.amazon.com/systems-manager/session-manager/{{ instance.instance_id }}?region={{ instance.region }}" class="btn btn-outline-secondary btn-sm" target="_ssm">
|
||||||
|
{{ icon("terminal") }}
|
||||||
|
</a>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
@ -103,6 +110,10 @@
|
||||||
</div>
|
</div>
|
||||||
{% endmacro %}
|
{% endmacro %}
|
||||||
|
|
||||||
|
{% macro eotk_table(instances) %}
|
||||||
|
{{ instances_table("eotk", instances) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
{% macro automations_table(automations) %}
|
{% macro automations_table(automations) %}
|
||||||
<div class="table-responsive">
|
<div class="table-responsive">
|
||||||
<table class="table table-striped table-sm">
|
<table class="table table-striped table-sm">
|
||||||
|
@ -198,7 +209,8 @@
|
||||||
<tr>
|
<tr>
|
||||||
<th scope="col">Name</th>
|
<th scope="col">Name</th>
|
||||||
<th scope="col">Description</th>
|
<th scope="col">Description</th>
|
||||||
<th scope="col">Auto-rotation</th>
|
<th scope="col">Auto-Rotation</th>
|
||||||
|
<th scope="col">Smart Proxy</th>
|
||||||
<th scope="col">Onion Service</th>
|
<th scope="col">Onion Service</th>
|
||||||
<th scope="col">Group</th>
|
<th scope="col">Group</th>
|
||||||
<th scope="col">Actions</th>
|
<th scope="col">Actions</th>
|
||||||
|
@ -215,6 +227,7 @@
|
||||||
</td>
|
</td>
|
||||||
<td>{{ origin.description }}</td>
|
<td>{{ origin.description }}</td>
|
||||||
<td>{% if origin.auto_rotation %}✅{% else %}❌{% endif %}</td>
|
<td>{% if origin.auto_rotation %}✅{% else %}❌{% endif %}</td>
|
||||||
|
<td>{% if origin.smart %}✅{% else %}❌{% endif %}</td>
|
||||||
<td>{% if origin.onion() %}✅{% else %}❌{% endif %}</td>
|
<td>{% if origin.onion() %}✅{% else %}❌{% endif %}</td>
|
||||||
<td>
|
<td>
|
||||||
<a href="{{ url_for("portal.group.group_edit", group_id=origin.group.id) }}">{{ origin.group.group_name }}</a>
|
<a href="{{ url_for("portal.group.group_edit", group_id=origin.group.id) }}">{{ origin.group.group_name }}</a>
|
||||||
|
@ -257,9 +270,9 @@
|
||||||
<td>{{ origin.description }}</td>
|
<td>{{ origin.description }}</td>
|
||||||
<td>
|
<td>
|
||||||
{% if origin.onion() %}
|
{% if origin.onion() %}
|
||||||
<a href="https://{{ origin.onion() }}.onion" target="_bypass" rel="noopener noreferer"
|
<a href="https://{{ origin.onion() }}" target="_bypass" rel="noopener noreferer"
|
||||||
class="btn btn-secondary btn-sm">⎋</a>
|
class="btn btn-secondary btn-sm">⎋</a>
|
||||||
{{ origin.onion() }}.onion
|
{{ origin.onion() }}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
from abc import ABCMeta, abstractmethod
|
from abc import ABCMeta, abstractmethod
|
||||||
import os
|
import os
|
||||||
from typing import Tuple, Optional
|
from typing import Tuple, Optional, Any
|
||||||
|
|
||||||
|
import jinja2
|
||||||
|
|
||||||
from app import app
|
from app import app
|
||||||
|
|
||||||
|
@ -34,3 +36,16 @@ class BaseAutomation(metaclass=ABCMeta):
|
||||||
self.short_name or self.__class__.__name__.lower(),
|
self.short_name or self.__class__.__name__.lower(),
|
||||||
filename or ""
|
filename or ""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def tmpl_write(self, filename: str, template: str, **kwargs: Any) -> None:
|
||||||
|
"""
|
||||||
|
Write a Jinja2 template to the working directory for use by an automation module.
|
||||||
|
|
||||||
|
:param filename: filename to write to
|
||||||
|
:param template: Jinja2 template
|
||||||
|
:param kwargs: variables for use with the template
|
||||||
|
:return: None
|
||||||
|
"""
|
||||||
|
tmpl = jinja2.Template(template)
|
||||||
|
with open(self.working_directory(filename), 'w') as tf:
|
||||||
|
tf.write(tmpl.render(**kwargs))
|
||||||
|
|
|
@ -12,10 +12,32 @@ from tldextract import tldextract
|
||||||
from app import app
|
from app import app
|
||||||
from app.extensions import db
|
from app.extensions import db
|
||||||
from app.models.base import Group
|
from app.models.base import Group
|
||||||
from app.models.mirrors import Proxy
|
from app.models.mirrors import Proxy, Origin, SmartProxy
|
||||||
from app.terraform.terraform import TerraformAutomation
|
from app.terraform.terraform import TerraformAutomation
|
||||||
|
|
||||||
|
|
||||||
|
def update_smart_proxy_instance(group_id: int,
|
||||||
|
provider: str,
|
||||||
|
region: str,
|
||||||
|
instance_id: str) -> None:
|
||||||
|
print("SMART PROXY")
|
||||||
|
instance = SmartProxy.query.filter(
|
||||||
|
SmartProxy.group_id == group_id,
|
||||||
|
SmartProxy.region == region,
|
||||||
|
SmartProxy.provider == provider,
|
||||||
|
SmartProxy.destroyed.is_(None)
|
||||||
|
).first()
|
||||||
|
if instance is None:
|
||||||
|
instance = SmartProxy()
|
||||||
|
instance.added = datetime.datetime.utcnow()
|
||||||
|
instance.group_id = group_id
|
||||||
|
instance.provider = provider
|
||||||
|
instance.region = region
|
||||||
|
db.session.add(instance)
|
||||||
|
instance.updated = datetime.datetime.utcnow()
|
||||||
|
instance.instance_id = instance_id
|
||||||
|
|
||||||
|
|
||||||
class ProxyAutomation(TerraformAutomation):
|
class ProxyAutomation(TerraformAutomation):
|
||||||
subgroup_max = math.inf
|
subgroup_max = math.inf
|
||||||
"""
|
"""
|
||||||
|
@ -35,6 +57,11 @@ class ProxyAutomation(TerraformAutomation):
|
||||||
in the templating of the Terraform configuration.
|
in the templating of the Terraform configuration.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
smart_proxies = False
|
||||||
|
"""
|
||||||
|
Whether this provider supports "smart" proxies.
|
||||||
|
"""
|
||||||
|
|
||||||
def get_subgroups(self) -> Dict[int, Dict[int, int]]:
|
def get_subgroups(self) -> Dict[int, Dict[int, int]]:
|
||||||
conn = db.engine.connect()
|
conn = db.engine.connect()
|
||||||
result = conn.execute(text("""
|
result = conn.execute(text("""
|
||||||
|
@ -118,18 +145,40 @@ class ProxyAutomation(TerraformAutomation):
|
||||||
self.import_state(self.tf_show())
|
self.import_state(self.tf_show())
|
||||||
|
|
||||||
def tf_generate(self) -> None:
|
def tf_generate(self) -> None:
|
||||||
|
groups = Group.query.all()
|
||||||
self.tf_write(
|
self.tf_write(
|
||||||
self.template,
|
self.template,
|
||||||
groups=Group.query.all(),
|
groups=groups,
|
||||||
proxies=Proxy.query.filter(
|
proxies=Proxy.query.filter(
|
||||||
Proxy.provider == self.provider,
|
Proxy.provider == self.provider, Proxy.destroyed.is_(None)).all(), subgroups=self.get_subgroups(),
|
||||||
Proxy.destroyed.is_(None)
|
global_namespace=app.config['GLOBAL_NAMESPACE'], bypass_token=app.config['BYPASS_TOKEN'],
|
||||||
).all(),
|
**{k: app.config[k.upper()] for k in self.template_parameters})
|
||||||
subgroups=self.get_subgroups(),
|
if self.smart_proxies:
|
||||||
global_namespace=app.config['GLOBAL_NAMESPACE'],
|
for group in groups:
|
||||||
bypass_token=app.config['BYPASS_TOKEN'],
|
self.sp_config(group)
|
||||||
**{
|
|
||||||
k: app.config[k.upper()]
|
def sp_config(self, group: Group) -> None:
|
||||||
for k in self.template_parameters
|
group_origins: List[Origin] = Origin.query.filter(
|
||||||
}
|
Origin.group_id == group.id,
|
||||||
)
|
Origin.destroyed.is_(None),
|
||||||
|
Origin.smart.is_(True)
|
||||||
|
).all()
|
||||||
|
self.tmpl_write(f"smart_proxy.{group.id}.conf", """
|
||||||
|
{% for origin in origins %}
|
||||||
|
server {
|
||||||
|
listen 443 ssl;
|
||||||
|
server_name origin-{{ origin.id }}.{{ provider }}.smart.censorship.guide;
|
||||||
|
location / {
|
||||||
|
proxy_set_header Accept-Encoding "";
|
||||||
|
proxy_ssl_server_name on;
|
||||||
|
proxy_pass https://{{ origin.domain_name }}/;
|
||||||
|
subs_filter_types text/html text/css text/xml;
|
||||||
|
subs_filter https://{{ origin.domain_name }}/ /;
|
||||||
|
}
|
||||||
|
ssl_certificate /etc/ssl/smart_proxy.crt;
|
||||||
|
ssl_certificate_key /etc/ssl/private/smart_proxy.key;
|
||||||
|
}
|
||||||
|
{% endfor %}
|
||||||
|
""",
|
||||||
|
provider=self.provider,
|
||||||
|
origins=group_origins)
|
||||||
|
|
|
@ -3,34 +3,66 @@ from typing import Any
|
||||||
|
|
||||||
from app.extensions import db
|
from app.extensions import db
|
||||||
from app.models.mirrors import Proxy
|
from app.models.mirrors import Proxy
|
||||||
from app.terraform.proxy import ProxyAutomation
|
from app.terraform.proxy import ProxyAutomation, update_smart_proxy_instance
|
||||||
|
|
||||||
|
|
||||||
class ProxyCloudfrontAutomation(ProxyAutomation):
|
class ProxyCloudfrontAutomation(ProxyAutomation):
|
||||||
short_name = "proxy_cloudfront"
|
short_name = "proxy_cloudfront"
|
||||||
description = "Deploy proxies to AWS CloudFront"
|
description = "Deploy proxies to AWS CloudFront"
|
||||||
provider = "cloudfront"
|
provider = "cloudfront"
|
||||||
|
smart_proxies = True
|
||||||
|
|
||||||
template_parameters = [
|
template_parameters = [
|
||||||
"aws_access_key",
|
"aws_access_key",
|
||||||
"aws_secret_key"
|
"aws_secret_key",
|
||||||
|
"rfc2136_nameserver",
|
||||||
|
"rfc2136_tsig_key",
|
||||||
|
"rfc2136_tsig_secret",
|
||||||
|
"smart_zone"
|
||||||
]
|
]
|
||||||
|
|
||||||
template = """
|
template = """
|
||||||
terraform {
|
terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
|
acme = {
|
||||||
|
source = "vancluever/acme"
|
||||||
|
version = "~> 2.8.0"
|
||||||
|
}
|
||||||
aws = {
|
aws = {
|
||||||
version = "~> 4.4.0"
|
version = "~> 4.4.0"
|
||||||
}
|
}
|
||||||
|
dns = {
|
||||||
|
version = "~> 3.2.3"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
provider "acme" {
|
||||||
|
server_url = "https://acme-v02.api.letsencrypt.org/directory"
|
||||||
|
}
|
||||||
|
|
||||||
provider "aws" {
|
provider "aws" {
|
||||||
access_key = "{{ aws_access_key }}"
|
access_key = "{{ aws_access_key }}"
|
||||||
secret_key = "{{ aws_secret_key }}"
|
secret_key = "{{ aws_secret_key }}"
|
||||||
region = "us-east-2"
|
region = "us-east-2"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
provider "dns" {
|
||||||
|
update {
|
||||||
|
server = local.rfc2136_nameserver
|
||||||
|
key_name = local.rfc2136_tsig_key
|
||||||
|
key_secret = local.rfc2136_tsig_secret
|
||||||
|
key_algorithm = "hmac-sha512"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
locals {
|
||||||
|
rfc2136_nameserver = "{{ rfc2136_nameserver }}"
|
||||||
|
rfc2136_tsig_key = "{{ rfc2136_tsig_key }}"
|
||||||
|
rfc2136_tsig_secret = "{{ rfc2136_tsig_secret }}"
|
||||||
|
smart_zone = "{{ smart_zone }}"
|
||||||
|
}
|
||||||
|
|
||||||
{% for group in groups %}
|
{% for group in groups %}
|
||||||
module "label_{{ group.id }}" {
|
module "label_{{ group.id }}" {
|
||||||
source = "cloudposse/label/null"
|
source = "cloudposse/label/null"
|
||||||
|
@ -55,13 +87,47 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
|
||||||
resource "aws_sns_topic" "alarms_{{ group.id }}" {
|
resource "aws_sns_topic" "alarms_{{ group.id }}" {
|
||||||
name = "${module.label_{{ group.id }}.id}-cloudfront-alarms"
|
name = "${module.label_{{ group.id }}.id}-cloudfront-alarms"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
{% for origin in group.origins | selectattr("destroyed", "none") | selectattr("smart") %}
|
||||||
|
{% if loop.first %}
|
||||||
|
module "smart_proxy_{{ group.id }}" {
|
||||||
|
source = "sr2c/bc-smart-proxy-instance/aws"
|
||||||
|
version = "0.0.1"
|
||||||
|
context = module.label_{{ group.id }}.context
|
||||||
|
name = "smart-proxy"
|
||||||
|
disable_api_termination = false
|
||||||
|
domain_name = "cloudfront.smart.${local.smart_zone}"
|
||||||
|
rfc2136_nameserver = local.rfc2136_nameserver
|
||||||
|
rfc2136_tsig_key = local.rfc2136_tsig_key
|
||||||
|
rfc2136_tsig_secret = local.rfc2136_tsig_secret
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_object" "smart_config_{{ group.id }}" {
|
||||||
|
bucket = module.smart_proxy_{{ group.id }}.config_bucket_name
|
||||||
|
key = "default"
|
||||||
|
source = "smart_proxy.{{ group.id }}.conf"
|
||||||
|
etag = filemd5("smart_proxy.{{ group.id }}.conf")
|
||||||
|
}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
resource "dns_a_record_set" "smart_dns_{{ origin.id }}" {
|
||||||
|
zone = "{{ smart_zone }}"
|
||||||
|
name = "origin-{{ origin.id }}.cloudfront.smart"
|
||||||
|
addresses = module.smart_proxy_{{ origin.group.id }}.ip_addresses
|
||||||
|
ttl = 60
|
||||||
|
}
|
||||||
|
{% endfor %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
||||||
{% for proxy in proxies %}
|
{% for proxy in proxies %}
|
||||||
module "cloudfront_{{ proxy.id }}" {
|
module "cloudfront_{{ proxy.id }}" {
|
||||||
source = "sr2c/bc-proxy/aws"
|
source = "sr2c/bc-proxy/aws"
|
||||||
version = "0.0.7"
|
version = "0.0.7"
|
||||||
|
{% if proxy.origin.smart %}
|
||||||
|
origin_domain = "origin-{{ proxy.origin.id }}.cloudfront.smart.{{ smart_zone[:-1] }}"
|
||||||
|
{% else %}
|
||||||
origin_domain = "{{ proxy.origin.domain_name }}"
|
origin_domain = "{{ proxy.origin.domain_name }}"
|
||||||
|
{% endif %}
|
||||||
logging_bucket = module.log_bucket_{{ proxy.origin.group.id }}.bucket_domain_name
|
logging_bucket = module.log_bucket_{{ proxy.origin.group.id }}.bucket_domain_name
|
||||||
sns_topic_arn = aws_sns_topic.alarms_{{ proxy.origin.group.id }}.arn
|
sns_topic_arn = aws_sns_topic.alarms_{{ proxy.origin.group.id }}.arn
|
||||||
low_bandwidth_alarm = false
|
low_bandwidth_alarm = false
|
||||||
|
@ -88,4 +154,12 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
|
||||||
proxy.slug = res['values']['id']
|
proxy.slug = res['values']['id']
|
||||||
proxy.terraform_updated = datetime.datetime.utcnow()
|
proxy.terraform_updated = datetime.datetime.utcnow()
|
||||||
break
|
break
|
||||||
|
for g in state["values"]["root_module"]["child_modules"]:
|
||||||
|
if g["address"].startswith("module.smart_proxy_"):
|
||||||
|
group_id = int(g["address"][len("module.smart_proxy_"):])
|
||||||
|
for s in g["child_modules"]:
|
||||||
|
if s["address"].endswith(".module.instance"):
|
||||||
|
for x in s["resources"]:
|
||||||
|
if x["address"].endswith(".module.instance.aws_instance.default[0]"):
|
||||||
|
update_smart_proxy_instance(group_id, self.provider, "us-east-2", x['values']['id'])
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
|
@ -3,8 +3,6 @@ import subprocess # nosec
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from typing import Any, Optional, Tuple
|
from typing import Any, Optional, Tuple
|
||||||
|
|
||||||
import jinja2
|
|
||||||
|
|
||||||
from app.terraform import BaseAutomation
|
from app.terraform import BaseAutomation
|
||||||
|
|
||||||
|
|
||||||
|
@ -151,6 +149,4 @@ class TerraformAutomation(BaseAutomation):
|
||||||
return json.loads(terraform.stdout)
|
return json.loads(terraform.stdout)
|
||||||
|
|
||||||
def tf_write(self, template: str, **kwargs: Any) -> None:
|
def tf_write(self, template: str, **kwargs: Any) -> None:
|
||||||
tmpl = jinja2.Template(template)
|
self.tmpl_write("main.tf", template, **kwargs)
|
||||||
with open(self.working_directory("main.tf"), 'w') as tf:
|
|
||||||
tf.write(tmpl.render(**kwargs))
|
|
||||||
|
|
50
migrations/versions/133961a48525_add_smart_proxies.py
Normal file
50
migrations/versions/133961a48525_add_smart_proxies.py
Normal file
|
@ -0,0 +1,50 @@
|
||||||
|
"""add smart proxies
|
||||||
|
|
||||||
|
Revision ID: 133961a48525
|
||||||
|
Revises: 31aec2f86c40
|
||||||
|
Create Date: 2022-05-24 14:56:43.071054
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '133961a48525'
|
||||||
|
down_revision = '31aec2f86c40'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.create_table('smart_proxy',
|
||||||
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('added', sa.DateTime(), nullable=False),
|
||||||
|
sa.Column('updated', sa.DateTime(), nullable=False),
|
||||||
|
sa.Column('deprecated', sa.DateTime(), nullable=True),
|
||||||
|
sa.Column('deprecation_reason', sa.String(), nullable=True),
|
||||||
|
sa.Column('destroyed', sa.DateTime(), nullable=True),
|
||||||
|
sa.Column('group_id', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('instance_id', sa.String(length=100), nullable=True),
|
||||||
|
sa.Column('provider', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('region', sa.String(length=20), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['group_id'], ['group.id'], name=op.f('fk_smart_proxy_group_id_group')),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_smart_proxy'))
|
||||||
|
)
|
||||||
|
with op.batch_alter_table('origin', schema=None) as batch_op:
|
||||||
|
batch_op.add_column(sa.Column('smart', sa.Boolean(), nullable=True))
|
||||||
|
with op.batch_alter_table('origin', schema=None) as batch_op:
|
||||||
|
batch_op.execute("UPDATE origin SET smart=FALSE")
|
||||||
|
batch_op.alter_column(sa.Column('smart', sa.Boolean(), nullable=False))
|
||||||
|
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
with op.batch_alter_table('origin', schema=None) as batch_op:
|
||||||
|
batch_op.drop_column('smart')
|
||||||
|
|
||||||
|
op.drop_table('smart_proxy')
|
||||||
|
# ### end Alembic commands ###
|
Loading…
Add table
Add a link
Reference in a new issue