jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

mirrors: record deprecation reason

Browse source
This commit is contained in:
Iain Learmonth 2022-05-01 16:23:45 +01:00
parent cacb35a671
commit 18e046dc42
9 changed files with 96 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

11
app/models/__init__.py
View file

@ -15,10 +15,9 @@ class AbstractConfiguration(db.Model):
def destroy(self):
self.destroyed = datetime.utcnow()
self.updated = datetime.utcnow()
db.session.commit()
@classmethod
def csv_header(self):
def csv_header(cls):
return [
"id", "description", "added", "updated", "destroyed"
]
@ -36,22 +35,22 @@ class AbstractResource(db.Model):
added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
deprecated = db.Column(db.DateTime(), nullable=True)
deprecation_reason = db.Column(db.String(), nullable=True)
destroyed = db.Column(db.DateTime(), nullable=True)
def deprecate(self):
def deprecate(self, *, reason: str):
self.deprecated = datetime.utcnow()
self.deprecation_reason = reason
self.updated = datetime.utcnow()
db.session.commit()
def destroy(self):
if self.deprecated is None:
self.deprecated = datetime.utcnow()
self.destroyed = datetime.utcnow()
self.updated = datetime.utcnow()
db.session.commit()
@classmethod
def csv_header(self):
def csv_header(cls):
return [
"id", "added", "updated", "deprecated", "destroyed"
]

4
app/models/base.py
View file

@ -13,7 +13,7 @@ class Group(AbstractConfiguration):
alarms = db.relationship("Alarm", back_populates="group")
@classmethod
def csv_header(self):
def csv_header(cls):
return super().csv_header() + [
"group_name", "eotk"
]
@ -40,7 +40,7 @@ class MirrorList(AbstractConfiguration):
return f"s3://{self.container}/{self.filename}"
@classmethod
def csv_header(self):
def csv_header(cls):
return super().csv_header() + [
"provider", "format", "container", "branch", "filename"
]

7
app/models/mirrors.py
View file

@ -5,6 +5,7 @@ from app.models import AbstractConfiguration, AbstractResource
class Origin(AbstractConfiguration):
group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
domain_name = db.Column(db.String(255), unique=True, nullable=False)
auto_rotation = db.Column(db.Boolean, nullable=False)
group = db.relationship("Group", back_populates="origins")
mirrors = db.relationship("Mirror", back_populates="origin")
@ -12,7 +13,7 @@ class Origin(AbstractConfiguration):
alarms = db.relationship("Alarm", back_populates="origin")
@classmethod
def csv_header(self):
def csv_header(cls):
return super().csv_header() + [
"group_id", "domain_name"
]
@ -35,7 +36,7 @@ class Proxy(AbstractResource):
alarms = db.relationship("Alarm", back_populates="proxy")
@classmethod
def csv_header(self):
def csv_header(cls):
return super().csv_header() + [
"origin_id", "provider", "psg", "slug", "terraform_updated", "url"
]
@ -48,7 +49,7 @@ class Mirror(AbstractResource):
origin = db.relationship("Origin", back_populates="mirrors")
@classmethod
def csv_header(self):
def csv_header(cls):
return super().csv_header() + [
"origin_id", "url"
]

22
app/portal/__init__.py
View file

@ -39,7 +39,8 @@ def portal_home():
last24 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=1))).all())
last72 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=3))).all())
lastweek = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=7))).all())
return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72, lastweek=lastweek, proxies=proxies)
return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72,
lastweek=lastweek, proxies=proxies)
@portal.route("/groups")
@ -168,7 +169,8 @@ def blocked_proxy(proxy_id):
message="The requested proxy could not be found."))
form = LifecycleForm()
if form.validate_on_submit():
proxy.deprecate()
proxy.deprecate(reason="manual")
db.session.commit()
flash("Proxy will be shortly replaced.", "success")
return redirect(url_for("portal.edit_origin", origin_id=proxy.origin.id))
return render_template("lifecycle.html.j2",
@ -204,6 +206,7 @@ def view_mirror_lists():
def destroy_mirror_list(list_id):
return "not implemented"
@portal.route("/list/new", methods=['GET', 'POST'])
@portal.route("/list/new/<group_id>", methods=['GET', 'POST'])
def new_mirror_list(group_id=None):
@ -325,14 +328,15 @@ def edit_bridgeconf(bridgeconf_id):
@portal.route("/bridge/block/<bridge_id>", methods=['GET', 'POST'])
def blocked_bridge(bridge_id):
bridge = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first()
bridge: Bridge = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first()
if bridge is None:
return Response(render_template("error.html.j2",
header="404 Proxy Not Found",
message="The requested bridge could not be found."))
form = LifecycleForm()
if form.validate_on_submit():
bridge.deprecate()
bridge.deprecate(reason="manual")
db.session.commit()
flash("Bridge will be shortly replaced.", "success")
return redirect(url_for("portal.edit_bridgeconf", bridgeconf_id=bridge.conf_id))
return render_template("lifecycle.html.j2",
@ -344,8 +348,8 @@ def blocked_bridge(bridge_id):
def response_404(message: str):
return Response(render_template("error.html.j2",
header="404 Not Found",
message=message))
header="404 Not Found",
message=message))
def view_lifecycle(*,
@ -361,7 +365,11 @@ def view_lifecycle(*,
if action == "destroy":
resource.destroy()
elif action == "deprecate":
resource.deprecate()
resource.deprecate(reason="manual")
else:
flash("Unknown action")
return redirect(url_for("portal.portal_home"))
db.session.commit()
flash(success_message, "success")
return redirect(url_for(success_view))
return render_template("lifecycle.html.j2",

6
app/terraform/block_bridge_github.py
View file

@ -4,6 +4,7 @@ from dateutil.parser import isoparse
from github import Github
from app import app
from app.extensions import db
from app.models.bridges import Bridge
@ -17,11 +18,12 @@ def check_blocks():
if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)):
continue
if int(parts[1]) < 40:
bridge = Bridge.query.filter(
bridge: Bridge = Bridge.query.filter(
Bridge.hashed_fingerprint == parts[0]
).first()
if bridge is not None:
bridge.deprecate()
bridge.deprecate(reason="github")
db.session.commit()
if __name__ == "__main__":

6
app/terraform/block_external.py
View file

@ -2,6 +2,7 @@ from bs4 import BeautifulSoup
import requests
from app import app
from app.extensions import db
from app.models.mirrors import Proxy
@ -44,7 +45,7 @@ def check_blocks():
if proxy.deprecated:
print("Proxy already marked blocked")
continue
proxy.deprecate()
proxy.deprecate(reason="external")
if "azureedge.net" in url:
slug = url[len('https://'):][:-len('.azureedge.net')]
print(f"Found {slug} blocked")
@ -58,7 +59,8 @@ def check_blocks():
if proxy.deprecated:
print("Proxy already marked blocked")
continue
proxy.deprecate()
proxy.deprecate(reason="external")
db.session.commit()
if __name__ == "__main__":

10
app/terraform/bridge/__init__.py
View file

@ -1,4 +1,5 @@
import datetime
from typing import Iterable
from app import app
from app.extensions import db
@ -9,8 +10,9 @@ from app.terraform import BaseAutomation
class BridgeAutomation(BaseAutomation):
def create_missing(self):
bridgeconfs = BridgeConf.query.filter(
BridgeConf.provider == self.provider
bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter(
BridgeConf.provider == self.provider,
BridgeConf.destroyed == None
).all()
for bridgeconf in bridgeconfs:
active_bridges = Bridge.query.filter(
@ -27,10 +29,11 @@ class BridgeAutomation(BaseAutomation):
elif len(active_bridges) > bridgeconf.number:
active_bridge_count = len(active_bridges)
for bridge in active_bridges:
bridge.deprecate()
bridge.deprecate("redundant")
active_bridge_count -= 1
if active_bridge_count == bridgeconf.number:
break
db.session.commit()
def destroy_expired(self):
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=0)
@ -40,6 +43,7 @@ class BridgeAutomation(BaseAutomation):
).all() if b.conf.provider == self.provider]
for bridge in bridges:
bridge.destroy()
db.session.commit()
def generate_terraform(self):
self.write_terraform_config(

2
app/terraform/proxy/__init__.py
View file

@ -61,7 +61,7 @@ class ProxyAutomation(BaseAutomation):
Proxy.provider == self.provider,
Proxy.destroyed == None
).all(),
subgroups = self.get_subgroups(),
subgroups=self.get_subgroups(),
global_namespace=app.config['GLOBAL_NAMESPACE'],
bypass_token=app.config['BYPASS_TOKEN'],
**{

54
migrations/versions/56fbcfa1138c_record_deprecation_reason.py Normal file
View file

@ -0,0 +1,54 @@
"""record deprecation reason
Revision ID: 56fbcfa1138c
Revises: 22a33ecf3474
Create Date: 2022-05-01 16:13:03.425508
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '56fbcfa1138c'
down_revision = '22a33ecf3474'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('bridge', schema=None) as batch_op:
batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True))
with op.batch_alter_table('mirror', schema=None) as batch_op:
batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True))
with op.batch_alter_table('origin', schema=None) as batch_op:
batch_op.add_column(sa.Column('auto_rotation', sa.Boolean(), nullable=True))
with op.batch_alter_table('origin', schema=None) as batch_op:
batch_op.execute("UPDATE origin SET auto_rotation = true")
batch_op.alter_column('auto_rotation', nullable=False)
with op.batch_alter_table('proxy', schema=None) as batch_op:
batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('proxy', schema=None) as batch_op:
batch_op.drop_column('deprecation_reason')
with op.batch_alter_table('origin', schema=None) as batch_op:
batch_op.drop_column('auto_rotation')
with op.batch_alter_table('mirror', schema=None) as batch_op:
batch_op.drop_column('deprecation_reason')
with op.batch_alter_table('bridge', schema=None) as batch_op:
batch_op.drop_column('deprecation_reason')
# ### end Alembic commands ###