diff --git a/app/models/__init__.py b/app/models/__init__.py index 360266b..4c70c2e 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -15,10 +15,9 @@ class AbstractConfiguration(db.Model): def destroy(self): self.destroyed = datetime.utcnow() self.updated = datetime.utcnow() - db.session.commit() @classmethod - def csv_header(self): + def csv_header(cls): return [ "id", "description", "added", "updated", "destroyed" ] @@ -36,22 +35,22 @@ class AbstractResource(db.Model): added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False) updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False) deprecated = db.Column(db.DateTime(), nullable=True) + deprecation_reason = db.Column(db.String(), nullable=True) destroyed = db.Column(db.DateTime(), nullable=True) - def deprecate(self): + def deprecate(self, *, reason: str): self.deprecated = datetime.utcnow() + self.deprecation_reason = reason self.updated = datetime.utcnow() - db.session.commit() def destroy(self): if self.deprecated is None: self.deprecated = datetime.utcnow() self.destroyed = datetime.utcnow() self.updated = datetime.utcnow() - db.session.commit() @classmethod - def csv_header(self): + def csv_header(cls): return [ "id", "added", "updated", "deprecated", "destroyed" ] diff --git a/app/models/base.py b/app/models/base.py index 2e95f93..ff8fcfe 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -13,7 +13,7 @@ class Group(AbstractConfiguration): alarms = db.relationship("Alarm", back_populates="group") @classmethod - def csv_header(self): + def csv_header(cls): return super().csv_header() + [ "group_name", "eotk" ] @@ -40,7 +40,7 @@ class MirrorList(AbstractConfiguration): return f"s3://{self.container}/{self.filename}" @classmethod - def csv_header(self): + def csv_header(cls): return super().csv_header() + [ "provider", "format", "container", "branch", "filename" ] diff --git a/app/models/mirrors.py b/app/models/mirrors.py index e993b6c..d7a9f3d 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -5,6 +5,7 @@ from app.models import AbstractConfiguration, AbstractResource class Origin(AbstractConfiguration): group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False) domain_name = db.Column(db.String(255), unique=True, nullable=False) + auto_rotation = db.Column(db.Boolean, nullable=False) group = db.relationship("Group", back_populates="origins") mirrors = db.relationship("Mirror", back_populates="origin") @@ -12,7 +13,7 @@ class Origin(AbstractConfiguration): alarms = db.relationship("Alarm", back_populates="origin") @classmethod - def csv_header(self): + def csv_header(cls): return super().csv_header() + [ "group_id", "domain_name" ] @@ -35,7 +36,7 @@ class Proxy(AbstractResource): alarms = db.relationship("Alarm", back_populates="proxy") @classmethod - def csv_header(self): + def csv_header(cls): return super().csv_header() + [ "origin_id", "provider", "psg", "slug", "terraform_updated", "url" ] @@ -48,7 +49,7 @@ class Mirror(AbstractResource): origin = db.relationship("Origin", back_populates="mirrors") @classmethod - def csv_header(self): + def csv_header(cls): return super().csv_header() + [ "origin_id", "url" ] diff --git a/app/portal/__init__.py b/app/portal/__init__.py index 115b6c8..2e025b2 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -39,7 +39,8 @@ def portal_home(): last24 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=1))).all()) last72 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=3))).all()) lastweek = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=7))).all()) - return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72, lastweek=lastweek, proxies=proxies) + return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72, + lastweek=lastweek, proxies=proxies) @portal.route("/groups") @@ -168,7 +169,8 @@ def blocked_proxy(proxy_id): message="The requested proxy could not be found.")) form = LifecycleForm() if form.validate_on_submit(): - proxy.deprecate() + proxy.deprecate(reason="manual") + db.session.commit() flash("Proxy will be shortly replaced.", "success") return redirect(url_for("portal.edit_origin", origin_id=proxy.origin.id)) return render_template("lifecycle.html.j2", @@ -204,6 +206,7 @@ def view_mirror_lists(): def destroy_mirror_list(list_id): return "not implemented" + @portal.route("/list/new", methods=['GET', 'POST']) @portal.route("/list/new/", methods=['GET', 'POST']) def new_mirror_list(group_id=None): @@ -325,14 +328,15 @@ def edit_bridgeconf(bridgeconf_id): @portal.route("/bridge/block/", methods=['GET', 'POST']) def blocked_bridge(bridge_id): - bridge = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first() + bridge: Bridge = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first() if bridge is None: return Response(render_template("error.html.j2", header="404 Proxy Not Found", message="The requested bridge could not be found.")) form = LifecycleForm() if form.validate_on_submit(): - bridge.deprecate() + bridge.deprecate(reason="manual") + db.session.commit() flash("Bridge will be shortly replaced.", "success") return redirect(url_for("portal.edit_bridgeconf", bridgeconf_id=bridge.conf_id)) return render_template("lifecycle.html.j2", @@ -344,8 +348,8 @@ def blocked_bridge(bridge_id): def response_404(message: str): return Response(render_template("error.html.j2", - header="404 Not Found", - message=message)) + header="404 Not Found", + message=message)) def view_lifecycle(*, @@ -361,7 +365,11 @@ def view_lifecycle(*, if action == "destroy": resource.destroy() elif action == "deprecate": - resource.deprecate() + resource.deprecate(reason="manual") + else: + flash("Unknown action") + return redirect(url_for("portal.portal_home")) + db.session.commit() flash(success_message, "success") return redirect(url_for(success_view)) return render_template("lifecycle.html.j2", diff --git a/app/terraform/block_bridge_github.py b/app/terraform/block_bridge_github.py index a9bc98d..dc61fd9 100644 --- a/app/terraform/block_bridge_github.py +++ b/app/terraform/block_bridge_github.py @@ -4,6 +4,7 @@ from dateutil.parser import isoparse from github import Github from app import app +from app.extensions import db from app.models.bridges import Bridge @@ -17,11 +18,12 @@ def check_blocks(): if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)): continue if int(parts[1]) < 40: - bridge = Bridge.query.filter( + bridge: Bridge = Bridge.query.filter( Bridge.hashed_fingerprint == parts[0] ).first() if bridge is not None: - bridge.deprecate() + bridge.deprecate(reason="github") + db.session.commit() if __name__ == "__main__": diff --git a/app/terraform/block_external.py b/app/terraform/block_external.py index 26aa3aa..be32510 100644 --- a/app/terraform/block_external.py +++ b/app/terraform/block_external.py @@ -2,6 +2,7 @@ from bs4 import BeautifulSoup import requests from app import app +from app.extensions import db from app.models.mirrors import Proxy @@ -44,7 +45,7 @@ def check_blocks(): if proxy.deprecated: print("Proxy already marked blocked") continue - proxy.deprecate() + proxy.deprecate(reason="external") if "azureedge.net" in url: slug = url[len('https://'):][:-len('.azureedge.net')] print(f"Found {slug} blocked") @@ -58,7 +59,8 @@ def check_blocks(): if proxy.deprecated: print("Proxy already marked blocked") continue - proxy.deprecate() + proxy.deprecate(reason="external") + db.session.commit() if __name__ == "__main__": diff --git a/app/terraform/bridge/__init__.py b/app/terraform/bridge/__init__.py index 6d579cf..d22f9f2 100644 --- a/app/terraform/bridge/__init__.py +++ b/app/terraform/bridge/__init__.py @@ -1,4 +1,5 @@ import datetime +from typing import Iterable from app import app from app.extensions import db @@ -9,8 +10,9 @@ from app.terraform import BaseAutomation class BridgeAutomation(BaseAutomation): def create_missing(self): - bridgeconfs = BridgeConf.query.filter( - BridgeConf.provider == self.provider + bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter( + BridgeConf.provider == self.provider, + BridgeConf.destroyed == None ).all() for bridgeconf in bridgeconfs: active_bridges = Bridge.query.filter( @@ -27,10 +29,11 @@ class BridgeAutomation(BaseAutomation): elif len(active_bridges) > bridgeconf.number: active_bridge_count = len(active_bridges) for bridge in active_bridges: - bridge.deprecate() + bridge.deprecate("redundant") active_bridge_count -= 1 if active_bridge_count == bridgeconf.number: break + db.session.commit() def destroy_expired(self): cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=0) @@ -40,6 +43,7 @@ class BridgeAutomation(BaseAutomation): ).all() if b.conf.provider == self.provider] for bridge in bridges: bridge.destroy() + db.session.commit() def generate_terraform(self): self.write_terraform_config( diff --git a/app/terraform/proxy/__init__.py b/app/terraform/proxy/__init__.py index 55ecc3e..9365c0e 100644 --- a/app/terraform/proxy/__init__.py +++ b/app/terraform/proxy/__init__.py @@ -61,7 +61,7 @@ class ProxyAutomation(BaseAutomation): Proxy.provider == self.provider, Proxy.destroyed == None ).all(), - subgroups = self.get_subgroups(), + subgroups=self.get_subgroups(), global_namespace=app.config['GLOBAL_NAMESPACE'], bypass_token=app.config['BYPASS_TOKEN'], **{ diff --git a/migrations/versions/56fbcfa1138c_record_deprecation_reason.py b/migrations/versions/56fbcfa1138c_record_deprecation_reason.py new file mode 100644 index 0000000..577a8b2 --- /dev/null +++ b/migrations/versions/56fbcfa1138c_record_deprecation_reason.py @@ -0,0 +1,54 @@ +"""record deprecation reason + +Revision ID: 56fbcfa1138c +Revises: 22a33ecf3474 +Create Date: 2022-05-01 16:13:03.425508 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '56fbcfa1138c' +down_revision = '22a33ecf3474' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('bridge', schema=None) as batch_op: + batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True)) + + with op.batch_alter_table('mirror', schema=None) as batch_op: + batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True)) + + with op.batch_alter_table('origin', schema=None) as batch_op: + batch_op.add_column(sa.Column('auto_rotation', sa.Boolean(), nullable=True)) + + with op.batch_alter_table('origin', schema=None) as batch_op: + batch_op.execute("UPDATE origin SET auto_rotation = true") + batch_op.alter_column('auto_rotation', nullable=False) + + with op.batch_alter_table('proxy', schema=None) as batch_op: + batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('proxy', schema=None) as batch_op: + batch_op.drop_column('deprecation_reason') + + with op.batch_alter_table('origin', schema=None) as batch_op: + batch_op.drop_column('auto_rotation') + + with op.batch_alter_table('mirror', schema=None) as batch_op: + batch_op.drop_column('deprecation_reason') + + with op.batch_alter_table('bridge', schema=None) as batch_op: + batch_op.drop_column('deprecation_reason') + + # ### end Alembic commands ###