mirrors: record deprecation reason

app/models/__init__.py

@@ -15,10 +15,9 @@ class AbstractConfiguration(db.Model):
     def destroy(self):
         self.destroyed = datetime.utcnow()
         self.updated = datetime.utcnow()
-        db.session.commit()
 
     @classmethod
-    def csv_header(self):
+    def csv_header(cls):
         return [
             "id", "description", "added", "updated", "destroyed"
         ]
@@ -36,22 +35,22 @@ class AbstractResource(db.Model):
     added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
     updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
     deprecated = db.Column(db.DateTime(), nullable=True)
+    deprecation_reason = db.Column(db.String(), nullable=True)
     destroyed = db.Column(db.DateTime(), nullable=True)
 
-    def deprecate(self):
+    def deprecate(self, *, reason: str):
         self.deprecated = datetime.utcnow()
+        self.deprecation_reason = reason
         self.updated = datetime.utcnow()
-        db.session.commit()
 
     def destroy(self):
         if self.deprecated is None:
             self.deprecated = datetime.utcnow()
         self.destroyed = datetime.utcnow()
         self.updated = datetime.utcnow()
-        db.session.commit()
 
     @classmethod
-    def csv_header(self):
+    def csv_header(cls):
         return [
             "id", "added", "updated", "deprecated", "destroyed"
         ]

app/models/base.py

@@ -13,7 +13,7 @@ class Group(AbstractConfiguration):
     alarms = db.relationship("Alarm", back_populates="group")
 
     @classmethod
-    def csv_header(self):
+    def csv_header(cls):
         return super().csv_header() + [
             "group_name", "eotk"
         ]
@@ -40,7 +40,7 @@ class MirrorList(AbstractConfiguration):
             return f"s3://{self.container}/{self.filename}"
 
     @classmethod
-    def csv_header(self):
+    def csv_header(cls):
         return super().csv_header() + [
             "provider", "format", "container", "branch", "filename"
         ]

app/models/mirrors.py

@@ -5,6 +5,7 @@ from app.models import AbstractConfiguration, AbstractResource
 class Origin(AbstractConfiguration):
     group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
     domain_name = db.Column(db.String(255), unique=True, nullable=False)
+    auto_rotation = db.Column(db.Boolean, nullable=False)
 
     group = db.relationship("Group", back_populates="origins")
     mirrors = db.relationship("Mirror", back_populates="origin")
@@ -12,7 +13,7 @@ class Origin(AbstractConfiguration):
     alarms = db.relationship("Alarm", back_populates="origin")
 
     @classmethod
-    def csv_header(self):
+    def csv_header(cls):
         return super().csv_header() + [
             "group_id", "domain_name"
         ]
@@ -35,7 +36,7 @@ class Proxy(AbstractResource):
     alarms = db.relationship("Alarm", back_populates="proxy")
 
     @classmethod
-    def csv_header(self):
+    def csv_header(cls):
         return super().csv_header() + [
             "origin_id", "provider", "psg", "slug", "terraform_updated", "url"
         ]
@@ -48,7 +49,7 @@ class Mirror(AbstractResource):
     origin = db.relationship("Origin", back_populates="mirrors")
 
     @classmethod
-    def csv_header(self):
+    def csv_header(cls):
         return super().csv_header() + [
             "origin_id", "url"
         ]

app/portal/__init__.py

@@ -39,7 +39,8 @@ def portal_home():
     last24 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=1))).all())
     last72 = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=3))).all())
     lastweek = len(Proxy.query.filter(Proxy.deprecated > (now - timedelta(days=7))).all())
-    return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72, lastweek=lastweek, proxies=proxies)
+    return render_template("home.html.j2", section="home", groups=groups, last24=last24, last72=last72,
+                           lastweek=lastweek, proxies=proxies)
 
 
 @portal.route("/groups")
@@ -168,7 +169,8 @@ def blocked_proxy(proxy_id):
                                         message="The requested proxy could not be found."))
     form = LifecycleForm()
     if form.validate_on_submit():
-        proxy.deprecate()
+        proxy.deprecate(reason="manual")
+        db.session.commit()
         flash("Proxy will be shortly replaced.", "success")
         return redirect(url_for("portal.edit_origin", origin_id=proxy.origin.id))
     return render_template("lifecycle.html.j2",
@@ -204,6 +206,7 @@ def view_mirror_lists():
 def destroy_mirror_list(list_id):
     return "not implemented"
 
+
 @portal.route("/list/new", methods=['GET', 'POST'])
 @portal.route("/list/new/<group_id>", methods=['GET', 'POST'])
 def new_mirror_list(group_id=None):
@@ -325,14 +328,15 @@ def edit_bridgeconf(bridgeconf_id):
 
 @portal.route("/bridge/block/<bridge_id>", methods=['GET', 'POST'])
 def blocked_bridge(bridge_id):
-    bridge = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first()
+    bridge: Bridge = Bridge.query.filter(Bridge.id == bridge_id, Bridge.destroyed == None).first()
     if bridge is None:
         return Response(render_template("error.html.j2",
                                         header="404 Proxy Not Found",
                                         message="The requested bridge could not be found."))
     form = LifecycleForm()
     if form.validate_on_submit():
-        bridge.deprecate()
+        bridge.deprecate(reason="manual")
+        db.session.commit()
         flash("Bridge will be shortly replaced.", "success")
         return redirect(url_for("portal.edit_bridgeconf", bridgeconf_id=bridge.conf_id))
     return render_template("lifecycle.html.j2",
@@ -344,8 +348,8 @@ def blocked_bridge(bridge_id):
 
 def response_404(message: str):
     return Response(render_template("error.html.j2",
-                             header="404 Not Found",
+                                    header="404 Not Found",
-                             message=message))
+                                    message=message))
 
 
 def view_lifecycle(*,
@@ -361,7 +365,11 @@ def view_lifecycle(*,
         if action == "destroy":
             resource.destroy()
         elif action == "deprecate":
-            resource.deprecate()
+            resource.deprecate(reason="manual")
+        else:
+            flash("Unknown action")
+            return redirect(url_for("portal.portal_home"))
+        db.session.commit()
         flash(success_message, "success")
         return redirect(url_for(success_view))
     return render_template("lifecycle.html.j2",

app/terraform/block_bridge_github.py

@@ -4,6 +4,7 @@ from dateutil.parser import isoparse
 from github import Github
 
 from app import app
+from app.extensions import db
 from app.models.bridges import Bridge
 
 
@@ -17,11 +18,12 @@ def check_blocks():
             if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)):
                 continue
             if int(parts[1]) < 40:
-                bridge = Bridge.query.filter(
+                bridge: Bridge = Bridge.query.filter(
                     Bridge.hashed_fingerprint == parts[0]
                 ).first()
                 if bridge is not None:
-                    bridge.deprecate()
+                    bridge.deprecate(reason="github")
+                    db.session.commit()
 
 
 if __name__ == "__main__":

app/terraform/block_external.py

@@ -2,6 +2,7 @@ from bs4 import BeautifulSoup
 import requests
 
 from app import app
+from app.extensions import db
 from app.models.mirrors import Proxy
 
 
@@ -44,7 +45,7 @@ def check_blocks():
                 if proxy.deprecated:
                     print("Proxy already marked blocked")
                     continue
-                proxy.deprecate()
+                proxy.deprecate(reason="external")
             if "azureedge.net" in url:
                 slug = url[len('https://'):][:-len('.azureedge.net')]
                 print(f"Found {slug} blocked")
@@ -58,7 +59,8 @@ def check_blocks():
                 if proxy.deprecated:
                     print("Proxy already marked blocked")
                     continue
-                proxy.deprecate()
+                proxy.deprecate(reason="external")
+    db.session.commit()
 
 
 if __name__ == "__main__":

app/terraform/bridge/__init__.py

@@ -1,4 +1,5 @@
 import datetime
+from typing import Iterable
 
 from app import app
 from app.extensions import db
@@ -9,8 +10,9 @@ from app.terraform import BaseAutomation
 
 class BridgeAutomation(BaseAutomation):
     def create_missing(self):
-        bridgeconfs = BridgeConf.query.filter(
+        bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter(
-            BridgeConf.provider == self.provider
+            BridgeConf.provider == self.provider,
+            BridgeConf.destroyed == None
         ).all()
         for bridgeconf in bridgeconfs:
             active_bridges = Bridge.query.filter(
@@ -27,10 +29,11 @@ class BridgeAutomation(BaseAutomation):
             elif len(active_bridges) > bridgeconf.number:
                 active_bridge_count = len(active_bridges)
                 for bridge in active_bridges:
-                    bridge.deprecate()
+                    bridge.deprecate("redundant")
                     active_bridge_count -= 1
                     if active_bridge_count == bridgeconf.number:
                         break
+        db.session.commit()
 
     def destroy_expired(self):
         cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=0)
@@ -40,6 +43,7 @@ class BridgeAutomation(BaseAutomation):
         ).all() if b.conf.provider == self.provider]
         for bridge in bridges:
             bridge.destroy()
+        db.session.commit()
 
     def generate_terraform(self):
         self.write_terraform_config(

app/terraform/proxy/__init__.py

@@ -61,7 +61,7 @@ class ProxyAutomation(BaseAutomation):
                 Proxy.provider == self.provider,
                 Proxy.destroyed == None
             ).all(),
-            subgroups = self.get_subgroups(),
+            subgroups=self.get_subgroups(),
             global_namespace=app.config['GLOBAL_NAMESPACE'],
             bypass_token=app.config['BYPASS_TOKEN'],
             **{

migrations/versions/56fbcfa1138c_record_deprecation_reason.py

@@ -0,0 +1,54 @@
+"""record deprecation reason
+
+Revision ID: 56fbcfa1138c
+Revises: 22a33ecf3474
+Create Date: 2022-05-01 16:13:03.425508
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '56fbcfa1138c'
+down_revision = '22a33ecf3474'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('bridge', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True))
+
+    with op.batch_alter_table('mirror', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True))
+
+    with op.batch_alter_table('origin', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('auto_rotation', sa.Boolean(), nullable=True))
+
+    with op.batch_alter_table('origin', schema=None) as batch_op:
+        batch_op.execute("UPDATE origin SET auto_rotation = true")
+        batch_op.alter_column('auto_rotation', nullable=False)
+
+    with op.batch_alter_table('proxy', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('deprecation_reason', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('proxy', schema=None) as batch_op:
+        batch_op.drop_column('deprecation_reason')
+
+    with op.batch_alter_table('origin', schema=None) as batch_op:
+        batch_op.drop_column('auto_rotation')
+
+    with op.batch_alter_table('mirror', schema=None) as batch_op:
+        batch_op.drop_column('deprecation_reason')
+
+    with op.batch_alter_table('bridge', schema=None) as batch_op:
+        batch_op.drop_column('deprecation_reason')
+
+    # ### end Alembic commands ###