models: big refactor

app/__init__.py

@@ -6,7 +6,8 @@ from app.extensions import db
 from app.extensions import migrate
 from app.extensions import bootstrap
 from app.mirror_sites import mirror_sites
-from app.models import Group, Origin, Proxy, Mirror
+from app.models.mirrors import Origin, Proxy, Mirror
+from app.models.base import Group
 from app.portal import portal
 
 app = Flask(__name__)

app/alarms.py

@@ -1,7 +1,7 @@
 import datetime
 
 from app.extensions import db
-from app.models import Alarm
+from app.models.alarms import Alarm
 
 
 def _get_alarm(target: str,

app/cli/db.py

@@ -6,7 +6,10 @@ import sys
 
 from app import app
 from app.extensions import db
-from app.models import Group, Origin, Proxy, BridgeConf, Alarm
+from app.models.base import Group
+from app.models.bridges import BridgeConf
+from app.models.mirrors import Origin, Proxy
+from app.models.alarms import Alarm
 
 models = {
     "group": Group,

app/mirror_sites.py

@@ -1,6 +1,7 @@
 from tldextract import extract
 
-from app.models import Origin, Bridge, Proxy
+from app.models.bridges import Bridge
+from app.models.mirrors import Origin, Proxy
 
 
 def mirror_sites():

app/models.py

@@ -1,250 +0,0 @@
-import enum
-from datetime import datetime
-
-from app.extensions import db
-
-
-class AbstractConfiguration(db.Model):
-    __abstract__ = True
-
-    id = db.Column(db.Integer, primary_key=True)
-    description = db.Column(db.String(255), nullable=False)
-    added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    destroyed = db.Column(db.DateTime(), nullable=True)
-
-    def destroy(self):
-        self.destroyed = datetime.utcnow()
-        self.updated = datetime.utcnow()
-        db.session.commit()
-
-    @classmethod
-    def csv_header(self):
-        return [
-            "id", "description", "added", "updated", "destroyed"
-        ]
-
-    def csv_row(self):
-        return [
-            getattr(self, x) for x in self.csv_header()
-        ]
-
-
-class AbstractResource(db.Model):
-    __abstract__ = True
-
-    id = db.Column(db.Integer, primary_key=True)
-    added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    deprecated = db.Column(db.DateTime(), nullable=True)
-    destroyed = db.Column(db.DateTime(), nullable=True)
-
-    def deprecate(self):
-        self.deprecated = datetime.utcnow()
-        self.updated = datetime.utcnow()
-        db.session.commit()
-
-    def destroy(self):
-        if self.deprecated is None:
-            self.deprecated = datetime.utcnow()
-        self.destroyed = datetime.utcnow()
-        self.updated = datetime.utcnow()
-        db.session.commit()
-
-    @classmethod
-    def csv_header(self):
-        return [
-            "id", "added", "updated", "deprecated", "destroyed"
-        ]
-
-    def csv_row(self):
-        return [
-            getattr(self, x) for x in self.csv_header()
-        ]
-
-    def __repr__(self):
-        return f"<{self.__class__.__name__} #{self.id}>"
-
-
-class Group(AbstractConfiguration):
-    group_name = db.Column(db.String(80), unique=True, nullable=False)
-    eotk = db.Column(db.Boolean())
-
-    origins = db.relationship("Origin", back_populates="group")
-    bridgeconfs = db.relationship("BridgeConf", back_populates="group")
-    alarms = db.relationship("Alarm", back_populates="group")
-
-    @classmethod
-    def csv_header(self):
-        return super().csv_header() + [
-            "group_name", "eotk"
-        ]
-
-    def __repr__(self):
-        return '<Group %r>' % self.group_name
-
-
-class Origin(AbstractConfiguration):
-    group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
-    domain_name = db.Column(db.String(255), unique=True, nullable=False)
-
-    group = db.relationship("Group", back_populates="origins")
-    mirrors = db.relationship("Mirror", back_populates="origin")
-    proxies = db.relationship("Proxy", back_populates="origin")
-    alarms = db.relationship("Alarm", back_populates="origin")
-
-    @classmethod
-    def csv_header(self):
-        return super().csv_header() + [
-            "group_id", "domain_name"
-        ]
-
-    def destroy(self):
-        super().destroy()
-        for proxy in self.proxies:
-            proxy.destroy()
-
-    def __repr__(self):
-        return '<Origin %r>' % self.domain_name
-
-
-class Proxy(AbstractResource):
-    origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)
-    provider = db.Column(db.String(20), nullable=False)
-    slug = db.Column(db.String(20), nullable=True)
-    terraform_updated = db.Column(db.DateTime(), nullable=True)
-    url = db.Column(db.String(255), nullable=True)
-
-    origin = db.relationship("Origin", back_populates="proxies")
-    alarms = db.relationship("Alarm", back_populates="proxy")
-
-    @classmethod
-    def csv_header(self):
-        return super().csv_header() + [
-            "origin_id", "provider", "slug", "terraform_updated", "url"
-        ]
-
-
-class Mirror(AbstractResource):
-    origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)
-    url = db.Column(db.String(255), unique=True, nullable=False)
-
-    origin = db.relationship("Origin", back_populates="mirrors")
-
-    @classmethod
-    def csv_header(self):
-        return super().csv_header() + [
-            "origin_id", "url"
-        ]
-
-    def __repr__(self):
-        return '<Mirror %r_%r>' % (self.origin.domain_name, self.id)
-
-
-class AlarmState(enum.Enum):
-    UNKNOWN = 0
-    OK = 1
-    WARNING = 2
-    CRITICAL = 3
-
-
-class Alarm(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    target = db.Column(db.String(60), nullable=False)
-    group_id = db.Column(db.Integer, db.ForeignKey("group.id"))
-    origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"))
-    proxy_id = db.Column(db.Integer, db.ForeignKey("proxy.id"))
-    bridge_id = db.Column(db.Integer, db.ForeignKey("bridge.id"))
-    alarm_type = db.Column(db.String(255), nullable=False)
-    alarm_state = db.Column(db.Enum(AlarmState), default=AlarmState.UNKNOWN, nullable=False)
-    state_changed = db.Column(db.DateTime(), nullable=False)
-    last_updated = db.Column(db.DateTime())
-    text = db.Column(db.String(255))
-
-    group = db.relationship("Group", back_populates="alarms")
-    origin = db.relationship("Origin", back_populates="alarms")
-    proxy = db.relationship("Proxy", back_populates="alarms")
-    bridge = db.relationship("Bridge", back_populates="alarms")
-
-    @classmethod
-    def csv_header(cls):
-        return [
-            "id", "target", "group_id", "origin_id", "proxy_id", "bridge_id", "alarm_type",
-            "alarm_state", "state_changed", "last_updated", "text"
-        ]
-
-    def csv_row(self):
-        return [
-            self[x] for x in self.csv_header()
-        ]
-
-    def update_state(self, state: AlarmState, text: str):
-        if self.alarm_state != state or self.state_changed is None:
-            self.state_changed = datetime.utcnow()
-        self.alarm_state = state
-        self.text = text
-        self.last_updated = datetime.utcnow()
-        db.session.commit()
-
-
-class BridgeConf(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
-    provider = db.Column(db.String(20), nullable=False)
-    method = db.Column(db.String(20), nullable=False)
-    description = db.Column(db.String(255))
-    number = db.Column(db.Integer())
-    added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    destroyed = db.Column(db.DateTime(), nullable=True)
-
-    group = db.relationship("Group", back_populates="bridgeconfs")
-    bridges = db.relationship("Bridge", back_populates="conf")
-
-    def destroy(self):
-        self.destroyed = datetime.utcnow()
-        self.updated = datetime.utcnow()
-        for bridge in self.bridges:
-            if bridge.destroyed is None:
-                bridge.destroyed = datetime.utcnow()
-                bridge.updated = datetime.utcnow()
-        db.session.commit()
-
-
-class Bridge(AbstractResource):
-    conf_id = db.Column(db.Integer, db.ForeignKey("bridge_conf.id"), nullable=False)
-    terraform_updated = db.Column(db.DateTime(), nullable=True)
-    nickname = db.Column(db.String(255), nullable=True)
-    fingerprint = db.Column(db.String(255), nullable=True)
-    hashed_fingerprint = db.Column(db.String(255), nullable=True)
-    bridgeline = db.Column(db.String(255), nullable=True)
-
-    conf = db.relationship("BridgeConf", back_populates="bridges")
-    alarms = db.relationship("Alarm", back_populates="bridge")
-
-
-class MirrorList(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    provider = db.Column(db.String(255), nullable=False)
-    description = db.Column(db.String(255), nullable=False)
-    format = db.Column(db.String(20), nullable=False)
-    container = db.Column(db.String(255), nullable=False)
-    branch = db.Column(db.String(255), nullable=False)
-    filename = db.Column(db.String(255), nullable=False)
-    added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
-    deprecated = db.Column(db.DateTime(), nullable=True)
-    destroyed = db.Column(db.DateTime(), nullable=True)
-
-    def destroy(self):
-        self.destroyed = datetime.utcnow()
-        self.updated = datetime.utcnow()
-        db.session.commit()
-
-    def url(self):
-        if self.provider == "gitlab":
-            return f"https://gitlab.com/{self.container}/-/raw/{self.branch}/{self.filename}"
-        if self.provider == "github":
-            return f"https://raw.githubusercontent.com/{self.container}/{self.branch}/{self.filename}"
-        if self.provider == "s3":
-            return f"s3://{self.container}/{self.filename}"

app/models/__init__.py

@@ -0,0 +1,62 @@
+from datetime import datetime
+
+from app.extensions import db
+
+
+class AbstractConfiguration(db.Model):
+    __abstract__ = True
+
+    id = db.Column(db.Integer, primary_key=True)
+    description = db.Column(db.String(255), nullable=False)
+    added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
+    updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
+    destroyed = db.Column(db.DateTime(), nullable=True)
+
+    def destroy(self):
+        self.destroyed = datetime.utcnow()
+        self.updated = datetime.utcnow()
+        db.session.commit()
+
+    @classmethod
+    def csv_header(self):
+        return [
+            "id", "description", "added", "updated", "destroyed"
+        ]
+
+    def csv_row(self):
+        return [
+            getattr(self, x) for x in self.csv_header()
+        ]
+
+
+class AbstractResource(db.Model):
+    __abstract__ = True
+
+    id = db.Column(db.Integer, primary_key=True)
+    added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
+    updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
+    deprecated = db.Column(db.DateTime(), nullable=True)
+    destroyed = db.Column(db.DateTime(), nullable=True)
+
+    def deprecate(self):
+        self.deprecated = datetime.utcnow()
+        self.updated = datetime.utcnow()
+        db.session.commit()
+
+    def destroy(self):
+        if self.deprecated is None:
+            self.deprecated = datetime.utcnow()
+        self.destroyed = datetime.utcnow()
+        self.updated = datetime.utcnow()
+        db.session.commit()
+
+    @classmethod
+    def csv_header(self):
+        return [
+            "id", "added", "updated", "deprecated", "destroyed"
+        ]
+
+    def csv_row(self):
+        return [
+            getattr(self, x) for x in self.csv_header()
+        ]

app/models/alarms.py

@@ -0,0 +1,50 @@
+import enum
+from datetime import datetime
+
+from app import db
+
+
+class AlarmState(enum.Enum):
+    UNKNOWN = 0
+    OK = 1
+    WARNING = 2
+    CRITICAL = 3
+
+
+class Alarm(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    target = db.Column(db.String(60), nullable=False)
+    group_id = db.Column(db.Integer, db.ForeignKey("group.id"))
+    origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"))
+    proxy_id = db.Column(db.Integer, db.ForeignKey("proxy.id"))
+    bridge_id = db.Column(db.Integer, db.ForeignKey("bridge.id"))
+    alarm_type = db.Column(db.String(255), nullable=False)
+    alarm_state = db.Column(db.Enum(AlarmState), default=AlarmState.UNKNOWN, nullable=False)
+    state_changed = db.Column(db.DateTime(), nullable=False)
+    last_updated = db.Column(db.DateTime())
+    text = db.Column(db.String(255))
+
+    group = db.relationship("Group", back_populates="alarms")
+    origin = db.relationship("Origin", back_populates="alarms")
+    proxy = db.relationship("Proxy", back_populates="alarms")
+    bridge = db.relationship("Bridge", back_populates="alarms")
+
+    @classmethod
+    def csv_header(cls):
+        return [
+            "id", "target", "group_id", "origin_id", "proxy_id", "bridge_id", "alarm_type",
+            "alarm_state", "state_changed", "last_updated", "text"
+        ]
+
+    def csv_row(self):
+        return [
+            self[x] for x in self.csv_header()
+        ]
+
+    def update_state(self, state: AlarmState, text: str):
+        if self.alarm_state != state or self.state_changed is None:
+            self.state_changed = datetime.utcnow()
+        self.alarm_state = state
+        self.text = text
+        self.last_updated = datetime.utcnow()
+        db.session.commit()

app/models/base.py

@@ -0,0 +1,46 @@
+from datetime import datetime
+
+from app import db
+from app.models import AbstractConfiguration
+
+
+class Group(AbstractConfiguration):
+    group_name = db.Column(db.String(80), unique=True, nullable=False)
+    eotk = db.Column(db.Boolean())
+
+    origins = db.relationship("Origin", back_populates="group")
+    bridgeconfs = db.relationship("BridgeConf", back_populates="group")
+    alarms = db.relationship("Alarm", back_populates="group")
+
+    @classmethod
+    def csv_header(self):
+        return super().csv_header() + [
+            "group_name", "eotk"
+        ]
+
+
+class MirrorList(AbstractConfiguration):
+    provider = db.Column(db.String(255), nullable=False)
+    format = db.Column(db.String(20), nullable=False)
+    container = db.Column(db.String(255), nullable=False)
+    branch = db.Column(db.String(255), nullable=False)
+    filename = db.Column(db.String(255), nullable=False)
+
+    def destroy(self):
+        self.destroyed = datetime.utcnow()
+        self.updated = datetime.utcnow()
+        db.session.commit()
+
+    def url(self):
+        if self.provider == "gitlab":
+            return f"https://gitlab.com/{self.container}/-/raw/{self.branch}/{self.filename}"
+        if self.provider == "github":
+            return f"https://raw.githubusercontent.com/{self.container}/{self.branch}/{self.filename}"
+        if self.provider == "s3":
+            return f"s3://{self.container}/{self.filename}"
+
+    @classmethod
+    def csv_header(self):
+        return super().csv_header() + [
+            "provider", "format", "container", "branch", "filename"
+        ]

app/models/bridges.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+
+from app import db
+from app.models import AbstractResource
+
+
+class BridgeConf(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
+    provider = db.Column(db.String(20), nullable=False)
+    method = db.Column(db.String(20), nullable=False)
+    description = db.Column(db.String(255))
+    number = db.Column(db.Integer())
+    added = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
+    updated = db.Column(db.DateTime(), default=datetime.utcnow, nullable=False)
+    destroyed = db.Column(db.DateTime(), nullable=True)
+
+    group = db.relationship("Group", back_populates="bridgeconfs")
+    bridges = db.relationship("Bridge", back_populates="conf")
+
+    def destroy(self):
+        self.destroyed = datetime.utcnow()
+        self.updated = datetime.utcnow()
+        for bridge in self.bridges:
+            if bridge.destroyed is None:
+                bridge.destroyed = datetime.utcnow()
+                bridge.updated = datetime.utcnow()
+        db.session.commit()
+
+
+class Bridge(AbstractResource):
+    conf_id = db.Column(db.Integer, db.ForeignKey("bridge_conf.id"), nullable=False)
+    terraform_updated = db.Column(db.DateTime(), nullable=True)
+    nickname = db.Column(db.String(255), nullable=True)
+    fingerprint = db.Column(db.String(255), nullable=True)
+    hashed_fingerprint = db.Column(db.String(255), nullable=True)
+    bridgeline = db.Column(db.String(255), nullable=True)
+
+    conf = db.relationship("BridgeConf", back_populates="bridges")
+    alarms = db.relationship("Alarm", back_populates="bridge")

app/models/mirrors.py

@@ -0,0 +1,53 @@
+from app import db
+from app.models import AbstractConfiguration, AbstractResource
+
+
+class Origin(AbstractConfiguration):
+    group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False)
+    domain_name = db.Column(db.String(255), unique=True, nullable=False)
+
+    group = db.relationship("Group", back_populates="origins")
+    mirrors = db.relationship("Mirror", back_populates="origin")
+    proxies = db.relationship("Proxy", back_populates="origin")
+    alarms = db.relationship("Alarm", back_populates="origin")
+
+    @classmethod
+    def csv_header(self):
+        return super().csv_header() + [
+            "group_id", "domain_name"
+        ]
+
+    def destroy(self):
+        super().destroy()
+        for proxy in self.proxies:
+            proxy.destroy()
+
+
+class Proxy(AbstractResource):
+    origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)
+    provider = db.Column(db.String(20), nullable=False)
+    slug = db.Column(db.String(20), nullable=True)
+    terraform_updated = db.Column(db.DateTime(), nullable=True)
+    url = db.Column(db.String(255), nullable=True)
+
+    origin = db.relationship("Origin", back_populates="proxies")
+    alarms = db.relationship("Alarm", back_populates="proxy")
+
+    @classmethod
+    def csv_header(self):
+        return super().csv_header() + [
+            "origin_id", "provider", "slug", "terraform_updated", "url"
+        ]
+
+
+class Mirror(AbstractResource):
+    origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False)
+    url = db.Column(db.String(255), unique=True, nullable=False)
+
+    origin = db.relationship("Origin", back_populates="mirrors")
+
+    @classmethod
+    def csv_header(self):
+        return super().csv_header() + [
+            "origin_id", "url"
+        ]

app/portal/__init__.py

@@ -4,7 +4,11 @@ from flask import Blueprint, render_template, Response, flash, redirect, url_for
 from sqlalchemy import exc, desc, or_
 
 from app.extensions import db
-from app.models import Group, Origin, Proxy, Alarm, BridgeConf, Bridge, MirrorList, AbstractResource
+from app.models import AbstractResource
+from app.models.bridges import BridgeConf, Bridge
+from app.models.alarms import Alarm
+from app import Origin, Proxy
+from app.models.base import Group, MirrorList
 from app.portal.forms import EditGroupForm, NewGroupForm, NewOriginForm, EditOriginForm, LifecycleForm, \
     NewBridgeConfForm, EditBridgeConfForm, NewMirrorListForm
 

app/terraform/block_bridge_github.py

@@ -4,7 +4,7 @@ from dateutil.parser import isoparse
 from github import Github
 
 from app import app
-from app.models import Bridge
+from app.models.bridges import Bridge
 
 
 def check_blocks():

app/terraform/block_external.py

@@ -2,7 +2,7 @@ from bs4 import BeautifulSoup
 import requests
 
 from app import app
-from app.models import Proxy
+from app.models.mirrors import Proxy
 
 
 def check_blocks():

app/terraform/block_ooni.py

@@ -7,7 +7,8 @@ import requests
 
 from app import app
 from app.extensions import db
-from app.models import Origin, AlarmState, Alarm
+from app.models.alarms import Alarm, AlarmState
+from app.models.mirrors import Proxy
 
 
 def check_origin(domain_name: str):

app/terraform/bridge/__init__.py

@@ -2,7 +2,8 @@ import datetime
 
 from app import app
 from app.extensions import db
-from app.models import BridgeConf, Bridge, Group
+from app.models.bridges import BridgeConf, Bridge
+from app.models.base import Group
 from app.terraform import BaseAutomation
 
 

app/terraform/eotk.py

@@ -1,5 +1,5 @@
 from app import app
-from app.models import Group
+from app.models.base import Group
 from app.terraform import BaseAutomation
 
 

app/terraform/list/__init__.py

@@ -2,7 +2,7 @@ import json
 
 from app import app
 from app.mirror_sites import bridgelines, mirror_sites, mirror_mapping
-from app.models import MirrorList
+from app.models.base import MirrorList
 from app.terraform import BaseAutomation
 
 

app/terraform/proxy/__init__.py

@@ -2,7 +2,8 @@ import datetime
 
 from app import app
 from app.extensions import db
-from app.models import Group, Origin, Proxy
+from app.models.base import Group
+from app.models.mirrors import Origin, Proxy
 from app.terraform import BaseAutomation
 
 

app/terraform/proxy/azure_cdn.py

@@ -9,7 +9,9 @@ import tldextract
 from app import app
 from app.alarms import get_proxy_alarm
 from app.extensions import db
-from app.models import Group, Proxy, Alarm, AlarmState
+from app.models.base import Group
+from app.models.mirrors import Proxy
+from app.models.alarms import AlarmState
 from app.terraform.proxy import ProxyAutomation
 
 

app/terraform/proxy/cloudfront.py

@@ -8,7 +8,8 @@ import boto3
 from app import app
 from app.alarms import get_proxy_alarm
 from app.extensions import db
-from app.models import Proxy, Alarm, AlarmState
+from app.models.alarms import Alarm, AlarmState
+from app.models.mirrors import Proxy
 from app.terraform.proxy import ProxyAutomation
 
 

app/terraform/proxy/fastly.py

@@ -8,7 +8,8 @@ import tldextract
 
 from app import app
 from app.extensions import db
-from app.models import Group, Origin, Proxy
+from app.models.base import Group
+from app.models.mirrors import Origin, Proxy
 
 TEMPLATE = """
 terraform {

app/terraform/proxy_check.py

@@ -2,7 +2,8 @@ import requests
 
 from app import app
 from app.extensions import db
-from app.models import AlarmState, Alarm, Proxy
+from app.models.alarms import Alarm, AlarmState
+from app.models.mirrors import Proxy
 
 
 def set_http_alarm(proxy_id: int, state: AlarmState, text: str):