import json import tldextract from datetime import datetime, timedelta from typing import Optional, List, Union, Any, Dict from tldextract import extract from werkzeug.datastructures import FileStorage from app.brm.brn import BRN from app.brm.utils import thumbnail_uploaded_image, create_data_uri, normalize_color from app.extensions import db from app.models import AbstractConfiguration, AbstractResource, Deprecation from app.models.onions import Onion country_origin = db.Table( 'country_origin', db.metadata, db.Column('country_id', db.ForeignKey('country.id'), primary_key=True), db.Column('origin_id', db.ForeignKey('origin.id'), primary_key=True), extend_existing=True, ) class Origin(AbstractConfiguration): group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False) domain_name = db.Column(db.String(255), unique=True, nullable=False) auto_rotation = db.Column(db.Boolean, nullable=False) smart = db.Column(db.Boolean(), nullable=False) assets = db.Column(db.Boolean(), nullable=False) risk_level_override = db.Column(db.Integer(), nullable=True) group = db.relationship("Group", back_populates="origins") proxies = db.relationship("Proxy", back_populates="origin") countries = db.relationship("Country", secondary=country_origin, back_populates='origins') @property def brn(self) -> BRN: return BRN( group_id=self.group_id, product="mirror", provider="conf", resource_type="origin", resource_id=self.domain_name ) @classmethod def csv_header(cls) -> List[str]: return super().csv_header() + [ "group_id", "domain_name", "auto_rotation", "smart", "assets", "country" ] def destroy(self) -> None: super().destroy() for proxy in self.proxies: proxy.destroy() @property def normalised_domain_name(self): extracted_domain = tldextract.extract(self.domain_name) return extracted_domain.registered_domain def onion(self) -> Optional[str]: tld = extract(self.domain_name).registered_domain onion = Onion.query.filter(Onion.domain_name == tld).first() if not onion: return None domain_name: str = self.domain_name return f"https://{domain_name.replace(tld, onion.onion_name)}.onion" @property def risk_level(self) -> Dict[str, int]: if self.risk_level_override: return {country.country_code: self.risk_level_override for country in self.countries} frequency_factor = 0 recency_factor = 0 recent_deprecations = ( db.session.query(Deprecation) # type: ignore[no-untyped-call] .join(Proxy, Deprecation.resource_id == Proxy.id) .join(Origin, Origin.id == Proxy.origin_id) .filter( Origin.id == self.id, Deprecation.resource_type == 'Proxy', Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168) ) .distinct(Proxy.id) .all() ) for deprecation in recent_deprecations: recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1) frequency_factor += 1 risk_levels: Dict[str, int] = {} for country in self.countries: risk_levels[country.country_code.upper()] = int(max(1, min(10, frequency_factor * recency_factor))) + country.risk_level return risk_levels class Country(AbstractConfiguration): @property def brn(self) -> BRN: return BRN( group_id=0, product="country", provider="iso3166-1", resource_type="alpha2", resource_id=self.country_code ) country_code = db.Column(db.String(2), nullable=False) risk_level_override = db.Column(db.Integer(), nullable=True) origins = db.relationship("Origin", secondary=country_origin, back_populates='countries') @property def risk_level(self) -> int: if self.risk_level_override: return int(self.risk_level_override // 2) frequency_factor = 0 recency_factor = 0 recent_deprecations = ( db.session.query(Deprecation) # type: ignore[no-untyped-call] .join(Proxy, Deprecation.resource_id == Proxy.id) .join(Origin, Origin.id == Proxy.origin_id) .join(Origin.countries) .filter( Country.id == self.id, Deprecation.resource_type == 'Proxy', Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168) ) .distinct(Proxy.id) .all() ) for deprecation in recent_deprecations: recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1) frequency_factor += 1 return int(max(1, min(10, frequency_factor * recency_factor))) class StaticOrigin(AbstractConfiguration): group_id = db.Column(db.Integer, db.ForeignKey("group.id"), nullable=False) storage_cloud_account_id = db.Column(db.Integer(), db.ForeignKey("cloud_account.id"), nullable=False) source_cloud_account_id = db.Column(db.Integer(), db.ForeignKey("cloud_account.id"), nullable=False) source_project = db.Column(db.String(255), nullable=False) auto_rotate = db.Column(db.Boolean, nullable=False) matrix_homeserver = db.Column(db.String(255), nullable=True) keanu_convene_path = db.Column(db.String(255), nullable=True) keanu_convene_config = db.Column(db.String(), nullable=True) clean_insights_backend = db.Column(db.String(255), nullable=True) origin_domain_name = db.Column(db.String(255), nullable=True) @property def brn(self) -> BRN: return BRN( group_id=self.group_id, product="mirror", provider="aws", resource_type="static", resource_id=self.domain_name ) group = db.relationship("Group", back_populates="statics") storage_cloud_account = db.relationship("CloudAccount", back_populates="statics", foreign_keys=[storage_cloud_account_id]) source_cloud_account = db.relationship("CloudAccount", back_populates="statics", foreign_keys=[source_cloud_account_id]) def destroy(self) -> None: # TODO: The StaticMetaAutomation will clean up for now, but it should probably happen here for consistency super().destroy() def update( self, source_project: str, description: str, auto_rotate: bool, matrix_homeserver: Optional[str], keanu_convene_path: Optional[str], keanu_convene_logo: Optional[FileStorage], keanu_convene_color: Optional[str], clean_insights_backend: Optional[Union[str, bool]], db_session_commit: bool, ) -> None: if isinstance(source_project, str): self.source_project = source_project else: raise ValueError("source project must be a str") if isinstance(description, str): self.description = description else: raise ValueError("description must be a str") if isinstance(auto_rotate, bool): self.auto_rotate = auto_rotate else: raise ValueError("auto_rotate must be a bool") if isinstance(matrix_homeserver, str): self.matrix_homeserver = matrix_homeserver else: raise ValueError("matrix_homeserver must be a str") if isinstance(keanu_convene_path, str): self.keanu_convene_path = keanu_convene_path else: raise ValueError("keanu_convene_path must be a str") if self.keanu_convene_config is None: self.keanu_convene_config = "{}" keanu_convene_config: Dict[str, Any] = json.loads(self.keanu_convene_config) if keanu_convene_logo is None: pass elif isinstance(keanu_convene_logo, FileStorage): if keanu_convene_logo.filename: # if False, no file was uploaded keanu_convene_config["logo"] = create_data_uri( thumbnail_uploaded_image(keanu_convene_logo), keanu_convene_logo.filename) else: raise ValueError("keanu_convene_logo must be a FileStorage") try: if isinstance(keanu_convene_color, str): keanu_convene_config["color"] = normalize_color(keanu_convene_color) # can raise ValueError else: raise ValueError() # re-raised below with message except ValueError: raise ValueError("keanu_convene_path must be a str containing an HTML color (CSS name or hex)") self.keanu_convene_config = json.dumps(keanu_convene_config, separators=(',', ':')) del keanu_convene_config # done with this temporary variable if clean_insights_backend is None or (isinstance(clean_insights_backend, bool) and not clean_insights_backend): self.clean_insights_backend = None elif isinstance(clean_insights_backend, bool) and clean_insights_backend: self.clean_insights_backend = "metrics.cleaninsights.org" elif isinstance(clean_insights_backend, str): self.clean_insights_backend = clean_insights_backend else: raise ValueError("clean_insights_backend must be a str, bool, or None") if db_session_commit: db.session.commit() self.updated = datetime.utcnow() class Proxy(AbstractResource): origin_id = db.Column(db.Integer, db.ForeignKey("origin.id"), nullable=False) pool_id = db.Column(db.Integer, db.ForeignKey("pool.id")) provider = db.Column(db.String(20), nullable=False) psg = db.Column(db.Integer, nullable=True) slug = db.Column(db.String(20), nullable=True) terraform_updated = db.Column(db.DateTime(), nullable=True) url = db.Column(db.String(255), nullable=True) origin = db.relationship("Origin", back_populates="proxies") pool = db.relationship("Pool", back_populates="proxies") @property def brn(self) -> BRN: return BRN( group_id=self.origin.group_id, product="mirror", provider=self.provider, resource_type="proxy", resource_id=str(self.id) ) @classmethod def csv_header(cls) -> List[str]: return super().csv_header() + [ "origin_id", "provider", "psg", "slug", "terraform_updated", "url" ] class SmartProxy(AbstractResource): group_id = db.Column(db.Integer(), db.ForeignKey("group.id"), nullable=False) instance_id = db.Column(db.String(100), nullable=True) provider = db.Column(db.String(20), nullable=False) region = db.Column(db.String(20), nullable=False) group = db.relationship("Group", back_populates="smart_proxies") @property def brn(self) -> BRN: return BRN( group_id=self.group_id, product="mirror", provider=self.provider, resource_type="smart_proxy", resource_id=str(1) )