diff --git a/app/alarms.py b/app/alarms.py index 61c49fc..0be0b2c 100644 --- a/app/alarms.py +++ b/app/alarms.py @@ -1,5 +1,5 @@ -import datetime -from typing import Optional, List +from datetime import datetime, timezone +from typing import List, Optional from app.brm.brn import BRN from app.extensions import db @@ -25,8 +25,8 @@ def _get_alarm(target: BRN, alarm.aspect = aspect alarm.target = target_str alarm.text = "New alarm" - alarm.state_changed = datetime.datetime.utcnow() - alarm.last_updated = datetime.datetime.utcnow() + alarm.state_changed = datetime.now(tz=timezone.utc) + alarm.last_updated = datetime.now(tz=timezone.utc) db.session.add(alarm) return alarm diff --git a/app/cli/automate.py b/app/cli/automate.py index ca2073a..a4a198e 100644 --- a/app/cli/automate.py +++ b/app/cli/automate.py @@ -1,44 +1,48 @@ -import datetime import logging import os import shutil import tempfile +from datetime import datetime, timedelta, timezone from traceback import TracebackException from typing import Type from app import app -from app.cli import _SubparserType, BaseCliHandler +from app.cli import BaseCliHandler, _SubparserType from app.extensions import db from app.models.activity import Activity -from app.models.automation import Automation, AutomationState, AutomationLogs +from app.models.automation import Automation, AutomationLogs, AutomationState from app.terraform import BaseAutomation +from app.terraform.alarms.eotk_aws import AlarmEotkAwsAutomation +from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation +from app.terraform.alarms.proxy_cloudfront import \ + AlarmProxyCloudfrontAutomation +from app.terraform.alarms.proxy_http_status import \ + AlarmProxyHTTPStatusAutomation +from app.terraform.alarms.smart_aws import AlarmSmartAwsAutomation from app.terraform.block.block_blocky import BlockBlockyAutomation +from app.terraform.block.block_scriptzteam import \ + BlockBridgeScriptzteamAutomation from app.terraform.block.bridge_github import BlockBridgeGitHubAutomation from app.terraform.block.bridge_gitlab import BlockBridgeGitlabAutomation -from app.terraform.block.bridge_roskomsvoboda import BlockBridgeRoskomsvobodaAutomation -from app.terraform.block.block_scriptzteam import BlockBridgeScriptzteamAutomation +from app.terraform.block.bridge_roskomsvoboda import \ + BlockBridgeRoskomsvobodaAutomation from app.terraform.block_external import BlockExternalAutomation from app.terraform.block_ooni import BlockOONIAutomation from app.terraform.block_roskomsvoboda import BlockRoskomsvobodaAutomation -from app.terraform.bridge.meta import BridgeMetaAutomation -from app.terraform.eotk.aws import EotkAWSAutomation -from app.terraform.alarms.eotk_aws import AlarmEotkAwsAutomation -from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation -from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation -from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation -from app.terraform.alarms.smart_aws import AlarmSmartAwsAutomation from app.terraform.bridge.aws import BridgeAWSAutomation from app.terraform.bridge.gandi import BridgeGandiAutomation from app.terraform.bridge.hcloud import BridgeHcloudAutomation +from app.terraform.bridge.meta import BridgeMetaAutomation from app.terraform.bridge.ovh import BridgeOvhAutomation +from app.terraform.eotk.aws import EotkAWSAutomation from app.terraform.list.github import ListGithubAutomation from app.terraform.list.gitlab import ListGitlabAutomation from app.terraform.list.http_post import ListHttpPostAutomation from app.terraform.list.s3 import ListS3Automation -from app.terraform.proxy.meta import ProxyMetaAutomation from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation from app.terraform.proxy.fastly import ProxyFastlyAutomation +from app.terraform.proxy.meta import ProxyMetaAutomation from app.terraform.static.aws import StaticAWSAutomation from app.terraform.static.meta import StaticMetaAutomation @@ -108,7 +112,7 @@ def run_job(job_cls: Type[BaseAutomation], *, automation.description = job_cls.description automation.enabled = False automation.next_is_full = False - automation.added = datetime.datetime.utcnow() + automation.added = datetime.now(tz=timezone.utc) automation.updated = automation.added db.session.add(automation) db.session.commit() @@ -117,7 +121,7 @@ def run_job(job_cls: Type[BaseAutomation], *, logging.warning("Not running an already running automation") return if not ignore_schedule and not force: - if automation.next_run is not None and automation.next_run > datetime.datetime.utcnow(): + if automation.next_run is not None and automation.next_run > datetime.now(tz=timezone.utc): logging.warning("Not time to run this job yet") return if not automation.enabled and not force: @@ -145,7 +149,7 @@ def run_job(job_cls: Type[BaseAutomation], *, logs = "\n".join(trace.format()) if job is not None and success: automation.state = AutomationState.IDLE - automation.next_run = datetime.datetime.utcnow() + datetime.timedelta( + automation.next_run = datetime.now(tz=timezone.utc) + timedelta( minutes=getattr(job, "frequency", 7)) if 'TERRAFORM_DIRECTORY' not in app.config and working_dir is not None: # We used a temporary working directory @@ -168,8 +172,8 @@ def run_job(job_cls: Type[BaseAutomation], *, automation.next_run = None log = AutomationLogs() log.automation_id = automation.id - log.added = datetime.datetime.utcnow() - log.updated = datetime.datetime.utcnow() + log.added = datetime.now(tz=timezone.utc) + log.updated = datetime.now(tz=timezone.utc) log.logs = str(logs) db.session.add(log) db.session.commit() @@ -182,7 +186,7 @@ def run_job(job_cls: Type[BaseAutomation], *, ) db.session.add(activity) activity.notify() # Notify before commit because the failure occurred even if we can't commit. - automation.last_run = datetime.datetime.utcnow() + automation.last_run = datetime.now(tz=timezone.utc) db.session.commit() diff --git a/app/lists/mirror_mapping.py b/app/lists/mirror_mapping.py index c66c25d..f191d8a 100644 --- a/app/lists/mirror_mapping.py +++ b/app/lists/mirror_mapping.py @@ -1,6 +1,7 @@ import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, TypedDict + from flask import current_app from sqlalchemy import or_ from sqlalchemy.orm import selectinload @@ -8,7 +9,7 @@ from tldextract import extract from app.extensions import db from app.models.base import Group, Pool -from app.models.mirrors import Proxy, Origin +from app.models.mirrors import Origin, Proxy class MirrorMappingMirror(TypedDict): @@ -29,7 +30,7 @@ class MirrorMapping(TypedDict): def mirror_mapping(_: Optional[Pool]) -> MirrorMapping: - two_days_ago = datetime.utcnow() - timedelta(days=2) + two_days_ago = datetime.now(tz=timezone.utc) - timedelta(days=2) proxies = ( db.session.query(Proxy) diff --git a/app/models/__init__.py b/app/models/__init__.py index 7f723d9..c967bec 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,7 +1,7 @@ import logging from abc import abstractmethod -from datetime import datetime -from typing import Union, List, Optional, Any, Dict +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Union from sqlalchemy.orm import Mapped, mapped_column @@ -14,9 +14,9 @@ class AbstractConfiguration(db.Model): # type: ignore id: Mapped[int] = mapped_column(db.Integer, primary_key=True) description: Mapped[str] - added: Mapped[datetime] - updated: Mapped[datetime] - destroyed: Mapped[Optional[datetime]] + added: Mapped[datetime] = mapped_column(db.DateTime(timezone=True)) + updated: Mapped[datetime] = mapped_column(db.DateTime(timezone=True)) + destroyed: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True) @property @abstractmethod @@ -24,8 +24,8 @@ class AbstractConfiguration(db.Model): # type: ignore raise NotImplementedError() def destroy(self) -> None: - self.destroyed = datetime.utcnow() - self.updated = datetime.utcnow() + self.destroyed = datetime.now(tz=timezone.utc) + self.updated = datetime.now(tz=timezone.utc) @classmethod def csv_header(cls) -> List[str]: @@ -41,11 +41,11 @@ class AbstractConfiguration(db.Model): # type: ignore class Deprecation(db.Model): # type: ignore[name-defined,misc] id: Mapped[int] = mapped_column(db.Integer, primary_key=True) - resource_type: Mapped[str] = mapped_column(db.String(50)) - resource_id: Mapped[int] = mapped_column(db.Integer) - deprecated_at: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow, nullable=False) + resource_type: Mapped[str] + resource_id: Mapped[int] + deprecated_at: Mapped[datetime] = mapped_column(db.DateTime(timezone=True)) meta: Mapped[Optional[Dict[str, Any]]] = mapped_column(db.JSON()) - reason: Mapped[str] = mapped_column(db.String(), nullable=False) + reason: Mapped[str] @property def resource(self) -> "AbstractResource": @@ -58,11 +58,11 @@ class AbstractResource(db.Model): # type: ignore __abstract__ = True id: Mapped[int] = mapped_column(db.Integer, primary_key=True) - added: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow) - updated: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow) - deprecated: Mapped[Optional[datetime]] + added: Mapped[datetime] = mapped_column(db.DateTime(timezone=True)) + updated: Mapped[datetime] = mapped_column(db.DateTime(timezone=True)) + deprecated: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True) deprecation_reason: Mapped[Optional[str]] - destroyed: Mapped[Optional[datetime]] + destroyed: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True) def __init__(self, *, id: Optional[int] = None, @@ -73,9 +73,9 @@ class AbstractResource(db.Model): # type: ignore destroyed: Optional[datetime] = None, **kwargs: Any) -> None: if added is None: - added = datetime.utcnow() + added = datetime.now(tz=timezone.utc) if updated is None: - updated = datetime.utcnow() + updated = datetime.now(tz=timezone.utc) super().__init__(id=id, added=added, updated=updated, @@ -101,9 +101,9 @@ class AbstractResource(db.Model): # type: ignore """ if self.deprecated is None: logging.info("Deprecating %s (reason=%s)", self.brn, reason) - self.deprecated = datetime.utcnow() + self.deprecated = datetime.now(tz=timezone.utc) self.deprecation_reason = reason - self.updated = datetime.utcnow() + self.updated = datetime.now(tz=timezone.utc) if reason not in [d.reason for d in self.deprecations]: new_deprecation = Deprecation( resource_type=type(self).__name__, @@ -132,8 +132,8 @@ class AbstractResource(db.Model): # type: ignore """ if self.deprecated is None: self.deprecate(reason="destroyed") - self.destroyed = datetime.utcnow() - self.updated = datetime.utcnow() + self.destroyed = datetime.now(tz=timezone.utc) + self.updated = datetime.now(tz=timezone.utc) @classmethod def csv_header(cls) -> List[str]: diff --git a/app/models/activity.py b/app/models/activity.py index 3ad5124..3ba8eb7 100644 --- a/app/models/activity.py +++ b/app/models/activity.py @@ -1,39 +1,40 @@ -import datetime +from datetime import datetime, timezone from typing import Any, Optional import requests +from sqlalchemy.orm import Mapped, mapped_column from app.brm.brn import BRN -from app.models import AbstractConfiguration from app.extensions import db +from app.models import AbstractConfiguration class Activity(db.Model): # type: ignore - id = db.Column(db.Integer(), primary_key=True) - group_id = db.Column(db.Integer(), nullable=True) - activity_type = db.Column(db.String(20), nullable=False) - text = db.Column(db.Text(), nullable=False) - added = db.Column(db.DateTime(), nullable=False) + id: Mapped[int] = mapped_column(primary_key=True) + group_id: Mapped[Optional[int]] + activity_type: Mapped[str] + text: Mapped[str] + added: Mapped[datetime] = mapped_column(db.DateTime(timezone=True)) def __init__(self, *, id: Optional[int] = None, group_id: Optional[int] = None, activity_type: str, text: str, - added: Optional[datetime.datetime] = None, + added: Optional[datetime] = None, **kwargs: Any) -> None: if not isinstance(activity_type, str) or len(activity_type) > 20 or activity_type == "": raise TypeError("expected string for activity type between 1 and 20 characters") if not isinstance(text, str): raise TypeError("expected string for text") + if added is None: + added = datetime.now(tz=timezone.utc) super().__init__(id=id, group_id=group_id, activity_type=activity_type, text=text, added=added, **kwargs) - if self.added is None: - self.added = datetime.datetime.utcnow() def notify(self) -> int: count = 0 @@ -47,8 +48,8 @@ class Activity(db.Model): # type: ignore class Webhook(AbstractConfiguration): - format = db.Column(db.String(20)) - url = db.Column(db.String(255)) + format: Mapped[str] + url: Mapped[str] @property def brn(self) -> BRN: diff --git a/app/models/alarms.py b/app/models/alarms.py index 2ffd694..88714e7 100644 --- a/app/models/alarms.py +++ b/app/models/alarms.py @@ -1,6 +1,8 @@ import enum -from datetime import datetime -from typing import List, Any +from datetime import datetime, timezone +from typing import Any, List + +from sqlalchemy.orm import Mapped, mapped_column from app.extensions import db from app.models.activity import Activity @@ -24,13 +26,13 @@ class AlarmState(enum.Enum): class Alarm(db.Model): # type: ignore - id = db.Column(db.Integer, primary_key=True) - target = db.Column(db.String(255), nullable=False) - aspect = db.Column(db.String(255), nullable=False) - alarm_state = db.Column(db.Enum(AlarmState), default=AlarmState.UNKNOWN, nullable=False) - state_changed = db.Column(db.DateTime(), nullable=False) - last_updated = db.Column(db.DateTime(), nullable=False) - text = db.Column(db.String(255), nullable=False) + id: Mapped[int] = mapped_column(db.Integer, primary_key=True) + target: Mapped[str] + aspect: Mapped[str] + alarm_state: Mapped[AlarmState] = mapped_column(default=AlarmState.UNKNOWN) + state_changed: Mapped[datetime] = mapped_column(db.DateTime(timezone=True)) + last_updated: Mapped[datetime] = mapped_column(db.DateTime(timezone=True)) + text: Mapped[str] @classmethod def csv_header(cls) -> List[str]: @@ -40,11 +42,8 @@ class Alarm(db.Model): # type: ignore return [getattr(self, x) for x in self.csv_header()] def update_state(self, state: AlarmState, text: str) -> None: - if self.alarm_state is None: - self.alarm_state = AlarmState.UNKNOWN - if self.alarm_state != state or self.state_changed is None: - self.state_changed = datetime.utcnow() + self.state_changed = datetime.now(tz=timezone.utc) activity = Activity(activity_type="alarm_state", text=f"[{self.aspect}] {state.emoji} Alarm state changed from " f"{self.alarm_state.name} to {state.name} on {self.target}: {text}.") @@ -56,4 +55,4 @@ class Alarm(db.Model): # type: ignore db.session.add(activity) self.alarm_state = state self.text = text - self.last_updated = datetime.utcnow() + self.last_updated = datetime.now(tz=timezone.utc) diff --git a/app/models/automation.py b/app/models/automation.py index cb54843..99e5387 100644 --- a/app/models/automation.py +++ b/app/models/automation.py @@ -1,5 +1,8 @@ -import datetime import enum +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy.orm import Mapped, mapped_column from app.brm.brn import BRN from app.extensions import db @@ -13,12 +16,12 @@ class AutomationState(enum.Enum): class Automation(AbstractConfiguration): - short_name = db.Column(db.String(25), nullable=False) - state = db.Column(db.Enum(AutomationState), default=AutomationState.IDLE, nullable=False) - enabled = db.Column(db.Boolean, nullable=False) - last_run = db.Column(db.DateTime(), nullable=True) - next_run = db.Column(db.DateTime(), nullable=True) - next_is_full = db.Column(db.Boolean(), nullable=False) + short_name: Mapped[str] + state: Mapped[AutomationState] = mapped_column(default=AutomationState.IDLE) + enabled: Mapped[bool] + last_run: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True)) + next_run: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True)) + next_is_full: Mapped[bool] logs = db.relationship("AutomationLogs", back_populates="automation") @@ -34,13 +37,13 @@ class Automation(AbstractConfiguration): def kick(self) -> None: self.enabled = True - self.next_run = datetime.datetime.utcnow() - self.updated = datetime.datetime.utcnow() + self.next_run = datetime.now(tz=timezone.utc) + self.updated = datetime.now(tz=timezone.utc) class AutomationLogs(AbstractResource): - automation_id = db.Column(db.Integer, db.ForeignKey(Automation.id), nullable=False) - logs = db.Column(db.Text) + automation_id: Mapped[int] = mapped_column(db.ForeignKey("automation.id")) + logs: Mapped[str] automation = db.relationship("Automation", back_populates="logs") diff --git a/app/models/base.py b/app/models/base.py index c990f49..9ecb35f 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -1,5 +1,5 @@ -from datetime import datetime -from typing import List, TypedDict, Optional, TYPE_CHECKING +from datetime import datetime, timezone +from typing import TYPE_CHECKING, List, Optional, TypedDict from sqlalchemy import and_ from sqlalchemy.orm import Mapped, aliased, mapped_column, relationship @@ -22,7 +22,7 @@ class GroupDict(TypedDict): class Group(AbstractConfiguration): - group_name: Mapped[str] = db.Column(db.String(80), unique=True, nullable=False) + group_name: Mapped[str] = mapped_column(unique=True) eotk: Mapped[bool] origins: Mapped[List["Origin"]] = relationship("Origin", back_populates="group") @@ -93,19 +93,19 @@ class Pool(AbstractConfiguration): class PoolGroup(db.Model): # type: ignore[name-defined,misc] - pool_id = db.Column(db.Integer, db.ForeignKey("pool.id"), primary_key=True) - group_id = db.Column(db.Integer, db.ForeignKey("group.id"), primary_key=True) + pool_id: Mapped[int] = mapped_column(db.ForeignKey("pool.id"), primary_key=True) + group_id: Mapped[int] = mapped_column(db.ForeignKey("group.id"), primary_key=True) class MirrorList(AbstractConfiguration): - pool_id = db.Column(db.Integer, db.ForeignKey("pool.id")) - provider = db.Column(db.String(255), nullable=False) - format = db.Column(db.String(20), nullable=False) - encoding = db.Column(db.String(20), nullable=False) - container = db.Column(db.String(255), nullable=False) - branch = db.Column(db.String(255), nullable=False) - role = db.Column(db.String(255), nullable=True) - filename = db.Column(db.String(255), nullable=False) + pool_id: Mapped[int] = mapped_column(db.ForeignKey("pool.id")) + provider: Mapped[str] + format: Mapped[str] + encoding: Mapped[str] + container: Mapped[str] + branch: Mapped[str] + role: Mapped[Optional[str]] + filename: Mapped[str] pool = db.relationship("Pool", back_populates="lists") @@ -132,8 +132,8 @@ class MirrorList(AbstractConfiguration): } def destroy(self) -> None: - self.destroyed = datetime.utcnow() - self.updated = datetime.utcnow() + self.destroyed = datetime.now(tz=timezone.utc) + self.updated = datetime.now(tz=timezone.utc) def url(self) -> str: if self.provider == "gitlab": diff --git a/app/models/bridges.py b/app/models/bridges.py index 87420c6..979eecf 100644 --- a/app/models/bridges.py +++ b/app/models/bridges.py @@ -1,6 +1,6 @@ import enum -from datetime import datetime -from typing import List +from datetime import datetime, timezone +from typing import List, Optional from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -37,12 +37,12 @@ class BridgeConf(AbstractConfiguration): ) def destroy(self) -> None: - self.destroyed = datetime.utcnow() - self.updated = datetime.utcnow() + self.destroyed = datetime.now(tz=timezone.utc) + self.updated = datetime.now(tz=timezone.utc) for bridge in self.bridges: if bridge.destroyed is None: - bridge.destroyed = datetime.utcnow() - bridge.updated = datetime.utcnow() + bridge.destroyed = datetime.now(tz=timezone.utc) + bridge.updated = datetime.now(tz=timezone.utc) @classmethod def csv_header(cls) -> List[str]: @@ -52,13 +52,13 @@ class BridgeConf(AbstractConfiguration): class Bridge(AbstractResource): - conf_id = db.Column(db.Integer, db.ForeignKey("bridge_conf.id"), nullable=False) - cloud_account_id = db.Column(db.Integer, db.ForeignKey("cloud_account.id")) - terraform_updated = db.Column(db.DateTime(), nullable=True) - nickname = db.Column(db.String(255), nullable=True) - fingerprint = db.Column(db.String(255), nullable=True) - hashed_fingerprint = db.Column(db.String(255), nullable=True) - bridgeline = db.Column(db.String(255), nullable=True) + conf_id: Mapped[int] = mapped_column(db.ForeignKey("bridge_conf.id")) + cloud_account_id: Mapped[int] = mapped_column(db.ForeignKey("cloud_account.id")) + terraform_updated: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True) + nickname: Mapped[Optional[str]] + fingerprint: Mapped[Optional[str]] + hashed_fingerprint: Mapped[Optional[str]] + bridgeline: Mapped[Optional[str]] conf = db.relationship("BridgeConf", back_populates="bridges") cloud_account = db.relationship("CloudAccount", back_populates="bridges") diff --git a/app/models/mirrors.py b/app/models/mirrors.py index 1f81683..b213d54 100644 --- a/app/models/mirrors.py +++ b/app/models/mirrors.py @@ -1,8 +1,8 @@ from __future__ import annotations import json -from datetime import datetime, timedelta -from typing import Optional, List, Union, Any, Dict, TypedDict, Literal +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Literal, Optional, TypedDict, Union import tldextract from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -10,10 +10,11 @@ from tldextract import extract from werkzeug.datastructures import FileStorage from app.brm.brn import BRN -from app.brm.utils import thumbnail_uploaded_image, create_data_uri, normalize_color +from app.brm.utils import (create_data_uri, normalize_color, + thumbnail_uploaded_image) from app.extensions import db from app.models import AbstractConfiguration, AbstractResource, Deprecation -from app.models.base import Pool, Group +from app.models.base import Group, Pool from app.models.onions import Onion country_origin = db.Table( @@ -93,14 +94,14 @@ class Origin(AbstractConfiguration): .filter( Origin.id == self.id, Deprecation.resource_type == 'Proxy', - Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168), + Deprecation.deprecated_at >= datetime.now(tz=timezone.utc) - timedelta(hours=168), Deprecation.reason != "destroyed" ) .distinct(Proxy.id) .all() ) for deprecation in recent_deprecations: - recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1) + recency_factor += 1 / max((datetime.now(tz=timezone.utc) - deprecation.deprecated_at).total_seconds() // 3600, 1) frequency_factor += 1 risk_levels: Dict[str, int] = {} for country in self.countries: @@ -149,14 +150,14 @@ class Country(AbstractConfiguration): .filter( Country.id == self.id, Deprecation.resource_type == 'Proxy', - Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168), + Deprecation.deprecated_at >= datetime.now(tz=timezone.utc) - timedelta(hours=168), Deprecation.reason != "destroyed" ) .distinct(Proxy.id) .all() ) for deprecation in recent_deprecations: - recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1) + recency_factor += 1 / max((datetime.now(tz=timezone.utc) - deprecation.deprecated_at).total_seconds() // 3600, 1) frequency_factor += 1 return int(max(1, min(10, frequency_factor * recency_factor))) @@ -255,7 +256,7 @@ class StaticOrigin(AbstractConfiguration): raise ValueError("clean_insights_backend must be a str, bool, or None") if db_session_commit: db.session.commit() - self.updated = datetime.utcnow() + self.updated = datetime.now(tz=timezone.utc) ResourceStatus = Union[Literal["active"], Literal["pending"], Literal["expiring"], Literal["destroyed"]] @@ -274,7 +275,7 @@ class Proxy(AbstractResource): provider: Mapped[str] = mapped_column(db.String(20), nullable=False) psg: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True) slug: Mapped[Optional[str]] = mapped_column(db.String(20), nullable=True) - terraform_updated: Mapped[Optional[datetime]] = mapped_column(db.DateTime(), nullable=True) + terraform_updated: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True) url: Mapped[Optional[str]] = mapped_column(db.String(255), nullable=True) origin: Mapped[Origin] = relationship("Origin", back_populates="proxies") diff --git a/app/portal/__init__.py b/app/portal/__init__.py index 868ae2f..d56a088 100644 --- a/app/portal/__init__.py +++ b/app/portal/__init__.py @@ -3,28 +3,28 @@ import logging from datetime import datetime, timedelta, timezone from typing import Optional -from flask import Blueprint, render_template, request, url_for, redirect +from flask import Blueprint, redirect, render_template, request, url_for from flask.typing import ResponseReturnValue from markupsafe import Markup -from sqlalchemy import desc, or_, func +from sqlalchemy import desc, func, or_ from app.alarms import alarms_for from app.models.activity import Activity from app.models.alarms import Alarm, AlarmState +from app.models.base import Group from app.models.bridges import Bridge from app.models.mirrors import Origin, Proxy -from app.models.base import Group from app.models.onions import Eotk -from app.portal.country import bp as country -from app.portal.cloud import bp as cloud from app.portal.automation import bp as automation -from app.portal.bridgeconf import bp as bridgeconf from app.portal.bridge import bp as bridge +from app.portal.bridgeconf import bp as bridgeconf +from app.portal.cloud import bp as cloud +from app.portal.country import bp as country from app.portal.eotk import bp as eotk from app.portal.group import bp as group from app.portal.list import bp as list_ -from app.portal.origin import bp as origin from app.portal.onion import bp as onion +from app.portal.origin import bp as origin from app.portal.pool import bp as pool from app.portal.proxy import bp as proxy from app.portal.smart_proxy import bp as smart_proxy @@ -57,7 +57,7 @@ def calculate_bridge_expiry(b: Bridge) -> str: logging.warning("Bridge expiry requested by template for a bridge %s that was not expiring.", b.id) return "Not expiring" expiry = b.deprecated + timedelta(hours=b.conf.expiry_hours) - countdown = expiry - datetime.utcnow() + countdown = expiry - datetime.now(tz=timezone.utc) if countdown.days == 0: return f"{countdown.seconds // 3600} hours" return f"{countdown.days} days" @@ -66,7 +66,7 @@ def calculate_bridge_expiry(b: Bridge) -> str: @portal.app_template_filter("mirror_expiry") def calculate_mirror_expiry(s: datetime) -> str: expiry = s + timedelta(days=3) - countdown = expiry - datetime.utcnow() + countdown = expiry - datetime.now(tz=timezone.utc) if countdown.days == 0: return f"{countdown.seconds // 3600} hours" return f"{countdown.days} days" diff --git a/app/portal/automation.py b/app/portal/automation.py index 01180fe..b1e878d 100644 --- a/app/portal/automation.py +++ b/app/portal/automation.py @@ -1,16 +1,16 @@ -from datetime import datetime +from datetime import datetime, timezone from typing import Optional -from flask import render_template, flash, Response, Blueprint, current_app +from flask import Blueprint, Response, current_app, flash, render_template from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm -from sqlalchemy import exc, desc -from wtforms import SubmitField, BooleanField +from sqlalchemy import desc, exc +from wtforms import BooleanField, SubmitField from app.extensions import db from app.models.automation import Automation, AutomationLogs from app.models.tfstate import TerraformState -from app.portal.util import view_lifecycle, response_404 +from app.portal.util import response_404, view_lifecycle bp = Blueprint("automation", __name__) @@ -54,7 +54,7 @@ def automation_edit(automation_id: int) -> ResponseReturnValue: form = EditAutomationForm(enabled=automation.enabled) if form.validate_on_submit(): automation.enabled = form.enabled.data - automation.updated = datetime.utcnow() + automation.updated = datetime.now(tz=timezone.utc) try: db.session.commit() flash("Saved changes to bridge configuration.", "success") diff --git a/app/portal/bridgeconf.py b/app/portal/bridgeconf.py index 5dd5d14..360f185 100644 --- a/app/portal/bridgeconf.py +++ b/app/portal/bridgeconf.py @@ -1,11 +1,12 @@ -from datetime import datetime -from typing import Optional, List +from datetime import datetime, timezone +from typing import List, Optional -from flask import render_template, url_for, flash, redirect, Response, Blueprint +from flask import (Blueprint, Response, flash, redirect, render_template, + url_for) from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm from sqlalchemy import exc -from wtforms import SelectField, StringField, IntegerField, SubmitField +from wtforms import IntegerField, SelectField, StringField, SubmitField from wtforms.validators import DataRequired, NumberRange from app.extensions import db @@ -99,8 +100,8 @@ def bridgeconf_new(group_id: Optional[int] = None) -> ResponseReturnValue: bridgeconf.max_number = form.max_number.data bridgeconf.expiry_hours = form.expiry_hours.data bridgeconf.provider_allocation = ProviderAllocation[form.provider_allocation.data] - bridgeconf.created = datetime.utcnow() - bridgeconf.updated = datetime.utcnow() + bridgeconf.added = datetime.now(tz=timezone.utc) + bridgeconf.updated = datetime.now(tz=timezone.utc) try: db.session.add(bridgeconf) db.session.commit() @@ -137,7 +138,7 @@ def bridgeconf_edit(bridgeconf_id: int) -> ResponseReturnValue: bridgeconf.max_number = form.max_number.data bridgeconf.expiry_hours = form.expiry_hours.data bridgeconf.provider_allocation = ProviderAllocation[form.provider_allocation.data] - bridgeconf.updated = datetime.utcnow() + bridgeconf.updated = datetime.now(tz=timezone.utc) try: db.session.commit() flash("Saved changes to bridge configuration.", "success") diff --git a/app/portal/country.py b/app/portal/country.py index 1a5b700..c824208 100644 --- a/app/portal/country.py +++ b/app/portal/country.py @@ -1,10 +1,10 @@ -from datetime import datetime +from datetime import datetime, timezone import sqlalchemy -from flask import Blueprint, render_template, Response, flash +from flask import Blueprint, Response, flash, render_template from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm -from wtforms import IntegerField, BooleanField, SubmitField +from wtforms import BooleanField, IntegerField, SubmitField from app.extensions import db from app.models.mirrors import Country @@ -63,7 +63,7 @@ def country_edit(country_id: int) -> ResponseReturnValue: country.risk_level_override = form.risk_level_override_number.data else: country.risk_level_override = None - country.updated = datetime.utcnow() + country.updated = datetime.now(tz=timezone.utc) try: db.session.commit() flash("Saved changes to country.", "success") diff --git a/app/portal/group.py b/app/portal/group.py index ed4179a..a1aceed 100644 --- a/app/portal/group.py +++ b/app/portal/group.py @@ -1,10 +1,11 @@ -from datetime import datetime +from datetime import datetime, timezone -from flask import render_template, url_for, flash, redirect, Response, Blueprint +import sqlalchemy +from flask import (Blueprint, Response, flash, redirect, render_template, + url_for) from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm -import sqlalchemy -from wtforms import StringField, BooleanField, SubmitField +from wtforms import BooleanField, StringField, SubmitField from wtforms.validators import DataRequired from app.extensions import db @@ -72,7 +73,7 @@ def group_edit(group_id: int) -> ResponseReturnValue: if form.validate_on_submit(): group.description = form.description.data group.eotk = form.eotk.data - group.updated = datetime.utcnow() + group.updated = datetime.now(tz=timezone.utc) try: db.session.commit() flash("Saved changes to group.", "success") diff --git a/app/portal/list.py b/app/portal/list.py index b716a5a..a147a69 100644 --- a/app/portal/list.py +++ b/app/portal/list.py @@ -1,8 +1,9 @@ import json -from datetime import datetime -from typing import Optional, Any +from datetime import datetime, timezone +from typing import Any, Optional -from flask import render_template, url_for, flash, redirect, Blueprint, Response +from flask import (Blueprint, Response, flash, redirect, render_template, + url_for) from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm from sqlalchemy import exc @@ -103,8 +104,8 @@ def list_new(group_id: Optional[int] = None) -> ResponseReturnValue: list_.branch = form.branch.data list_.role = form.role.data list_.filename = form.filename.data - list_.created = datetime.utcnow() - list_.updated = datetime.utcnow() + list_.added = datetime.now(tz=timezone.utc) + list_.updated = datetime.now(tz=timezone.utc) try: db.session.add(list_) db.session.commit() @@ -176,7 +177,7 @@ def list_edit(list_id: int) -> ResponseReturnValue: list_.branch = form.branch.data list_.role = form.role.data list_.filename = form.filename.data - list_.updated = datetime.utcnow() + list_.updated = datetime.now(tz=timezone.utc) try: db.session.commit() flash("Saved changes to group.", "success") diff --git a/app/portal/origin.py b/app/portal/origin.py index ce171f6..2de0f50 100644 --- a/app/portal/origin.py +++ b/app/portal/origin.py @@ -1,20 +1,22 @@ import urllib.parse -from datetime import datetime -from typing import Optional, List +from datetime import datetime, timezone +from typing import List, Optional import requests import sqlalchemy -from flask import flash, redirect, url_for, render_template, Response, Blueprint +from flask import (Blueprint, Response, flash, redirect, render_template, + url_for) from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm from sqlalchemy import exc -from wtforms import StringField, SelectField, SubmitField, BooleanField, IntegerField +from wtforms import (BooleanField, IntegerField, SelectField, StringField, + SubmitField) from wtforms.validators import DataRequired from app.extensions import db from app.models.base import Group -from app.models.mirrors import Origin, Country -from app.portal.util import response_404, view_lifecycle, LifecycleForm +from app.models.mirrors import Country, Origin +from app.portal.util import LifecycleForm, response_404, view_lifecycle bp = Blueprint("origin", __name__) @@ -64,8 +66,8 @@ def origin_new(group_id: Optional[int] = None) -> ResponseReturnValue: origin.auto_rotation = form.auto_rotate.data origin.smart = form.smart_proxy.data origin.assets = form.asset_domain.data - origin.created = datetime.utcnow() - origin.updated = datetime.utcnow() + origin.added = datetime.now(tz=timezone.utc) + origin.updated = datetime.now(tz=timezone.utc) try: db.session.add(origin) db.session.commit() @@ -106,7 +108,7 @@ def origin_edit(origin_id: int) -> ResponseReturnValue: origin.risk_level_override = form.risk_level_override_number.data else: origin.risk_level_override = None - origin.updated = datetime.utcnow() + origin.updated = datetime.now(tz=timezone.utc) try: db.session.commit() flash(f"Saved changes for origin {origin.domain_name}.", "success") diff --git a/app/portal/report.py b/app/portal/report.py index 219f1bc..a8ec4bb 100644 --- a/app/portal/report.py +++ b/app/portal/report.py @@ -1,22 +1,22 @@ # mypy: ignore-errors -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from flask import Blueprint, render_template from flask.typing import ResponseReturnValue -from sqlalchemy import func, and_, desc +from sqlalchemy import and_, desc, func from sqlalchemy.orm import aliased from app.extensions import db from app.models import Deprecation -from app.models.mirrors import Proxy, Origin, Country +from app.models.mirrors import Country, Origin, Proxy report = Blueprint("report", __name__) def generate_subqueries(): DeprecationAlias = aliased(Deprecation) - now = datetime.utcnow() + now = datetime.now(tz=timezone.utc) deprecations_24hr_subquery = ( db.session.query( DeprecationAlias.resource_id, @@ -98,7 +98,7 @@ def report_blocks() -> ResponseReturnValue: Proxy.deprecated, Proxy.deprecation_reason ).join(Origin, Origin.id == Proxy.origin_id - ).filter(and_(Proxy.deprecated > datetime.utcnow() - timedelta(days=1), + ).filter(and_(Proxy.deprecated > datetime.now(tz=timezone.utc) - timedelta(days=1), Proxy.deprecation_reason.like('block_%'))).all() return render_template("report_blocks.html.j2", diff --git a/app/portal/storage.py b/app/portal/storage.py index 62c2ec6..8fe4b90 100644 --- a/app/portal/storage.py +++ b/app/portal/storage.py @@ -1,16 +1,16 @@ -from datetime import datetime +from datetime import datetime, timezone from typing import Optional -from flask import render_template, flash, Response, Blueprint +from flask import Blueprint, Response, flash, render_template from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm from sqlalchemy import exc -from wtforms import SubmitField, BooleanField +from wtforms import BooleanField, SubmitField from app.extensions import db from app.models.automation import Automation from app.models.tfstate import TerraformState -from app.portal.util import view_lifecycle, response_404 +from app.portal.util import response_404, view_lifecycle bp = Blueprint("storage", __name__) @@ -39,7 +39,7 @@ def storage_edit(storage_key: str) -> ResponseReturnValue: if form.validate_on_submit(): if form.force_unlock.data: storage.lock = None - storage.updated = datetime.utcnow() + storage.updated = datetime.now(tz=timezone.utc) try: db.session.commit() flash("Storage has been force unlocked.", "success") diff --git a/app/portal/webhook.py b/app/portal/webhook.py index 2711e0a..4568744 100644 --- a/app/portal/webhook.py +++ b/app/portal/webhook.py @@ -1,11 +1,12 @@ -from datetime import datetime +from datetime import datetime, timezone from typing import Optional -from flask import Blueprint, flash, Response, render_template, redirect, url_for +from flask import (Blueprint, Response, flash, redirect, render_template, + url_for) from flask.typing import ResponseReturnValue from flask_wtf import FlaskForm from sqlalchemy import exc -from wtforms import StringField, SelectField, SubmitField +from wtforms import SelectField, StringField, SubmitField from wtforms.validators import DataRequired from app.extensions import db @@ -70,7 +71,7 @@ def webhook_edit(webhook_id: int) -> ResponseReturnValue: webhook.description = form.description.data webhook.format = form.description.data webhook.url = form.description.data - webhook.updated = datetime.utcnow() + webhook.updated = datetime.now(tz=timezone.utc) try: db.session.commit() flash("Saved changes to webhook.", "success") diff --git a/app/terraform/block/bridge.py b/app/terraform/block/bridge.py index c082a64..ed505f8 100644 --- a/app/terraform/block/bridge.py +++ b/app/terraform/block/bridge.py @@ -1,7 +1,7 @@ -from datetime import datetime, timedelta import logging from abc import abstractmethod -from typing import Tuple, List, Callable, Optional, Any +from datetime import datetime, timedelta, timezone +from typing import Any, Callable, List, Optional, Tuple from app.extensions import db from app.models.activity import Activity @@ -25,14 +25,14 @@ class BlockBridgeAutomation(BaseAutomation): super().__init__(*args, **kwargs) def perform_deprecations(self, ids: List[str], bridge_select_func: Callable[[str], Optional[Bridge]] - ) -> List[Tuple[str, str, str]]: + ) -> List[Tuple[Optional[str], Any, Any]]: rotated = [] for id_ in ids: bridge = bridge_select_func(id_) if bridge is None: continue logging.debug("Found %s blocked", bridge.hashed_fingerprint) - if bridge.added > datetime.utcnow() - timedelta(hours=3): + if bridge.added > datetime.now(tz=timezone.utc) - timedelta(hours=3): logging.debug("Not rotating a bridge less than 3 hours old") continue if bridge.deprecate(reason=self.short_name): diff --git a/app/terraform/block_mirror.py b/app/terraform/block_mirror.py index 2e8158b..6322894 100644 --- a/app/terraform/block_mirror.py +++ b/app/terraform/block_mirror.py @@ -1,9 +1,9 @@ -from collections import defaultdict -from datetime import datetime, timedelta +import fnmatch import logging from abc import abstractmethod -import fnmatch -from typing import Tuple, List, Any, Optional, Dict +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Tuple from app.extensions import db from app.models.activity import Activity @@ -41,7 +41,7 @@ class BlockMirrorAutomation(BaseAutomation): if not proxy.origin.auto_rotation: logging.debug("Proxy auto-rotation forbidden for origin") continue - if proxy.added > datetime.utcnow() - timedelta(hours=3): + if proxy.added > datetime.now(tz=timezone.utc) - timedelta(hours=3): logging.debug("Not rotating a proxy less than 3 hours old") continue if proxy.deprecate(reason=f"block_{source}"): diff --git a/app/terraform/block_ooni.py b/app/terraform/block_ooni.py index 3fdeb05..87ca310 100644 --- a/app/terraform/block_ooni.py +++ b/app/terraform/block_ooni.py @@ -1,7 +1,6 @@ from collections import defaultdict -from datetime import datetime -from datetime import timedelta -from typing import Dict, Tuple, Any +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Tuple import requests @@ -13,8 +12,8 @@ from app.terraform import BaseAutomation def check_origin(domain_name: str) -> Dict[str, Any]: - start_date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M") - end_date = datetime.utcnow().strftime("%Y-%m-%dT%H%%3A%M") + start_date = (datetime.now(tz=timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M") + end_date = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H%%3A%M") api_url = f"https://api.ooni.io/api/v1/measurements?domain={domain_name}&since={start_date}&until={end_date}" result: Dict[str, Dict[str, int]] = defaultdict(lambda: {"anomaly": 0, "confirmed": 0, "failure": 0, "ok": 0}) return _check_origin(api_url, result) diff --git a/app/terraform/bridge/__init__.py b/app/terraform/bridge/__init__.py index e18cee7..d432dd1 100644 --- a/app/terraform/bridge/__init__.py +++ b/app/terraform/bridge/__init__.py @@ -1,9 +1,9 @@ -import datetime import os import sys -from typing import Optional, Any, List, Sequence, Tuple +from datetime import datetime, timedelta, timezone +from typing import Any, List, Optional, Sequence, Tuple -from sqlalchemy import select, Row +from sqlalchemy import Row, select from app import app from app.extensions import db @@ -25,7 +25,7 @@ def active_bridges_by_provider(provider: CloudProvider) -> Sequence[BridgeResour def recently_destroyed_bridges_by_provider(provider: CloudProvider) -> Sequence[BridgeResourceRow]: - cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=72) + cutoff = datetime.now(tz=timezone.utc) - timedelta(hours=72) stmt = select(Bridge, BridgeConf, CloudAccount).join_from(Bridge, BridgeConf).join_from(Bridge, CloudAccount).where( CloudAccount.provider == provider, Bridge.destroyed.is_not(None), @@ -83,7 +83,7 @@ class BridgeAutomation(TerraformAutomation): bridge = Bridge.query.filter(Bridge.id == output[len('bridge_hashed_fingerprint_'):]).first() bridge.nickname = parts[0] bridge.hashed_fingerprint = parts[1] - bridge.terraform_updated = datetime.datetime.utcnow() + bridge.terraform_updated = datetime.now(tz=timezone.utc) if output.startswith('bridge_bridgeline_'): parts = outputs[output]['value'].split(" ") if len(parts) < 4: @@ -91,7 +91,7 @@ class BridgeAutomation(TerraformAutomation): bridge = Bridge.query.filter(Bridge.id == output[len('bridge_bridgeline_'):]).first() del parts[3] bridge.bridgeline = " ".join(parts) - bridge.terraform_updated = datetime.datetime.utcnow() + bridge.terraform_updated = datetime.now(tz=timezone.utc) db.session.commit() @classmethod diff --git a/app/terraform/bridge/meta.py b/app/terraform/bridge/meta.py index f9391c4..be46ef1 100644 --- a/app/terraform/bridge/meta.py +++ b/app/terraform/bridge/meta.py @@ -1,11 +1,11 @@ -import datetime import logging import random -from typing import Tuple, List +from datetime import datetime, timedelta, timezone +from typing import List, Tuple from app import db -from app.models.bridges import BridgeConf, Bridge, ProviderAllocation -from app.models.cloud import CloudProvider, CloudAccount +from app.models.bridges import Bridge, BridgeConf, ProviderAllocation +from app.models.cloud import CloudAccount, CloudProvider from app.terraform import BaseAutomation BRIDGE_PROVIDERS = [ @@ -33,8 +33,8 @@ def create_bridges_in_account(bridgeconf: BridgeConf, account: CloudAccount, cou bridge.pool_id = bridgeconf.pool.id bridge.conf_id = bridgeconf.id bridge.cloud_account = account - bridge.added = datetime.datetime.utcnow() - bridge.updated = datetime.datetime.utcnow() + bridge.added = datetime.now(tz=timezone.utc) + bridge.updated = datetime.now(tz=timezone.utc) logging.debug("Creating bridge %s", bridge) db.session.add(bridge) created += 1 @@ -129,7 +129,7 @@ class BridgeMetaAutomation(BaseAutomation): for bridge in deprecated_bridges: if bridge.deprecated is None: continue # Possible due to SQLAlchemy lazy loading - cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=bridge.conf.expiry_hours) + cutoff = datetime.now(tz=timezone.utc) - timedelta(hours=bridge.conf.expiry_hours) if bridge.deprecated < cutoff: logging.debug("Destroying expired bridge") bridge.destroy() diff --git a/app/terraform/eotk/aws.py b/app/terraform/eotk/aws.py index 61c771d..7d4ab1a 100644 --- a/app/terraform/eotk/aws.py +++ b/app/terraform/eotk/aws.py @@ -1,5 +1,5 @@ -import datetime import os +from datetime import datetime, timezone from typing import Any from app import app @@ -22,12 +22,12 @@ def update_eotk_instance(group_id: int, ).first() if instance is None: instance = Eotk() - instance.added = datetime.datetime.utcnow() + instance.added = datetime.now(tz=timezone.utc) instance.group_id = group_id instance.provider = "aws" instance.region = region db.session.add(instance) - instance.updated = datetime.datetime.utcnow() + instance.updated = datetime.now(tz=timezone.utc) instance.instance_id = instance_id diff --git a/app/terraform/proxy/cloudfront.py b/app/terraform/proxy/cloudfront.py index cfbf47a..bba8172 100644 --- a/app/terraform/proxy/cloudfront.py +++ b/app/terraform/proxy/cloudfront.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime, timezone from typing import Any from app.extensions import db @@ -122,7 +122,7 @@ class ProxyCloudfrontAutomation(ProxyAutomation): proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first() proxy.url = "https://" + res['values']['domain_name'] proxy.slug = res['values']['id'] - proxy.terraform_updated = datetime.datetime.utcnow() + proxy.terraform_updated = datetime.now(tz=timezone.utc) break # EC2 instances (smart proxies) for g in state["values"]["root_module"]["child_modules"]: diff --git a/app/terraform/proxy/meta.py b/app/terraform/proxy/meta.py index b6908ba..602322f 100644 --- a/app/terraform/proxy/meta.py +++ b/app/terraform/proxy/meta.py @@ -1,16 +1,17 @@ -import datetime import logging import random import string from collections import OrderedDict -from typing import Any, Dict, List, Optional, Tuple, Type +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional from typing import OrderedDict as OrderedDictT +from typing import Tuple, Type from tldextract import tldextract from app import db from app.models.base import Pool -from app.models.mirrors import Proxy, Origin +from app.models.mirrors import Origin, Proxy from app.terraform import BaseAutomation from app.terraform.proxy import ProxyAutomation from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation @@ -155,7 +156,7 @@ def auto_deprecate_proxies() -> None: .all()) logging.debug("Max age: %s", max_age_proxies) for proxy in max_age_proxies: - max_age_cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + max_age_cutoff = datetime.now(timezone.utc) - timedelta( days=1, seconds=86400 * random.random()) # nosec: B311 if proxy.added < max_age_cutoff: proxy.deprecate(reason="max_age_reached") @@ -168,7 +169,7 @@ def destroy_expired_proxies() -> None: This function finds all proxies that are not already destroyed and have been deprecated for more than 4 days. It then destroys these proxies. """ - expiry_cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=4) + expiry_cutoff = datetime.now(timezone.utc) - timedelta(days=4) proxies = Proxy.query.filter( Proxy.destroyed.is_(None), Proxy.deprecated < expiry_cutoff @@ -200,7 +201,7 @@ def promote_hot_spare_proxy(pool_id: int, origin: Origin) -> bool: if not proxy: return False proxy.pool_id = pool_id - proxy.added = datetime.datetime.utcnow() + proxy.added = datetime.now(tz=timezone.utc) return True @@ -281,8 +282,8 @@ class ProxyMetaAutomation(BaseAutomation): # The random usage below is good enough for its purpose: to create a slug that # hasn't been used recently. proxy.slug = random_slug(origin.domain_name) - proxy.added = datetime.datetime.utcnow() - proxy.updated = datetime.datetime.utcnow() + proxy.added = datetime.now(tz=timezone.utc) + proxy.updated = datetime.now(tz=timezone.utc) logging.debug("Creating proxy %s", proxy) db.session.add(proxy) return True diff --git a/app/terraform/static/aws.py b/app/terraform/static/aws.py index 26ff3bd..3286580 100644 --- a/app/terraform/static/aws.py +++ b/app/terraform/static/aws.py @@ -1,13 +1,13 @@ import logging import os -from datetime import datetime -from typing import List, Any +from datetime import datetime, timezone +from typing import Any, List from flask import current_app from app.extensions import db from app.models.base import Group -from app.models.cloud import CloudProvider, CloudAccount +from app.models.cloud import CloudAccount, CloudProvider from app.models.mirrors import StaticOrigin from app.terraform.terraform import TerraformAutomation @@ -30,7 +30,7 @@ def import_state(state: Any) -> None: static.origin_domain_name = res['values']['domain_name'] logging.debug("and found static origin: %s to update with domain name: %s", static.id, static.origin_domain_name) - static.terraform_updated = datetime.utcnow() + static.terraform_updated = datetime.now(tz=timezone.utc) break db.session.commit() diff --git a/migrations/versions/54b31e87fe33_enforce_more_not_null_constraints.py b/migrations/versions/54b31e87fe33_enforce_more_not_null_constraints.py new file mode 100644 index 0000000..459599f --- /dev/null +++ b/migrations/versions/54b31e87fe33_enforce_more_not_null_constraints.py @@ -0,0 +1,81 @@ +"""enforce more not null constraints + +Revision ID: 54b31e87fe33 +Revises: c14f25f364c5 +Create Date: 2024-12-06 16:06:59.182836 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '54b31e87fe33' +down_revision = 'c14f25f364c5' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table('activity', schema=None) as batch_op: + batch_op.alter_column('text', + existing_type=sa.TEXT(), + type_=sa.String(), + existing_nullable=False) + + with op.batch_alter_table('automation_logs', schema=None) as batch_op: + batch_op.alter_column('logs', + existing_type=sa.TEXT(), + type_=sa.String(), + nullable=False) + + with op.batch_alter_table('bridge', schema=None) as batch_op: + batch_op.alter_column('cloud_account_id', + existing_type=sa.INTEGER(), + nullable=False) + + with op.batch_alter_table('mirror_list', schema=None) as batch_op: + batch_op.alter_column('pool_id', + existing_type=sa.INTEGER(), + nullable=False) + + with op.batch_alter_table('webhook', schema=None) as batch_op: + batch_op.alter_column('format', + existing_type=sa.VARCHAR(length=20), + nullable=False) + batch_op.alter_column('url', + existing_type=sa.VARCHAR(length=255), + nullable=False) + + +def downgrade(): + with op.batch_alter_table('webhook', schema=None) as batch_op: + batch_op.alter_column('url', + existing_type=sa.VARCHAR(length=255), + nullable=True) + batch_op.alter_column('format', + existing_type=sa.VARCHAR(length=20), + nullable=True) + + with op.batch_alter_table('mirror_list', schema=None) as batch_op: + batch_op.alter_column('pool_id', + existing_type=sa.INTEGER(), + nullable=True) + + with op.batch_alter_table('bridge', schema=None) as batch_op: + batch_op.alter_column('cloud_account_id', + existing_type=sa.INTEGER(), + nullable=True) + + with op.batch_alter_table('automation_logs', schema=None) as batch_op: + batch_op.alter_column('logs', + existing_type=sa.String(), + type_=sa.TEXT(), + nullable=True) + + with op.batch_alter_table('activity', schema=None) as batch_op: + batch_op.alter_column('text', + existing_type=sa.String(), + type_=sa.TEXT(), + existing_nullable=False) +