jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: switch all timezone naive datetimes to timezone aware

Browse source
This commit is contained in:
Iain Learmonth 2024-12-06 16:08:48 +00:00
parent 41fc0a73a5
commit e22abb383c
30 changed files with 322 additions and 226 deletions
Download patch file Download diff file Expand all files Collapse all files

42
app/models/__init__.py
View file

@ -1,7 +1,7 @@
import logging
from abc import abstractmethod
from datetime import datetime
from typing import Union, List, Optional, Any, Dict
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Union
from sqlalchemy.orm import Mapped, mapped_column
@ -14,9 +14,9 @@ class AbstractConfiguration(db.Model): # type: ignore
id: Mapped[int] = mapped_column(db.Integer, primary_key=True)
description: Mapped[str]
added: Mapped[datetime]
updated: Mapped[datetime]
destroyed: Mapped[Optional[datetime]]
added: Mapped[datetime] = mapped_column(db.DateTime(timezone=True))
updated: Mapped[datetime] = mapped_column(db.DateTime(timezone=True))
destroyed: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True)
@property
@abstractmethod
@ -24,8 +24,8 @@ class AbstractConfiguration(db.Model): # type: ignore
raise NotImplementedError()
def destroy(self) -> None:
self.destroyed = datetime.utcnow()
self.updated = datetime.utcnow()
self.destroyed = datetime.now(tz=timezone.utc)
self.updated = datetime.now(tz=timezone.utc)
@classmethod
def csv_header(cls) -> List[str]:
@ -41,11 +41,11 @@ class AbstractConfiguration(db.Model): # type: ignore
class Deprecation(db.Model): # type: ignore[name-defined,misc]
id: Mapped[int] = mapped_column(db.Integer, primary_key=True)
resource_type: Mapped[str] = mapped_column(db.String(50))
resource_id: Mapped[int] = mapped_column(db.Integer)
deprecated_at: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow, nullable=False)
resource_type: Mapped[str]
resource_id: Mapped[int]
deprecated_at: Mapped[datetime] = mapped_column(db.DateTime(timezone=True))
meta: Mapped[Optional[Dict[str, Any]]] = mapped_column(db.JSON())
reason: Mapped[str] = mapped_column(db.String(), nullable=False)
reason: Mapped[str]
@property
def resource(self) -> "AbstractResource":
@ -58,11 +58,11 @@ class AbstractResource(db.Model): # type: ignore
__abstract__ = True
id: Mapped[int] = mapped_column(db.Integer, primary_key=True)
added: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow)
updated: Mapped[datetime] = mapped_column(db.DateTime(), default=datetime.utcnow)
deprecated: Mapped[Optional[datetime]]
added: Mapped[datetime] = mapped_column(db.DateTime(timezone=True))
updated: Mapped[datetime] = mapped_column(db.DateTime(timezone=True))
deprecated: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True)
deprecation_reason: Mapped[Optional[str]]
destroyed: Mapped[Optional[datetime]]
destroyed: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True)
def __init__(self, *,
id: Optional[int] = None,
@ -73,9 +73,9 @@ class AbstractResource(db.Model): # type: ignore
destroyed: Optional[datetime] = None,
**kwargs: Any) -> None:
if added is None:
added = datetime.utcnow()
added = datetime.now(tz=timezone.utc)
if updated is None:
updated = datetime.utcnow()
updated = datetime.now(tz=timezone.utc)
super().__init__(id=id,
added=added,
updated=updated,
@ -101,9 +101,9 @@ class AbstractResource(db.Model): # type: ignore
"""
if self.deprecated is None:
logging.info("Deprecating %s (reason=%s)", self.brn, reason)
self.deprecated = datetime.utcnow()
self.deprecated = datetime.now(tz=timezone.utc)
self.deprecation_reason = reason
self.updated = datetime.utcnow()
self.updated = datetime.now(tz=timezone.utc)
if reason not in [d.reason for d in self.deprecations]:
new_deprecation = Deprecation(
resource_type=type(self).__name__,
@ -132,8 +132,8 @@ class AbstractResource(db.Model): # type: ignore
"""
if self.deprecated is None:
self.deprecate(reason="destroyed")
self.destroyed = datetime.utcnow()
self.updated = datetime.utcnow()
self.destroyed = datetime.now(tz=timezone.utc)
self.updated = datetime.now(tz=timezone.utc)
@classmethod
def csv_header(cls) -> List[str]:

25
app/models/activity.py
View file

@ -1,39 +1,40 @@
import datetime
from datetime import datetime, timezone
from typing import Any, Optional
import requests
from sqlalchemy.orm import Mapped, mapped_column
from app.brm.brn import BRN
from app.models import AbstractConfiguration
from app.extensions import db
from app.models import AbstractConfiguration
class Activity(db.Model): # type: ignore
id = db.Column(db.Integer(), primary_key=True)
group_id = db.Column(db.Integer(), nullable=True)
activity_type = db.Column(db.String(20), nullable=False)
text = db.Column(db.Text(), nullable=False)
added = db.Column(db.DateTime(), nullable=False)
id: Mapped[int] = mapped_column(primary_key=True)
group_id: Mapped[Optional[int]]
activity_type: Mapped[str]
text: Mapped[str]
added: Mapped[datetime] = mapped_column(db.DateTime(timezone=True))
def __init__(self, *,
id: Optional[int] = None,
group_id: Optional[int] = None,
activity_type: str,
text: str,
added: Optional[datetime.datetime] = None,
added: Optional[datetime] = None,
**kwargs: Any) -> None:
if not isinstance(activity_type, str) or len(activity_type) > 20 or activity_type == "":
raise TypeError("expected string for activity type between 1 and 20 characters")
if not isinstance(text, str):
raise TypeError("expected string for text")
if added is None:
added = datetime.now(tz=timezone.utc)
super().__init__(id=id,
group_id=group_id,
activity_type=activity_type,
text=text,
added=added,
**kwargs)
if self.added is None:
self.added = datetime.datetime.utcnow()
def notify(self) -> int:
count = 0
@ -47,8 +48,8 @@ class Activity(db.Model): # type: ignore
class Webhook(AbstractConfiguration):
format = db.Column(db.String(20))
url = db.Column(db.String(255))
format: Mapped[str]
url: Mapped[str]
@property
def brn(self) -> BRN:

27
app/models/alarms.py
View file

@ -1,6 +1,8 @@
import enum
from datetime import datetime
from typing import List, Any
from datetime import datetime, timezone
from typing import Any, List
from sqlalchemy.orm import Mapped, mapped_column
from app.extensions import db
from app.models.activity import Activity
@ -24,13 +26,13 @@ class AlarmState(enum.Enum):
class Alarm(db.Model): # type: ignore
id = db.Column(db.Integer, primary_key=True)
target = db.Column(db.String(255), nullable=False)
aspect = db.Column(db.String(255), nullable=False)
alarm_state = db.Column(db.Enum(AlarmState), default=AlarmState.UNKNOWN, nullable=False)
state_changed = db.Column(db.DateTime(), nullable=False)
last_updated = db.Column(db.DateTime(), nullable=False)
text = db.Column(db.String(255), nullable=False)
id: Mapped[int] = mapped_column(db.Integer, primary_key=True)
target: Mapped[str]
aspect: Mapped[str]
alarm_state: Mapped[AlarmState] = mapped_column(default=AlarmState.UNKNOWN)
state_changed: Mapped[datetime] = mapped_column(db.DateTime(timezone=True))
last_updated: Mapped[datetime] = mapped_column(db.DateTime(timezone=True))
text: Mapped[str]
@classmethod
def csv_header(cls) -> List[str]:
@ -40,11 +42,8 @@ class Alarm(db.Model): # type: ignore
return [getattr(self, x) for x in self.csv_header()]
def update_state(self, state: AlarmState, text: str) -> None:
if self.alarm_state is None:
self.alarm_state = AlarmState.UNKNOWN
if self.alarm_state != state or self.state_changed is None:
self.state_changed = datetime.utcnow()
self.state_changed = datetime.now(tz=timezone.utc)
activity = Activity(activity_type="alarm_state",
text=f"[{self.aspect}] {state.emoji} Alarm state changed from "
f"{self.alarm_state.name} to {state.name} on {self.target}: {text}.")
@ -56,4 +55,4 @@ class Alarm(db.Model): # type: ignore
db.session.add(activity)
self.alarm_state = state
self.text = text
self.last_updated = datetime.utcnow()
self.last_updated = datetime.now(tz=timezone.utc)

25
app/models/automation.py
View file

@ -1,5 +1,8 @@
import datetime
import enum
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy.orm import Mapped, mapped_column
from app.brm.brn import BRN
from app.extensions import db
@ -13,12 +16,12 @@ class AutomationState(enum.Enum):
class Automation(AbstractConfiguration):
short_name = db.Column(db.String(25), nullable=False)
state = db.Column(db.Enum(AutomationState), default=AutomationState.IDLE, nullable=False)
enabled = db.Column(db.Boolean, nullable=False)
last_run = db.Column(db.DateTime(), nullable=True)
next_run = db.Column(db.DateTime(), nullable=True)
next_is_full = db.Column(db.Boolean(), nullable=False)
short_name: Mapped[str]
state: Mapped[AutomationState] = mapped_column(default=AutomationState.IDLE)
enabled: Mapped[bool]
last_run: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True))
next_run: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True))
next_is_full: Mapped[bool]
logs = db.relationship("AutomationLogs", back_populates="automation")
@ -34,13 +37,13 @@ class Automation(AbstractConfiguration):
def kick(self) -> None:
self.enabled = True
self.next_run = datetime.datetime.utcnow()
self.updated = datetime.datetime.utcnow()
self.next_run = datetime.now(tz=timezone.utc)
self.updated = datetime.now(tz=timezone.utc)
class AutomationLogs(AbstractResource):
automation_id = db.Column(db.Integer, db.ForeignKey(Automation.id), nullable=False)
logs = db.Column(db.Text)
automation_id: Mapped[int] = mapped_column(db.ForeignKey("automation.id"))
logs: Mapped[str]
automation = db.relationship("Automation", back_populates="logs")

30
app/models/base.py
View file

@ -1,5 +1,5 @@
from datetime import datetime
from typing import List, TypedDict, Optional, TYPE_CHECKING
from datetime import datetime, timezone
from typing import TYPE_CHECKING, List, Optional, TypedDict
from sqlalchemy import and_
from sqlalchemy.orm import Mapped, aliased, mapped_column, relationship
@ -22,7 +22,7 @@ class GroupDict(TypedDict):
class Group(AbstractConfiguration):
group_name: Mapped[str] = db.Column(db.String(80), unique=True, nullable=False)
group_name: Mapped[str] = mapped_column(unique=True)
eotk: Mapped[bool]
origins: Mapped[List["Origin"]] = relationship("Origin", back_populates="group")
@ -93,19 +93,19 @@ class Pool(AbstractConfiguration):
class PoolGroup(db.Model): # type: ignore[name-defined,misc]
pool_id = db.Column(db.Integer, db.ForeignKey("pool.id"), primary_key=True)
group_id = db.Column(db.Integer, db.ForeignKey("group.id"), primary_key=True)
pool_id: Mapped[int] = mapped_column(db.ForeignKey("pool.id"), primary_key=True)
group_id: Mapped[int] = mapped_column(db.ForeignKey("group.id"), primary_key=True)
class MirrorList(AbstractConfiguration):
pool_id = db.Column(db.Integer, db.ForeignKey("pool.id"))
provider = db.Column(db.String(255), nullable=False)
format = db.Column(db.String(20), nullable=False)
encoding = db.Column(db.String(20), nullable=False)
container = db.Column(db.String(255), nullable=False)
branch = db.Column(db.String(255), nullable=False)
role = db.Column(db.String(255), nullable=True)
filename = db.Column(db.String(255), nullable=False)
pool_id: Mapped[int] = mapped_column(db.ForeignKey("pool.id"))
provider: Mapped[str]
format: Mapped[str]
encoding: Mapped[str]
container: Mapped[str]
branch: Mapped[str]
role: Mapped[Optional[str]]
filename: Mapped[str]
pool = db.relationship("Pool", back_populates="lists")
@ -132,8 +132,8 @@ class MirrorList(AbstractConfiguration):
}
def destroy(self) -> None:
self.destroyed = datetime.utcnow()
self.updated = datetime.utcnow()
self.destroyed = datetime.now(tz=timezone.utc)
self.updated = datetime.now(tz=timezone.utc)
def url(self) -> str:
if self.provider == "gitlab":

26
app/models/bridges.py
View file

@ -1,6 +1,6 @@
import enum
from datetime import datetime
from typing import List
from datetime import datetime, timezone
from typing import List, Optional
from sqlalchemy.orm import Mapped, mapped_column, relationship
@ -37,12 +37,12 @@ class BridgeConf(AbstractConfiguration):
)
def destroy(self) -> None:
self.destroyed = datetime.utcnow()
self.updated = datetime.utcnow()
self.destroyed = datetime.now(tz=timezone.utc)
self.updated = datetime.now(tz=timezone.utc)
for bridge in self.bridges:
if bridge.destroyed is None:
bridge.destroyed = datetime.utcnow()
bridge.updated = datetime.utcnow()
bridge.destroyed = datetime.now(tz=timezone.utc)
bridge.updated = datetime.now(tz=timezone.utc)
@classmethod
def csv_header(cls) -> List[str]:
@ -52,13 +52,13 @@ class BridgeConf(AbstractConfiguration):
class Bridge(AbstractResource):
conf_id = db.Column(db.Integer, db.ForeignKey("bridge_conf.id"), nullable=False)
cloud_account_id = db.Column(db.Integer, db.ForeignKey("cloud_account.id"))
terraform_updated = db.Column(db.DateTime(), nullable=True)
nickname = db.Column(db.String(255), nullable=True)
fingerprint = db.Column(db.String(255), nullable=True)
hashed_fingerprint = db.Column(db.String(255), nullable=True)
bridgeline = db.Column(db.String(255), nullable=True)
conf_id: Mapped[int] = mapped_column(db.ForeignKey("bridge_conf.id"))
cloud_account_id: Mapped[int] = mapped_column(db.ForeignKey("cloud_account.id"))
terraform_updated: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True)
nickname: Mapped[Optional[str]]
fingerprint: Mapped[Optional[str]]
hashed_fingerprint: Mapped[Optional[str]]
bridgeline: Mapped[Optional[str]]
conf = db.relationship("BridgeConf", back_populates="bridges")
cloud_account = db.relationship("CloudAccount", back_populates="bridges")

21
app/models/mirrors.py
View file

@ -1,8 +1,8 @@
from __future__ import annotations
import json
from datetime import datetime, timedelta
from typing import Optional, List, Union, Any, Dict, TypedDict, Literal
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
import tldextract
from sqlalchemy.orm import Mapped, mapped_column, relationship
@ -10,10 +10,11 @@ from tldextract import extract
from werkzeug.datastructures import FileStorage
from app.brm.brn import BRN
from app.brm.utils import thumbnail_uploaded_image, create_data_uri, normalize_color
from app.brm.utils import (create_data_uri, normalize_color,
thumbnail_uploaded_image)
from app.extensions import db
from app.models import AbstractConfiguration, AbstractResource, Deprecation
from app.models.base import Pool, Group
from app.models.base import Group, Pool
from app.models.onions import Onion
country_origin = db.Table(
@ -93,14 +94,14 @@ class Origin(AbstractConfiguration):
.filter(
Origin.id == self.id,
Deprecation.resource_type == 'Proxy',
Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168),
Deprecation.deprecated_at >= datetime.now(tz=timezone.utc) - timedelta(hours=168),
Deprecation.reason != "destroyed"
)
.distinct(Proxy.id)
.all()
)
for deprecation in recent_deprecations:
recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1)
recency_factor += 1 / max((datetime.now(tz=timezone.utc) - deprecation.deprecated_at).total_seconds() // 3600, 1)
frequency_factor += 1
risk_levels: Dict[str, int] = {}
for country in self.countries:
@ -149,14 +150,14 @@ class Country(AbstractConfiguration):
.filter(
Country.id == self.id,
Deprecation.resource_type == 'Proxy',
Deprecation.deprecated_at >= datetime.utcnow() - timedelta(hours=168),
Deprecation.deprecated_at >= datetime.now(tz=timezone.utc) - timedelta(hours=168),
Deprecation.reason != "destroyed"
)
.distinct(Proxy.id)
.all()
)
for deprecation in recent_deprecations:
recency_factor += 1 / max((datetime.utcnow() - deprecation.deprecated_at).total_seconds() // 3600, 1)
recency_factor += 1 / max((datetime.now(tz=timezone.utc) - deprecation.deprecated_at).total_seconds() // 3600, 1)
frequency_factor += 1
return int(max(1, min(10, frequency_factor * recency_factor)))
@ -255,7 +256,7 @@ class StaticOrigin(AbstractConfiguration):
raise ValueError("clean_insights_backend must be a str, bool, or None")
if db_session_commit:
db.session.commit()
self.updated = datetime.utcnow()
self.updated = datetime.now(tz=timezone.utc)
ResourceStatus = Union[Literal["active"], Literal["pending"], Literal["expiring"], Literal["destroyed"]]
@ -274,7 +275,7 @@ class Proxy(AbstractResource):
provider: Mapped[str] = mapped_column(db.String(20), nullable=False)
psg: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
slug: Mapped[Optional[str]] = mapped_column(db.String(20), nullable=True)
terraform_updated: Mapped[Optional[datetime]] = mapped_column(db.DateTime(), nullable=True)
terraform_updated: Mapped[Optional[datetime]] = mapped_column(db.DateTime(timezone=True), nullable=True)
url: Mapped[Optional[str]] = mapped_column(db.String(255), nullable=True)
origin: Mapped[Origin] = relationship("Origin", back_populates="proxies")