jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: switch all timezone naive datetimes to timezone aware

Browse source
This commit is contained in:
Iain Learmonth 2024-12-06 16:08:48 +00:00
parent 41fc0a73a5
commit e22abb383c
30 changed files with 322 additions and 226 deletions
Download patch file Download diff file Expand all files Collapse all files

8
app/terraform/block/bridge.py
View file

@ -1,7 +1,7 @@
from datetime import datetime, timedelta
import logging
from abc import abstractmethod
from typing import Tuple, List, Callable, Optional, Any
from datetime import datetime, timedelta, timezone
from typing import Any, Callable, List, Optional, Tuple
from app.extensions import db
from app.models.activity import Activity
@ -25,14 +25,14 @@ class BlockBridgeAutomation(BaseAutomation):
super().__init__(*args, **kwargs)
def perform_deprecations(self, ids: List[str], bridge_select_func: Callable[[str], Optional[Bridge]]
) -> List[Tuple[str, str, str]]:
) -> List[Tuple[Optional[str], Any, Any]]:
rotated = []
for id_ in ids:
bridge = bridge_select_func(id_)
if bridge is None:
continue
logging.debug("Found %s blocked", bridge.hashed_fingerprint)
if bridge.added > datetime.utcnow() - timedelta(hours=3):
if bridge.added > datetime.now(tz=timezone.utc) - timedelta(hours=3):
logging.debug("Not rotating a bridge less than 3 hours old")
continue
if bridge.deprecate(reason=self.short_name):

10
app/terraform/block_mirror.py
View file

@ -1,9 +1,9 @@
from collections import defaultdict
from datetime import datetime, timedelta
import fnmatch
import logging
from abc import abstractmethod
import fnmatch
from typing import Tuple, List, Any, Optional, Dict
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Tuple
from app.extensions import db
from app.models.activity import Activity
@ -41,7 +41,7 @@ class BlockMirrorAutomation(BaseAutomation):
if not proxy.origin.auto_rotation:
logging.debug("Proxy auto-rotation forbidden for origin")
continue
if proxy.added > datetime.utcnow() - timedelta(hours=3):
if proxy.added > datetime.now(tz=timezone.utc) - timedelta(hours=3):
logging.debug("Not rotating a proxy less than 3 hours old")
continue
if proxy.deprecate(reason=f"block_{source}"):

9
app/terraform/block_ooni.py
View file

@ -1,7 +1,6 @@
from collections import defaultdict
from datetime import datetime
from datetime import timedelta
from typing import Dict, Tuple, Any
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Tuple
import requests
@ -13,8 +12,8 @@ from app.terraform import BaseAutomation
def check_origin(domain_name: str) -> Dict[str, Any]:
start_date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M")
end_date = datetime.utcnow().strftime("%Y-%m-%dT%H%%3A%M")
start_date = (datetime.now(tz=timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M")
end_date = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H%%3A%M")
api_url = f"https://api.ooni.io/api/v1/measurements?domain={domain_name}&since={start_date}&until={end_date}"
result: Dict[str, Dict[str, int]] = defaultdict(lambda: {"anomaly": 0, "confirmed": 0, "failure": 0, "ok": 0})
return _check_origin(api_url, result)

12
app/terraform/bridge/__init__.py
View file

@ -1,9 +1,9 @@
import datetime
import os
import sys
from typing import Optional, Any, List, Sequence, Tuple
from datetime import datetime, timedelta, timezone
from typing import Any, List, Optional, Sequence, Tuple
from sqlalchemy import select, Row
from sqlalchemy import Row, select
from app import app
from app.extensions import db
@ -25,7 +25,7 @@ def active_bridges_by_provider(provider: CloudProvider) -> Sequence[BridgeResour
def recently_destroyed_bridges_by_provider(provider: CloudProvider) -> Sequence[BridgeResourceRow]:
cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=72)
cutoff = datetime.now(tz=timezone.utc) - timedelta(hours=72)
stmt = select(Bridge, BridgeConf, CloudAccount).join_from(Bridge, BridgeConf).join_from(Bridge, CloudAccount).where(
CloudAccount.provider == provider,
Bridge.destroyed.is_not(None),
@ -83,7 +83,7 @@ class BridgeAutomation(TerraformAutomation):
bridge = Bridge.query.filter(Bridge.id == output[len('bridge_hashed_fingerprint_'):]).first()
bridge.nickname = parts[0]
bridge.hashed_fingerprint = parts[1]
bridge.terraform_updated = datetime.datetime.utcnow()
bridge.terraform_updated = datetime.now(tz=timezone.utc)
if output.startswith('bridge_bridgeline_'):
parts = outputs[output]['value'].split(" ")
if len(parts) < 4:
@ -91,7 +91,7 @@ class BridgeAutomation(TerraformAutomation):
bridge = Bridge.query.filter(Bridge.id == output[len('bridge_bridgeline_'):]).first()
del parts[3]
bridge.bridgeline = " ".join(parts)
bridge.terraform_updated = datetime.datetime.utcnow()
bridge.terraform_updated = datetime.now(tz=timezone.utc)
db.session.commit()
@classmethod

14
app/terraform/bridge/meta.py
View file

@ -1,11 +1,11 @@
import datetime
import logging
import random
from typing import Tuple, List
from datetime import datetime, timedelta, timezone
from typing import List, Tuple
from app import db
from app.models.bridges import BridgeConf, Bridge, ProviderAllocation
from app.models.cloud import CloudProvider, CloudAccount
from app.models.bridges import Bridge, BridgeConf, ProviderAllocation
from app.models.cloud import CloudAccount, CloudProvider
from app.terraform import BaseAutomation
BRIDGE_PROVIDERS = [
@ -33,8 +33,8 @@ def create_bridges_in_account(bridgeconf: BridgeConf, account: CloudAccount, cou
bridge.pool_id = bridgeconf.pool.id
bridge.conf_id = bridgeconf.id
bridge.cloud_account = account
bridge.added = datetime.datetime.utcnow()
bridge.updated = datetime.datetime.utcnow()
bridge.added = datetime.now(tz=timezone.utc)
bridge.updated = datetime.now(tz=timezone.utc)
logging.debug("Creating bridge %s", bridge)
db.session.add(bridge)
created += 1
@ -129,7 +129,7 @@ class BridgeMetaAutomation(BaseAutomation):
for bridge in deprecated_bridges:
if bridge.deprecated is None:
continue # Possible due to SQLAlchemy lazy loading
cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=bridge.conf.expiry_hours)
cutoff = datetime.now(tz=timezone.utc) - timedelta(hours=bridge.conf.expiry_hours)
if bridge.deprecated < cutoff:
logging.debug("Destroying expired bridge")
bridge.destroy()

6
app/terraform/eotk/aws.py
View file

@ -1,5 +1,5 @@
import datetime
import os
from datetime import datetime, timezone
from typing import Any
from app import app
@ -22,12 +22,12 @@ def update_eotk_instance(group_id: int,
).first()
if instance is None:
instance = Eotk()
instance.added = datetime.datetime.utcnow()
instance.added = datetime.now(tz=timezone.utc)
instance.group_id = group_id
instance.provider = "aws"
instance.region = region
db.session.add(instance)
instance.updated = datetime.datetime.utcnow()
instance.updated = datetime.now(tz=timezone.utc)
instance.instance_id = instance_id

4
app/terraform/proxy/cloudfront.py
View file

@ -1,4 +1,4 @@
import datetime
from datetime import datetime, timezone
from typing import Any
from app.extensions import db
@ -122,7 +122,7 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
proxy = Proxy.query.filter(Proxy.id == mod['address'][len('module.cloudfront_'):]).first()
proxy.url = "https://" + res['values']['domain_name']
proxy.slug = res['values']['id']
proxy.terraform_updated = datetime.datetime.utcnow()
proxy.terraform_updated = datetime.now(tz=timezone.utc)
break
# EC2 instances (smart proxies)
for g in state["values"]["root_module"]["child_modules"]:

17
app/terraform/proxy/meta.py
View file

@ -1,16 +1,17 @@
import datetime
import logging
import random
import string
from collections import OrderedDict
from typing import Any, Dict, List, Optional, Tuple, Type
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from typing import OrderedDict as OrderedDictT
from typing import Tuple, Type
from tldextract import tldextract
from app import db
from app.models.base import Pool
from app.models.mirrors import Proxy, Origin
from app.models.mirrors import Origin, Proxy
from app.terraform import BaseAutomation
from app.terraform.proxy import ProxyAutomation
from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
@ -155,7 +156,7 @@ def auto_deprecate_proxies() -> None:
.all())
logging.debug("Max age: %s", max_age_proxies)
for proxy in max_age_proxies:
max_age_cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
max_age_cutoff = datetime.now(timezone.utc) - timedelta(
days=1, seconds=86400 * random.random()) # nosec: B311
if proxy.added < max_age_cutoff:
proxy.deprecate(reason="max_age_reached")
@ -168,7 +169,7 @@ def destroy_expired_proxies() -> None:
This function finds all proxies that are not already destroyed and have been deprecated for more than 4 days.
It then destroys these proxies.
"""
expiry_cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=4)
expiry_cutoff = datetime.now(timezone.utc) - timedelta(days=4)
proxies = Proxy.query.filter(
Proxy.destroyed.is_(None),
Proxy.deprecated < expiry_cutoff
@ -200,7 +201,7 @@ def promote_hot_spare_proxy(pool_id: int, origin: Origin) -> bool:
if not proxy:
return False
proxy.pool_id = pool_id
proxy.added = datetime.datetime.utcnow()
proxy.added = datetime.now(tz=timezone.utc)
return True
@ -281,8 +282,8 @@ class ProxyMetaAutomation(BaseAutomation):
# The random usage below is good enough for its purpose: to create a slug that
# hasn't been used recently.
proxy.slug = random_slug(origin.domain_name)
proxy.added = datetime.datetime.utcnow()
proxy.updated = datetime.datetime.utcnow()
proxy.added = datetime.now(tz=timezone.utc)
proxy.updated = datetime.now(tz=timezone.utc)
logging.debug("Creating proxy %s", proxy)
db.session.add(proxy)
return True

8
app/terraform/static/aws.py
View file

@ -1,13 +1,13 @@
import logging
import os
from datetime import datetime
from typing import List, Any
from datetime import datetime, timezone
from typing import Any, List
from flask import current_app
from app.extensions import db
from app.models.base import Group
from app.models.cloud import CloudProvider, CloudAccount
from app.models.cloud import CloudAccount, CloudProvider
from app.models.mirrors import StaticOrigin
from app.terraform.terraform import TerraformAutomation
@ -30,7 +30,7 @@ def import_state(state: Any) -> None:
static.origin_domain_name = res['values']['domain_name']
logging.debug("and found static origin: %s to update with domain name: %s", static.id,
static.origin_domain_name)
static.terraform_updated = datetime.utcnow()
static.terraform_updated = datetime.now(tz=timezone.utc)
break
db.session.commit()