jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lots of typing fixes

Browse source
This commit is contained in:
Iain Learmonth 2022-05-16 11:44:03 +01:00
parent 51f580a304
commit 3665c34961
43 changed files with 260 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/terraform/__init__.py
View file

@ -1,6 +1,6 @@
from abc import ABCMeta, abstractmethod
import os
from typing import Tuple
from typing import Tuple, Optional
from app import app
@ -19,7 +19,7 @@ class BaseAutomation(metaclass=ABCMeta):
def automate(self, full: bool = False) -> Tuple[bool, str]:
raise NotImplementedError()
def working_directory(self, filename=None) -> str:
def working_directory(self, filename: Optional[str] = None) -> str:
"""
Provides a filesystem path that can be used during the automation run.
This is currently a persistent path, but this should not be relied upon

6
app/terraform/alarms/proxy_azure_cdn.py
View file

@ -1,3 +1,5 @@
from typing import Tuple
from azure.identity import ClientSecretCredential
from azure.mgmt.alertsmanagement import AlertsManagementClient
@ -12,7 +14,7 @@ class AlarmProxyAzureCdnAutomation(BaseAutomation):
short_name = "monitor_proxy_azure_cdn"
description = "Import alarms for Azure CDN proxies"
def automate(self):
def automate(self, full: bool = False) -> Tuple[bool, str]:
credential = ClientSecretCredential(
tenant_id=app.config['AZURE_TENANT_ID'],
client_id=app.config['AZURE_CLIENT_ID'],
@ -33,4 +35,4 @@ class AlarmProxyAzureCdnAutomation(BaseAutomation):
alarm.update_state(AlarmState.OK, "Azure monitor alert not firing")
else:
alarm.update_state(AlarmState.CRITICAL, "Azure monitor alert firing")
return True, []
return True, ""

7
app/terraform/alarms/proxy_cloudfront.py
View file

@ -1,4 +1,5 @@
import datetime
from typing import Tuple
import boto3
@ -14,7 +15,7 @@ class AlarmProxyCloudfrontAutomation(BaseAutomation):
short_name = "monitor_proxy_cloudfront"
description = "Import alarms for AWS CloudFront proxies"
def automate(self):
def automate(self, full: bool = False) -> Tuple[bool, str]:
cloudwatch = boto3.client('cloudwatch',
aws_access_key_id=app.config['AWS_ACCESS_KEY'],
aws_secret_access_key=app.config['AWS_SECRET_KEY'],
@ -39,7 +40,7 @@ class AlarmProxyCloudfrontAutomation(BaseAutomation):
Alarm.alarm_type == "cloudfront-quota"
).first()
if alarm is None:
alarm = Alarm()
alarm = Alarm() # type: ignore
alarm.target = "service/cloudfront"
alarm.alarm_type = "cloudfront-quota"
alarm.state_changed = datetime.datetime.utcnow()
@ -57,4 +58,4 @@ class AlarmProxyCloudfrontAutomation(BaseAutomation):
if alarm.alarm_state != old_state:
alarm.state_changed = datetime.datetime.utcnow()
db.session.commit()
return True, []
return True, ""

2
app/terraform/alarms/proxy_http_status.py
View file

@ -8,7 +8,7 @@ from app.models.mirrors import Proxy
from app.terraform import BaseAutomation
def set_http_alarm(proxy_id: int, state: AlarmState, text: str):
def set_http_alarm(proxy_id: int, state: AlarmState, text: str) -> None:
alarm = Alarm.query.filter(
Alarm.proxy_id == proxy_id,
Alarm.alarm_type == "http-status"

5
app/terraform/block_bridge_github.py
View file

@ -19,7 +19,10 @@ class BlockBridgeGitHubAutomation(BaseAutomation):
g = Github(app.config['GITHUB_API_KEY'])
repo = g.get_repo(app.config['GITHUB_BRIDGE_REPO'])
for vp in app.config['GITHUB_BRIDGE_VANTAGE_POINTS']:
results = repo.get_contents(f"recentResult_{vp}").decoded_content.decode('utf-8').splitlines()
contents = repo.get_contents(f"recentResult_{vp}")
if isinstance(contents, list):
return False, f"Expected a file at recentResult_{vp} but got a directory."
results = contents.decoded_content.decode('utf-8').splitlines()
for result in results:
parts = result.split("\t")
if isoparse(parts[2]) < (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=3)):

2
app/terraform/block_external.py
View file

@ -58,7 +58,7 @@ class BlockExternalAutomation(BaseAutomation):
continue
activities.append(Activity(
activity_type="block",
text=(f"Proxy {p.url} for {p.origin.domain_name} detected blocked according to external source. "
text=(f"Proxy {proxy.url} for {proxy.origin.domain_name} detected blocked according to external source. "
"Rotation scheduled.")
))
proxy.deprecate(reason="external")

10
app/terraform/block_ooni.py
View file

@ -1,7 +1,7 @@
from collections import defaultdict
from datetime import datetime
from datetime import timedelta
from typing import Dict, Tuple
from typing import Dict, Tuple, Union, Any
import requests
@ -11,7 +11,7 @@ from app.models.mirrors import Origin
from app.terraform import BaseAutomation
def check_origin(domain_name: str):
def check_origin(domain_name: str) -> Dict[str, Any]:
start_date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%dT%H%%3A%M")
end_date = datetime.utcnow().strftime("%Y-%m-%dT%H%%3A%M")
api_url = f"https://api.ooni.io/api/v1/measurements?domain={domain_name}&since={start_date}&until={end_date}"
@ -19,7 +19,7 @@ def check_origin(domain_name: str):
return _check_origin(api_url, result)
def _check_origin(api_url: str, result: Dict):
def _check_origin(api_url: str, result: Dict[str, Any]) -> Dict[str, Any]:
print(f"Processing {api_url}")
req = requests.get(api_url).json()
if 'results' not in req or not req['results']:
@ -38,7 +38,7 @@ def _check_origin(api_url: str, result: Dict):
return result
def threshold_origin(domain_name):
def threshold_origin(domain_name: str) -> Dict[str, Any]:
ooni = check_origin(domain_name)
for country in ooni:
total = sum([
@ -58,7 +58,7 @@ def threshold_origin(domain_name):
return ooni
def set_ooni_alarm(origin_id: int, country: str, state: AlarmState, text: str):
def set_ooni_alarm(origin_id: int, country: str, state: AlarmState, text: str) -> None:
alarm = Alarm.query.filter(
Alarm.origin_id == origin_id,
Alarm.alarm_type == f"origin-block-ooni-{country}"

20
app/terraform/bridge/__init__.py
View file

@ -1,5 +1,5 @@
import datetime
from typing import Iterable, Optional, Any
from typing import Iterable, Optional, Any, List
from app import app
from app.extensions import db
@ -9,7 +9,18 @@ from app.terraform.terraform import TerraformAutomation
class BridgeAutomation(TerraformAutomation):
def create_missing(self):
template: str
"""
Terraform configuration template using Jinja 2.
"""
template_parameters: List[str]
"""
List of parameters to be read from the application configuration for use
in the templating of the Terraform configuration.
"""
def create_missing(self) -> None:
bridgeconfs: Iterable[BridgeConf] = BridgeConf.query.filter(
BridgeConf.provider == self.provider,
BridgeConf.destroyed == None
@ -35,7 +46,7 @@ class BridgeAutomation(TerraformAutomation):
break
db.session.commit()
def destroy_expired(self):
def destroy_expired(self) -> None:
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=0)
bridges = [b for b in Bridge.query.filter(
Bridge.destroyed == None,
@ -48,8 +59,9 @@ class BridgeAutomation(TerraformAutomation):
def tf_prehook(self) -> Optional[Any]:
self.create_missing()
self.destroy_expired()
return None
def tf_generate(self):
def tf_generate(self) -> None:
self.tf_write(
self.template,
groups=Group.query.all(),

6
app/terraform/eotk/aws.py
View file

@ -8,7 +8,9 @@ from app.models.onions import Eotk
from app.terraform.terraform import TerraformAutomation
def update_eotk_instance(group_id: int, region: str, instance_id: str):
def update_eotk_instance(group_id: int,
region: str,
instance_id: str) -> None:
instance = Eotk.query.filter(
Eotk.group_id == group_id,
Eotk.region == region,
@ -74,7 +76,7 @@ class EotkAWSAutomation(TerraformAutomation):
{% endfor %}
"""
def tf_generate(self):
def tf_generate(self) -> None:
self.tf_write(
self.template,
groups=Group.query.filter(

14
app/terraform/list/__init__.py
View file

@ -1,4 +1,5 @@
import json
from typing import List
from app import app
from app.lists.mirror_mapping import mirror_mapping
@ -9,7 +10,18 @@ from app.terraform.terraform import TerraformAutomation
class ListAutomation(TerraformAutomation):
def tf_generate(self):
template: str
"""
Terraform configuration template using Jinja 2.
"""
template_parameters: List[str]
"""
List of parameters to be read from the application configuration for use
in the templating of the Terraform configuration.
"""
def tf_generate(self) -> None:
self.tf_write(
self.template,
lists=MirrorList.query.filter(

38
app/terraform/proxy/__init__.py
View file

@ -1,9 +1,10 @@
from abc import abstractmethod
from collections import defaultdict
import datetime
import math
import string
import random
from typing import Dict
from typing import Dict, Optional, Any, List
from sqlalchemy import text
from tldextract import tldextract
@ -17,6 +18,22 @@ from app.terraform.terraform import TerraformAutomation
class ProxyAutomation(TerraformAutomation):
subgroup_max = math.inf
"""
Maximum number of proxies to deploy per sub-group. This is required for some providers
where the number origins per group may exceed the number of proxies that can be configured
in a single "configuration block", e.g. Azure CDN's profiles.
"""
template: str
"""
Terraform configuration template using Jinja 2.
"""
template_parameters: List[str]
"""
List of parameters to be read from the application configuration for use
in the templating of the Terraform configuration.
"""
def get_subgroups(self) -> Dict[int, Dict[int, int]]:
conn = db.engine.connect()
@ -27,12 +44,12 @@ class ProxyAutomation(TerraformAutomation):
AND proxy.provider = :provider
GROUP BY origin.group_id, proxy.psg;
"""), provider=self.provider)
subgroups = defaultdict(lambda: defaultdict(lambda: 0))
subgroups: Dict[int, Dict[int, int]] = defaultdict(lambda: defaultdict(lambda: 0))
for row in result:
subgroups[row[0]][row[1]] = row[2]
return subgroups
def create_missing_proxies(self):
def create_missing_proxies(self) -> None:
groups = Group.query.all()
subgroups = self.get_subgroups()
for group in groups:
@ -62,7 +79,7 @@ class ProxyAutomation(TerraformAutomation):
db.session.add(proxy)
db.session.commit()
def deprecate_orphaned_proxies(self):
def deprecate_orphaned_proxies(self) -> None:
proxies = Proxy.query.filter(
Proxy.deprecated == None,
Proxy.destroyed == None,
@ -73,7 +90,7 @@ class ProxyAutomation(TerraformAutomation):
proxy.deprecate(reason="origin_destroyed")
db.session.commit()
def destroy_expired_proxies(self):
def destroy_expired_proxies(self) -> None:
cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
proxies = Proxy.query.filter(
Proxy.destroyed == None,
@ -85,15 +102,20 @@ class ProxyAutomation(TerraformAutomation):
proxy.updated = datetime.datetime.utcnow()
db.session.commit()
def tf_prehook(self):
@abstractmethod
def import_state(self, state: Any) -> None:
raise NotImplementedError()
def tf_prehook(self) -> Optional[Any]:
self.create_missing_proxies()
self.deprecate_orphaned_proxies()
self.destroy_expired_proxies()
return None
def tf_posthook(self, *, prehook_result):
def tf_posthook(self, *, prehook_result: Any = None) -> None:
self.import_state(self.tf_show())
def tf_generate(self):
def tf_generate(self) -> None:
self.tf_write(
self.template,
groups=Group.query.all(),

4
app/terraform/proxy/azure_cdn.py
View file

@ -1,3 +1,5 @@
from typing import Optional, Any
from app.extensions import db
from app.models.mirrors import Proxy
from app.terraform.proxy import ProxyAutomation
@ -157,7 +159,7 @@ class ProxyAzureCdnAutomation(ProxyAutomation):
{% endfor %}
"""
def import_state(self, state):
def import_state(self, state: Optional[Any]) -> None:
proxies = Proxy.query.filter(
Proxy.provider == self.provider,
Proxy.destroyed == None

7
app/terraform/proxy/cloudfront.py
View file

@ -1,4 +1,5 @@
import datetime
from typing import Any
from app.extensions import db
from app.models.mirrors import Proxy
@ -72,7 +73,11 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
{% endfor %}
"""
def import_state(self, state):
def import_state(self, state: Any) -> None:
assert(isinstance(state, dict))
if "child_modules" not in state['values']['root_module']:
# There are no CloudFront proxies deployed to import state for
return
for mod in state['values']['root_module']['child_modules']:
if mod['address'].startswith('module.cloudfront_'):
for res in mod['resources']:

3
app/terraform/proxy/fastly.py
View file

@ -1,3 +1,6 @@
# type: ignore
# TODO: This module doesn't work at all
import datetime
import os
import string

5
app/terraform/terraform.py
View file

@ -19,6 +19,11 @@ class TerraformAutomation(BaseAutomation):
Default parallelism for remote API calls.
"""
provider: str
"""
Short name for the provider used by this module.
"""
def automate(self, full: bool = False) -> Tuple[bool, str]:
"""
Runs the Terraform automation module. The run will follow these steps: