jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lint: reformat python code with black

Browse source
This commit is contained in:
Iain Learmonth 2024-12-06 18:15:47 +00:00
parent 331beb01b4
commit a406a7974b
88 changed files with 2579 additions and 1608 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/cli/__init__.py
View file

@ -3,7 +3,9 @@ from abc import abstractmethod
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
_SubparserType = argparse._SubParsersAction[argparse.ArgumentParser] # pylint: disable=protected-access
_SubparserType = argparse._SubParsersAction[
argparse.ArgumentParser
] # pylint: disable=protected-access
else:
_SubparserType = Any

7
app/cli/__main__.py
View file

@ -13,7 +13,9 @@ def parse_args(argv: List[str]) -> None:
if basename(argv[0]) == "__main__.py":
argv[0] = "bypass"
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbose", help="increase logging verbosity", action="store_true")
parser.add_argument(
"-v", "--verbose", help="increase logging verbosity", action="store_true"
)
subparsers = parser.add_subparsers(title="command", help="command to run")
AutomateCliHandler.add_subparser_to(subparsers)
DbCliHandler.add_subparser_to(subparsers)
@ -28,7 +30,6 @@ def parse_args(argv: List[str]) -> None:
if __name__ == "__main__":
VERBOSE = "-v" in sys.argv or "--verbose" in sys.argv
logging.basicConfig(
level=logging.DEBUG if VERBOSE else logging.INFO)
logging.basicConfig(level=logging.DEBUG if VERBOSE else logging.INFO)
logging.debug("Arguments: %s", sys.argv)
parse_args(sys.argv)

99
app/cli/automate.py
View file

@ -14,18 +14,14 @@ from app.models.automation import Automation, AutomationLogs, AutomationState
from app.terraform import BaseAutomation
from app.terraform.alarms.eotk_aws import AlarmEotkAwsAutomation
from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation
from app.terraform.alarms.proxy_cloudfront import \
AlarmProxyCloudfrontAutomation
from app.terraform.alarms.proxy_http_status import \
AlarmProxyHTTPStatusAutomation
from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation
from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation
from app.terraform.alarms.smart_aws import AlarmSmartAwsAutomation
from app.terraform.block.block_blocky import BlockBlockyAutomation
from app.terraform.block.block_scriptzteam import \
BlockBridgeScriptzteamAutomation
from app.terraform.block.block_scriptzteam import BlockBridgeScriptzteamAutomation
from app.terraform.block.bridge_github import BlockBridgeGitHubAutomation
from app.terraform.block.bridge_gitlab import BlockBridgeGitlabAutomation
from app.terraform.block.bridge_roskomsvoboda import \
BlockBridgeRoskomsvobodaAutomation
from app.terraform.block.bridge_roskomsvoboda import BlockBridgeRoskomsvobodaAutomation
from app.terraform.block_external import BlockExternalAutomation
from app.terraform.block_ooni import BlockOONIAutomation
from app.terraform.block_roskomsvoboda import BlockRoskomsvobodaAutomation
@ -58,12 +54,10 @@ jobs = {
BlockExternalAutomation,
BlockOONIAutomation,
BlockRoskomsvobodaAutomation,
# Create new resources
BridgeMetaAutomation,
StaticMetaAutomation,
ProxyMetaAutomation,
# Terraform
BridgeAWSAutomation,
BridgeGandiAutomation,
@ -74,14 +68,12 @@ jobs = {
ProxyAzureCdnAutomation,
ProxyCloudfrontAutomation,
ProxyFastlyAutomation,
# Import alarms
AlarmEotkAwsAutomation,
AlarmProxyAzureCdnAutomation,
AlarmProxyCloudfrontAutomation,
AlarmProxyHTTPStatusAutomation,
AlarmSmartAwsAutomation,
# Update lists
ListGithubAutomation,
ListGitlabAutomation,
@ -103,9 +95,12 @@ def run_all(**kwargs: bool) -> None:
run_job(job, **kwargs)
def run_job(job_cls: Type[BaseAutomation], *,
force: bool = False, ignore_schedule: bool = False) -> None:
automation = Automation.query.filter(Automation.short_name == job_cls.short_name).first()
def run_job(
job_cls: Type[BaseAutomation], *, force: bool = False, ignore_schedule: bool = False
) -> None:
automation = Automation.query.filter(
Automation.short_name == job_cls.short_name
).first()
if automation is None:
automation = Automation()
automation.short_name = job_cls.short_name
@ -121,18 +116,24 @@ def run_job(job_cls: Type[BaseAutomation], *,
logging.warning("Not running an already running automation")
return
if not ignore_schedule and not force:
if automation.next_run is not None and automation.next_run > datetime.now(tz=timezone.utc):
if automation.next_run is not None and automation.next_run > datetime.now(
tz=timezone.utc
):
logging.warning("Not time to run this job yet")
return
if not automation.enabled and not force:
logging.warning("job %s is disabled and --force not specified", job_cls.short_name)
logging.warning(
"job %s is disabled and --force not specified", job_cls.short_name
)
return
automation.state = AutomationState.RUNNING
db.session.commit()
try:
if 'TERRAFORM_DIRECTORY' in app.config:
working_dir = os.path.join(app.config['TERRAFORM_DIRECTORY'],
job_cls.short_name or job_cls.__class__.__name__.lower())
if "TERRAFORM_DIRECTORY" in app.config:
working_dir = os.path.join(
app.config["TERRAFORM_DIRECTORY"],
job_cls.short_name or job_cls.__class__.__name__.lower(),
)
else:
working_dir = tempfile.mkdtemp()
job: BaseAutomation = job_cls(working_dir)
@ -150,8 +151,9 @@ def run_job(job_cls: Type[BaseAutomation], *,
if job is not None and success:
automation.state = AutomationState.IDLE
automation.next_run = datetime.now(tz=timezone.utc) + timedelta(
minutes=getattr(job, "frequency", 7))
if 'TERRAFORM_DIRECTORY' not in app.config and working_dir is not None:
minutes=getattr(job, "frequency", 7)
)
if "TERRAFORM_DIRECTORY" not in app.config and working_dir is not None:
# We used a temporary working directory
shutil.rmtree(working_dir)
else:
@ -165,7 +167,7 @@ def run_job(job_cls: Type[BaseAutomation], *,
"list_gitlab",
"block_blocky",
"block_external",
"block_ooni"
"block_ooni",
]
if job.short_name not in safe_jobs:
automation.enabled = False
@ -179,10 +181,12 @@ def run_job(job_cls: Type[BaseAutomation], *,
db.session.commit()
activity = Activity(
activity_type="automation",
text=(f"[{automation.short_name}] 🚨 Automation failure: It was not possible to handle this failure safely "
"and so the automation task has been automatically disabled. It may be possible to simply re-enable "
"the task, but repeated failures will usually require deeper investigation. See logs for full "
"details.")
text=(
f"[{automation.short_name}] 🚨 Automation failure: It was not possible to handle this failure safely "
"and so the automation task has been automatically disabled. It may be possible to simply re-enable "
"the task, but repeated failures will usually require deeper investigation. See logs for full "
"details."
),
)
db.session.add(activity)
activity.notify() # Notify before commit because the failure occurred even if we can't commit.
@ -194,20 +198,43 @@ class AutomateCliHandler(BaseCliHandler):
@classmethod
def add_subparser_to(cls, subparsers: _SubparserType) -> None:
parser = subparsers.add_parser("automate", help="automation operations")
parser.add_argument("-a", "--all", dest="all", help="run all automation jobs", action="store_true")
parser.add_argument("-j", "--job", dest="job", choices=sorted(jobs.keys()),
help="run a specific automation job")
parser.add_argument("--force", help="run job even if disabled and it's not time yet", action="store_true")
parser.add_argument("--ignore-schedule", help="run job even if it's not time yet", action="store_true")
parser.add_argument(
"-a",
"--all",
dest="all",
help="run all automation jobs",
action="store_true",
)
parser.add_argument(
"-j",
"--job",
dest="job",
choices=sorted(jobs.keys()),
help="run a specific automation job",
)
parser.add_argument(
"--force",
help="run job even if disabled and it's not time yet",
action="store_true",
)
parser.add_argument(
"--ignore-schedule",
help="run job even if it's not time yet",
action="store_true",
)
parser.set_defaults(cls=cls)
def run(self) -> None:
with app.app_context():
if self.args.job:
run_job(jobs[self.args.job],
force=self.args.force,
ignore_schedule=self.args.ignore_schedule)
run_job(
jobs[self.args.job],
force=self.args.force,
ignore_schedule=self.args.ignore_schedule,
)
elif self.args.all:
run_all(force=self.args.force, ignore_schedule=self.args.ignore_schedule)
run_all(
force=self.args.force, ignore_schedule=self.args.ignore_schedule
)
else:
logging.error("No action requested")

14
app/cli/db.py
View file

@ -40,7 +40,7 @@ models: List[Model] = [
Eotk,
MirrorList,
TerraformState,
Webhook
Webhook,
]
@ -53,7 +53,7 @@ class ExportEncoder(json.JSONEncoder):
if isinstance(o, AutomationState):
return o.name
if isinstance(o, bytes):
return base64.encodebytes(o).decode('utf-8')
return base64.encodebytes(o).decode("utf-8")
if isinstance(o, (datetime.datetime, datetime.date, datetime.time)):
return o.isoformat()
return super().default(o)
@ -82,7 +82,7 @@ def db_export() -> None:
decoder: Dict[str, Callable[[Any], Any]] = {
"AlarmState": lambda x: AlarmState.__getattribute__(AlarmState, x),
"AutomationState": lambda x: AutomationState.__getattribute__(AutomationState, x),
"bytes": lambda x: base64.decodebytes(x.encode('utf-8')),
"bytes": lambda x: base64.decodebytes(x.encode("utf-8")),
"datetime": datetime.datetime.fromisoformat,
"int": int,
"str": lambda x: x,
@ -110,8 +110,12 @@ class DbCliHandler(BaseCliHandler):
@classmethod
def add_subparser_to(cls, subparsers: _SubparserType) -> None:
parser = subparsers.add_parser("db", help="database operations")
parser.add_argument("--export", help="export data to JSON format", action="store_true")
parser.add_argument("--import", help="import data from JSON format", action="store_true")
parser.add_argument(
"--export", help="export data to JSON format", action="store_true"
)
parser.add_argument(
"--import", help="import data from JSON format", action="store_true"
)
parser.set_defaults(cls=cls)
def run(self) -> None:

5
app/cli/list.py
View file

@ -17,8 +17,9 @@ class ListCliHandler(BaseCliHandler):
@classmethod
def add_subparser_to(cls, subparsers: _SubparserType) -> None:
parser = subparsers.add_parser("list", help="list operations")
parser.add_argument("--dump", choices=sorted(lists.keys()),
help="dump a list in JSON format")
parser.add_argument(
"--dump", choices=sorted(lists.keys()), help="dump a list in JSON format"
)
parser.set_defaults(cls=cls)
def run(self) -> None: