208 lines
8.7 KiB
Python
208 lines
8.7 KiB
Python
import datetime
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from traceback import TracebackException
|
|
from typing import Type
|
|
|
|
from app import app
|
|
from app.cli import _SubparserType, BaseCliHandler
|
|
from app.extensions import db
|
|
from app.models.activity import Activity
|
|
from app.models.automation import Automation, AutomationState, AutomationLogs
|
|
from app.terraform import BaseAutomation
|
|
from app.terraform.block.block_blocky import BlockBlockyAutomation
|
|
from app.terraform.block.bridge_github import BlockBridgeGitHubAutomation
|
|
from app.terraform.block.bridge_gitlab import BlockBridgeGitlabAutomation
|
|
from app.terraform.block.bridge_roskomsvoboda import BlockBridgeRoskomsvobodaAutomation
|
|
from app.terraform.block.block_scriptzteam import BlockBridgeScriptzteamAutomation
|
|
from app.terraform.block_external import BlockExternalAutomation
|
|
from app.terraform.block_ooni import BlockOONIAutomation
|
|
from app.terraform.block_roskomsvoboda import BlockRoskomsvobodaAutomation
|
|
from app.terraform.bridge.meta import BridgeMetaAutomation
|
|
from app.terraform.eotk.aws import EotkAWSAutomation
|
|
from app.terraform.alarms.eotk_aws import AlarmEotkAwsAutomation
|
|
from app.terraform.alarms.proxy_azure_cdn import AlarmProxyAzureCdnAutomation
|
|
from app.terraform.alarms.proxy_cloudfront import AlarmProxyCloudfrontAutomation
|
|
from app.terraform.alarms.proxy_http_status import AlarmProxyHTTPStatusAutomation
|
|
from app.terraform.alarms.smart_aws import AlarmSmartAwsAutomation
|
|
from app.terraform.bridge.aws import BridgeAWSAutomation
|
|
from app.terraform.bridge.gandi import BridgeGandiAutomation
|
|
from app.terraform.bridge.hcloud import BridgeHcloudAutomation
|
|
from app.terraform.bridge.ovh import BridgeOvhAutomation
|
|
from app.terraform.list.github import ListGithubAutomation
|
|
from app.terraform.list.gitlab import ListGitlabAutomation
|
|
from app.terraform.list.http_post import ListHttpPostAutomation
|
|
from app.terraform.list.s3 import ListS3Automation
|
|
from app.terraform.proxy.meta import ProxyMetaAutomation
|
|
from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
|
|
from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
|
|
from app.terraform.proxy.fastly import ProxyFastlyAutomation
|
|
from app.terraform.static.aws import StaticAWSAutomation
|
|
from app.terraform.static.meta import StaticMetaAutomation
|
|
|
|
jobs = {
|
|
x.short_name: x # type: ignore[attr-defined]
|
|
for x in [
|
|
# Check for blocked resources first
|
|
BlockBridgeGitHubAutomation,
|
|
BlockBridgeGitlabAutomation,
|
|
BlockBridgeRoskomsvobodaAutomation,
|
|
BlockBridgeScriptzteamAutomation,
|
|
BlockBlockyAutomation,
|
|
BlockExternalAutomation,
|
|
BlockOONIAutomation,
|
|
BlockRoskomsvobodaAutomation,
|
|
|
|
# Create new resources
|
|
BridgeMetaAutomation,
|
|
StaticMetaAutomation,
|
|
ProxyMetaAutomation,
|
|
|
|
# Terraform
|
|
BridgeAWSAutomation,
|
|
BridgeGandiAutomation,
|
|
BridgeHcloudAutomation,
|
|
BridgeOvhAutomation,
|
|
StaticAWSAutomation,
|
|
EotkAWSAutomation,
|
|
ProxyAzureCdnAutomation,
|
|
ProxyCloudfrontAutomation,
|
|
ProxyFastlyAutomation,
|
|
|
|
# Import alarms
|
|
AlarmEotkAwsAutomation,
|
|
AlarmProxyAzureCdnAutomation,
|
|
AlarmProxyCloudfrontAutomation,
|
|
AlarmProxyHTTPStatusAutomation,
|
|
AlarmSmartAwsAutomation,
|
|
|
|
# Update lists
|
|
ListGithubAutomation,
|
|
ListGitlabAutomation,
|
|
ListHttpPostAutomation,
|
|
ListS3Automation,
|
|
]
|
|
}
|
|
|
|
|
|
def run_all(**kwargs: bool) -> None:
|
|
"""
|
|
Run all automation tasks.
|
|
|
|
:param kwargs: this function takes the same arguments as :func:`run_job` and will pass the same arguments
|
|
to every task
|
|
:return: None
|
|
"""
|
|
for job in jobs.values():
|
|
run_job(job, **kwargs)
|
|
|
|
|
|
def run_job(job_cls: Type[BaseAutomation], *,
|
|
force: bool = False, ignore_schedule: bool = False) -> None:
|
|
automation = Automation.query.filter(Automation.short_name == job_cls.short_name).first()
|
|
if automation is None:
|
|
automation = Automation()
|
|
automation.short_name = job_cls.short_name
|
|
automation.description = job_cls.description
|
|
automation.enabled = False
|
|
automation.next_is_full = False
|
|
automation.added = datetime.datetime.utcnow()
|
|
automation.updated = automation.added
|
|
db.session.add(automation)
|
|
db.session.commit()
|
|
else:
|
|
if automation.state == AutomationState.RUNNING and not force:
|
|
logging.warning("Not running an already running automation")
|
|
return
|
|
if not ignore_schedule and not force:
|
|
if automation.next_run is not None and automation.next_run > datetime.datetime.utcnow():
|
|
logging.warning("Not time to run this job yet")
|
|
return
|
|
if not automation.enabled and not force:
|
|
logging.warning("job %s is disabled and --force not specified", job_cls.short_name)
|
|
return
|
|
automation.state = AutomationState.RUNNING
|
|
db.session.commit()
|
|
try:
|
|
if 'TERRAFORM_DIRECTORY' in app.config:
|
|
working_dir = os.path.join(app.config['TERRAFORM_DIRECTORY'],
|
|
job_cls.short_name or job_cls.__class__.__name__.lower())
|
|
else:
|
|
working_dir = tempfile.mkdtemp()
|
|
job: BaseAutomation = job_cls(working_dir)
|
|
success, logs = job.automate()
|
|
# We want to catch any and all exceptions that would cause problems here, because
|
|
# the error handling process isn't really handling the error, but rather causing it
|
|
# to be logged for investigation. Catching more specific exceptions would just mean that
|
|
# others go unrecorded and are difficult to debug.
|
|
except Exception as exc: # pylint: disable=broad-except
|
|
if logging.getLogger().level == logging.DEBUG:
|
|
raise exc
|
|
trace = TracebackException.from_exception(exc)
|
|
success = False
|
|
logs = "\n".join(trace.format())
|
|
if job is not None and success:
|
|
automation.state = AutomationState.IDLE
|
|
automation.next_run = datetime.datetime.utcnow() + datetime.timedelta(
|
|
minutes=getattr(job, "frequency", 7))
|
|
if 'TERRAFORM_DIRECTORY' not in app.config and working_dir is not None:
|
|
# We used a temporary working directory
|
|
shutil.rmtree(working_dir)
|
|
else:
|
|
automation.state = AutomationState.ERROR
|
|
safe_jobs = [
|
|
"proxy_cloudfront",
|
|
"static_aws",
|
|
"list_http_post",
|
|
"list_s3",
|
|
"list_github",
|
|
"list_gitlab",
|
|
"block_blocky",
|
|
"block_external"
|
|
]
|
|
if job.short_name not in safe_jobs:
|
|
automation.enabled = False
|
|
automation.next_run = None
|
|
log = AutomationLogs()
|
|
log.automation_id = automation.id
|
|
log.added = datetime.datetime.utcnow()
|
|
log.updated = datetime.datetime.utcnow()
|
|
log.logs = str(logs)
|
|
db.session.add(log)
|
|
db.session.commit()
|
|
activity = Activity(
|
|
activity_type="automation",
|
|
text=(f"[{automation.short_name}] 🚨 Automation failure: It was not possible to handle this failure safely "
|
|
"and so the automation task has been automatically disabled. It may be possible to simply re-enable "
|
|
"the task, but repeated failures will usually require deeper investigation. See logs for full "
|
|
"details.")
|
|
)
|
|
db.session.add(activity)
|
|
activity.notify() # Notify before commit because the failure occurred even if we can't commit.
|
|
automation.last_run = datetime.datetime.utcnow()
|
|
db.session.commit()
|
|
|
|
|
|
class AutomateCliHandler(BaseCliHandler):
|
|
@classmethod
|
|
def add_subparser_to(cls, subparsers: _SubparserType) -> None:
|
|
parser = subparsers.add_parser("automate", help="automation operations")
|
|
parser.add_argument("-a", "--all", dest="all", help="run all automation jobs", action="store_true")
|
|
parser.add_argument("-j", "--job", dest="job", choices=sorted(jobs.keys()),
|
|
help="run a specific automation job")
|
|
parser.add_argument("--force", help="run job even if disabled and it's not time yet", action="store_true")
|
|
parser.add_argument("--ignore-schedule", help="run job even if it's not time yet", action="store_true")
|
|
parser.set_defaults(cls=cls)
|
|
|
|
def run(self) -> None:
|
|
with app.app_context():
|
|
if self.args.job:
|
|
run_job(jobs[self.args.job],
|
|
force=self.args.force,
|
|
ignore_schedule=self.args.ignore_schedule)
|
|
elif self.args.all:
|
|
run_all(force=self.args.force, ignore_schedule=self.args.ignore_schedule)
|
|
else:
|
|
logging.error("No action requested")
|