init

.alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

.alembic/env.py

@@ -0,0 +1,80 @@
+from logging.config import fileConfig
+
+from sqlalchemy import create_engine
+# noinspection PyUnusedImportsInspection
+from sqlalchemy import pool
+
+from alembic import context
+
+from src.config import SQLALCHEMY_DATABASE_URI
+
+from src.database import Base
+# noinspection PyUnusedImportsInspection
+from src.misp.models import Domain
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = create_engine(SQLALCHEMY_DATABASE_URI.get_secret_value())
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

.alembic/script.py.mako

@@ -0,0 +1,28 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    ${downgrades if downgrades else "pass"}

.alembic/versions/2026-04-14_add_events_array_column.py

@@ -0,0 +1,32 @@
+"""add events array column
+
+Revision ID: de3a912a492c
+Revises: 85f864dba26f
+Create Date: 2026-04-14 16:36:51.595658
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'de3a912a492c'
+down_revision: Union[str, Sequence[str], None] = '85f864dba26f'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('domain', sa.Column('events', sa.ARRAY(sa.Integer()), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('domain', 'events')
+    # ### end Alembic commands ###

.alembic/versions/2026-04-14_initial_database_model.py

@@ -0,0 +1,38 @@
+"""initial database model
+
+Revision ID: 85f864dba26f
+Revises: 
+Create Date: 2026-04-14 13:30:14.536202
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '85f864dba26f'
+down_revision: Union[str, Sequence[str], None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('domain',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('domain', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_domain_domain'), 'domain', ['domain'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_domain_domain'), table_name='domain')
+    op.drop_table('domain')
+    # ### end Alembic commands ###

.alembic/versions/2026-04-15_add_allowlisting_columns.py

@@ -0,0 +1,34 @@
+"""add allowlisting columns
+
+Revision ID: 4ac5a5cbc12f
+Revises: de3a912a492c
+Create Date: 2026-04-15 14:44:03.825138
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '4ac5a5cbc12f'
+down_revision: Union[str, Sequence[str], None] = 'de3a912a492c'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('domain', sa.Column('always_allowed', sa.Boolean(), nullable=True))
+    op.add_column('domain', sa.Column('ignored_events', postgresql.ARRAY(sa.Integer()), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('domain', 'ignored_events')
+    op.drop_column('domain', 'always_allowed')
+    # ### end Alembic commands ###

.gitignore

@@ -0,0 +1,209 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+key

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.13-slim AS build-env
+
+COPY . /app
+WORKDIR /app
+
+RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install "psycopg[binary,pool]"
+
+FROM gcr.io/distroless/python3-debian13
+
+COPY --from=build-env /usr/local/lib /usr/local/lib
+COPY --from=build-env /usr/local/bin /usr/local/bin
+
+COPY --from=build-env /app /app
+
+EXPOSE 8000
+
+ENV PYTHONPATH=/usr/local/lib/python3.13/site-packages
+
+WORKDIR /app
+
+CMD ["/usr/local/bin/uvicorn", "src.main:app", "--host", "0.0.0.0"]

alembic.ini

@@ -0,0 +1,149 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts.
+# this is typically a path given in POSIX (e.g. forward slashes)
+# format, relative to the token %(here)s which refers to the location of this
+# ini file
+script_location = %(here)s/.alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+# Or organize into date-based subdirectories (requires recursive_version_locations = true)
+file_template = %%(year)d-%%(month).2d-%%(day).2d_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.  for multiple paths, the path separator
+# is defined by "path_separator" below.
+prepend_sys_path = .
+
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the tzdata library which can be installed by adding
+# `alembic[tz]` to the pip requirements.
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to <script_location>/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "path_separator"
+# below.
+# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
+
+# path_separator; This indicates what character is used to split lists of file
+# paths, including version_locations and prepend_sys_path within configparser
+# files such as alembic.ini.
+# The default rendered in new alembic.ini files is "os", which uses os.pathsep
+# to provide os-dependent path splitting.
+#
+# Note that in order to support legacy alembic.ini files, this default does NOT
+# take place if path_separator is not present in alembic.ini.  If this
+# option is omitted entirely, fallback logic is as follows:
+#
+# 1. Parsing of the version_locations option falls back to using the legacy
+#    "version_path_separator" key, which if absent then falls back to the legacy
+#    behavior of splitting on spaces and/or commas.
+# 2. Parsing of the prepend_sys_path option falls back to the legacy
+#    behavior of splitting on spaces, commas, or colons.
+#
+# Valid values for path_separator are:
+#
+# path_separator = :
+# path_separator = ;
+# path_separator = space
+# path_separator = newline
+#
+# Use os.pathsep. Default configuration used for new projects.
+path_separator = os
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# database URL.  This is consumed by the user-maintained env.py script only.
+# other means of configuring database URLs may be customized within the env.py
+# file.
+sqlalchemy.url = none
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
+# hooks = ruff
+# ruff.type = module
+# ruff.module = ruff
+# ruff.options = check --fix REVISION_SCRIPT_FILENAME
+
+# Alternatively, use the exec runner to execute a binary found on your PATH
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = ruff
+# ruff.options = check --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration.  This is also consumed by the user-maintained
+# env.py script only.
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

requirements.txt

@@ -0,0 +1,22 @@
+fastapi
+pydantic
+uvicorn
+jinja2
+python-dotenv
+requests
+itsdangerous
+starlette
+pydantic-settings
+authlib
+httpx
+types-authlib
+python-jose
+pytest
+uvloop; sys_platform != 'win32'
+sqlalchemy
+httptools
+psycopg
+alembic
+pymisp
+prometheus-client
+unbound_console

src/_module_template/config.py

@@ -0,0 +1,7 @@
+"""
+Configurations for <this module>
+
+Configurations:
+ - List: Description
+ - Configs: Description
+"""

src/_module_template/constants.py

@@ -0,0 +1,7 @@
+"""
+Constants and error codes for <this module>
+
+Constants:
+ - List: Description
+ - Consts: Description
+"""

src/_module_template/dependencies.py

@@ -0,0 +1,11 @@
+"""
+Router dependencies for <this module>
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/_module_template/exceptions.py

@@ -0,0 +1,7 @@
+"""
+Module specific exceptions for <this module>
+
+Exceptions:
+ - List: Description
+ - Exceptions: Description
+"""

src/_module_template/models.py

@@ -0,0 +1,7 @@
+"""
+Database models for <this module>
+
+Models:
+ - List: Description
+ - Models: Description
+"""

src/_module_template/router.py

@@ -0,0 +1,13 @@
+"""
+Router endpoints for <this module>
+
+Endpoints:
+ - List: Description
+ - Endpoints: Description
+"""
+from fastapi import APIRouter
+
+
+_router = APIRouter(
+	tags=[""],
+)

src/_module_template/schemas.py

@@ -0,0 +1,7 @@
+"""
+Pydantic models for <this module>
+
+Models:
+ - List: Description
+ - Models: Description
+"""

src/_module_template/service.py

@@ -0,0 +1,11 @@
+"""
+Module specific business logic for <this module>
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/_module_template/utils.py

@@ -0,0 +1,11 @@
+"""
+Non-business logic reusable functions and classes for <this module>
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/api.py

@@ -0,0 +1,21 @@
+"""
+This module hooks the routers for the main endpoints into a single router for importing to the app.
+"""
+from fastapi import APIRouter
+
+from src.auth.router import router as auth_router
+from src.misp.router import router as misp_router
+from src.control.router import router as control_router
+
+
+api_router = APIRouter()
+
+api_router.include_router(auth_router)
+api_router.include_router(misp_router)
+api_router.include_router(control_router)
+
+
+@api_router.get("/healthcheck", include_in_schema=False)
+def healthcheck():
+    """Simple health check endpoint."""
+    return {"status": "ok"}

src/auth/config.py

@@ -0,0 +1,13 @@
+"""
+Configurations for auth module, import auth_settings
+"""
+from src.config import CustomBaseSettings
+
+class AuthConfig(CustomBaseSettings):
+    OIDC_CONFIG: str = ""
+    OIDC_ISSUER: str = ""
+    OIDC_AUDIENCE: str = ""
+    CLIENT_ID: str = ""
+
+
+auth_settings = AuthConfig()

src/auth/constants.py

@@ -0,0 +1,7 @@
+"""
+Constants and error codes for auth module
+
+Constants:
+ - List: Description
+ - Consts: Description
+"""

src/auth/dependencies.py

@@ -0,0 +1,11 @@
+"""
+Router dependencies for auth module
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/auth/exceptions.py

@@ -0,0 +1,7 @@
+"""
+Module specific exceptions for auth module
+
+Exceptions:
+ - List: Description
+ - Exceptions: Description
+"""

src/auth/models.py

@@ -0,0 +1,7 @@
+"""
+Database models for auth module
+
+Models:
+ - List: Description
+ - Models: Description
+"""

src/auth/router.py

@@ -0,0 +1,11 @@
+"""
+Router endpoints for auth module
+Contains oauth registration
+
+Endpoints:
+"""
+from fastapi import APIRouter
+
+router = APIRouter(
+	tags=["auth"],
+)

src/auth/schemas.py

@@ -0,0 +1,7 @@
+"""
+Pydantic models for auth module
+
+Models:
+ - List: Description
+ - Models: Description
+"""

src/auth/service.py

@@ -0,0 +1,63 @@
+"""
+Module specific business logic for auth module
+
+Exports:
+ - claims_dependency
+ - authed_dependency
+"""
+import json
+
+from typing import Annotated
+from authlib.jose import jwt
+from urllib.request import urlopen
+
+from fastapi import Depends, HTTPException
+from fastapi.security import OpenIdConnect
+from authlib.jose.rfc7517.jwk import JsonWebKey
+from authlib.jose.rfc7517.key_set import KeySet
+from authlib.oauth2.rfc7523.validator import JWTBearerToken
+
+from src.auth.config import auth_settings
+
+
+oidc = OpenIdConnect(openIdConnectUrl=auth_settings.OIDC_CONFIG)
+oidc_dependency = Annotated[str, Depends(oidc)]
+
+
+async def get_current_user(oidc_auth_string: oidc_dependency) -> JWTBearerToken:
+	config_url = urlopen(auth_settings.OIDC_CONFIG)
+	config = json.loads(config_url.read())
+	jwks_uri = config["jwks_uri"]
+	key_response = urlopen(jwks_uri)
+	jwk_keys: KeySet = JsonWebKey.import_key_set(json.loads(key_response.read()))
+
+	claims_options = {
+		"exp": {"essential": True},
+		"aud": {"essential": True, "value": "account"},
+		"iss": {"essential": True, "value": auth_settings.OIDC_ISSUER},
+	}
+
+	claims: JWTBearerToken = jwt.decode(
+		oidc_auth_string.replace("Bearer ", ""),
+		jwk_keys,
+		claims_options=claims_options,
+		claims_cls=JWTBearerToken,
+	)
+
+	claims.validate()
+
+	return claims
+
+
+claims_dependency = Annotated[JWTBearerToken, Depends(get_current_user)]
+
+
+async def is_authed_user(claims: claims_dependency) -> bool:
+	authed_users: list[str] = ["chris@sr2.uk"]
+	user_email = claims.get("email", None)
+	if not user_email or user_email not in authed_users:
+		raise HTTPException(status_code=403, detail="Not authenticated")
+	return claims.get("email") in authed_users
+
+
+authed_dependency = Annotated[bool, Depends(is_authed_user)]

src/auth/utils.py

@@ -0,0 +1,11 @@
+"""
+Non-business logic reusable functions and classes for auth module
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/config.py

@@ -0,0 +1,53 @@
+"""
+Global configurations: import settings, app_configs
+
+Classes:
+ - CustomBaseSettings - Base class to be used by all modules for loading configs
+"""
+
+from typing import Any
+from urllib import parse
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic import SecretStr
+
+from src.constants import Environment
+
+
+class CustomBaseSettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env", env_file_encoding="utf-8", extra="ignore"
+    )
+
+
+class Config(CustomBaseSettings):
+    APP_VERSION: str = "0.1"
+    ENVIRONMENT: Environment = Environment.PRODUCTION
+    SECRET_KEY: SecretStr = ""
+
+    CORS_ORIGINS: list[str] = ["*"]
+    CORS_ORIGINS_REGEX: str | None = None
+    CORS_HEADERS: list[str] = ["*"]
+
+    DATABASE_NAME: str = "dns-exp"
+    DATABASE_PORT: str = "5432"
+    DATABASE_HOSTNAME: str = "localhost"
+    DATABASE_CREDENTIALS: SecretStr = ""
+
+settings = Config()
+
+DATABASE_NAME = settings.DATABASE_NAME
+DATABASE_PORT = settings.DATABASE_PORT
+DATABASE_HOSTNAME = settings.DATABASE_HOSTNAME
+DATABASE_CREDENTIALS = settings.DATABASE_CREDENTIALS.get_secret_value()
+# this will support special chars for credentials
+_DATABASE_CREDENTIAL_USER, _DATABASE_CREDENTIAL_PASSWORD = str(DATABASE_CREDENTIALS).split(":")
+_QUOTED_DATABASE_PASSWORD = parse.quote_plus(str(_DATABASE_CREDENTIAL_PASSWORD))
+
+SQLALCHEMY_DATABASE_URI = SecretStr(f"postgresql+psycopg://{_DATABASE_CREDENTIAL_USER}:{_QUOTED_DATABASE_PASSWORD}@{DATABASE_HOSTNAME}:{DATABASE_PORT}/{DATABASE_NAME}")
+
+app_configs: dict[str, Any] = {"title": "App API"}
+if settings.ENVIRONMENT.is_deployed:
+    app_configs["root_path"] = f"/v{settings.APP_VERSION}"
+
+if not settings.ENVIRONMENT.is_debug:
+    app_configs["openapi_url"] = None  # hide docs

src/constants.py

@@ -0,0 +1,36 @@
+"""
+Global constants
+
+Classes:
+ - Environment(StrEnum): LOCAL, TESTING, STAGING, PRODUCTION
+"""
+from enum import StrEnum, auto
+
+
+class Environment(StrEnum):
+	"""
+	Enumeration of environments.
+
+	Attributes:
+		LOCAL (str): Application is running locally
+		TESTING (str): Application is running in testing mode
+		STAGING (str): Application is running in staging mode (ie not testing)
+		PRODUCTION (str): Application is running in production mode
+	"""
+
+	LOCAL = auto()
+	TESTING = auto()
+	STAGING = auto()
+	PRODUCTION = auto()
+
+	@property
+	def is_debug(self):
+		return self in (self.LOCAL, self.STAGING, self.TESTING)
+
+	@property
+	def is_testing(self):
+		return self == self.TESTING
+
+	@property
+	def is_deployed(self) -> bool:
+		return self in (self.STAGING, self.PRODUCTION)

src/control/config.py

@@ -0,0 +1,7 @@
+"""
+Configurations for <this module>
+
+Configurations:
+ - List: Description
+ - Configs: Description
+"""

src/control/constants.py

@@ -0,0 +1,13 @@
+"""
+Constants and error codes for <this module>
+
+Constants:
+ - List: Description
+ - Consts: Description
+"""
+from enum import StrEnum, auto
+
+
+class TimerState(StrEnum):
+	STARTING = auto()
+	STOPPING = auto()

src/control/dependencies.py

@@ -0,0 +1,11 @@
+"""
+Router dependencies for <this module>
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/control/exceptions.py

@@ -0,0 +1,7 @@
+"""
+Module specific exceptions for <this module>
+
+Exceptions:
+ - List: Description
+ - Exceptions: Description
+"""

src/control/models.py

@@ -0,0 +1,7 @@
+"""
+Database models for <this module>
+
+Models:
+ - List: Description
+ - Models: Description
+"""

src/control/router.py

@@ -0,0 +1,34 @@
+"""
+Router endpoints for the control module
+
+Endpoints:
+ - List: Description
+ - Endpoints: Description
+"""
+from fastapi import APIRouter, Request
+from starlette import status
+
+from src.auth.service import authed_dependency
+
+from src.control.schemas import ControlTimerPutResponse
+
+router = APIRouter(
+	tags=["control"],
+	prefix="/control",
+)
+
+
+@router.put("/start_timer", status_code=status.HTTP_202_ACCEPTED, response_model=ControlTimerPutResponse)
+async def start_timer(request: Request):
+	misp_handler = request.app.misp_handler
+
+	await misp_handler.start_timer()
+	return {"state": "starting"}
+
+
+@router.put("/stop_timer", status_code=status.HTTP_202_ACCEPTED, response_model=ControlTimerPutResponse)
+async def stop_timer(request: Request):
+	misp_handler = request.app.misp_handler
+
+	misp_handler.stop_timer()
+	return {"state": "stopping"}

src/control/schemas.py

@@ -0,0 +1,14 @@
+"""
+Pydantic models for <this module>
+
+Models:
+ - List: Description
+ - Models: Description
+"""
+from src.schemas import CustomBaseModel
+
+from src.control.constants import TimerState
+
+
+class ControlTimerPutResponse(CustomBaseModel):
+	state: TimerState

src/control/service.py

@@ -0,0 +1,11 @@
+"""
+Module specific business logic for <this module>
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/control/utils.py

@@ -0,0 +1,11 @@
+"""
+Non-business logic reusable functions and classes for <this module>
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/database.py

@@ -0,0 +1,31 @@
+"""
+Database connections and init
+
+Exports:
+ - db_dependency
+ - Base (sqlalchemy base model)
+ - get_db
+"""
+from typing import Annotated
+from sqlalchemy import create_engine
+from sqlalchemy.orm import declarative_base, sessionmaker, Session
+
+from fastapi import Depends
+
+from src.config import SQLALCHEMY_DATABASE_URI
+
+engine = create_engine(SQLALCHEMY_DATABASE_URI.get_secret_value())
+
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+def get_db():
+	with SessionLocal.begin() as db:
+		try:
+			yield db
+		finally:
+			db.rollback()   # Anything not explicitly commited is rolled back
+			db.close()
+
+
+db_dependency = Annotated[Session, Depends(get_db)]
+Base = declarative_base()

src/exceptions.py

@@ -0,0 +1,3 @@
+"""
+Global exceptions
+"""

src/main.py

@@ -0,0 +1,67 @@
+"""
+Application root file: Inits the FastAPI application
+"""
+from contextlib import asynccontextmanager
+from typing import AsyncGenerator
+from prometheus_client import make_asgi_app
+
+from fastapi import FastAPI
+from starlette.middleware.sessions import SessionMiddleware
+from starlette.middleware.cors import CORSMiddleware
+
+from src.config import settings
+from src.api import api_router
+from src.prometheus import prometheus
+
+from src.auth.config import auth_settings
+from src.misp.service import MISPHandler
+
+# TODO: Create Pydantic request/response schemas
+
+
+@asynccontextmanager
+async def lifespan(_application: FastAPI) -> AsyncGenerator:
+	# Startup
+	yield
+	# Shutdown
+	_application.misp_handler.stop_timer()
+
+
+if settings.ENVIRONMENT.is_deployed:
+	# Do this only on prod
+	pass
+
+
+app = FastAPI(
+	swagger_ui_init_oauth={
+		"clientId": auth_settings.CLIENT_ID,
+		"usePkceWithAuthorizationCodeGrant": True,
+		"scopes": "openid profile email",
+	}
+)
+
+
+metrics_app = make_asgi_app()
+app.mount("/metrics", metrics_app)
+
+prometheus.APP_STATE.state("starting")
+
+# Type inspection disabled for middleware injection.
+# Known bug in FastAPI type checking: https://github.com/astral-sh/ty/issues/1635
+# noinspection PyTypeChecker
+app.add_middleware(SessionMiddleware, secret_key=settings.SECRET_KEY.get_secret_value())
+# noinspection PyTypeChecker
+app.add_middleware(
+	CORSMiddleware,
+	allow_origins=settings.CORS_ORIGINS,
+	allow_origin_regex=settings.CORS_ORIGINS_REGEX,
+	allow_credentials=True,
+	allow_methods=("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"),
+	allow_headers=settings.CORS_HEADERS,
+)
+
+app.include_router(api_router)
+
+app.misp_handler = MISPHandler()
+
+prometheus.APP_STATE.state("running")

src/misp/config.py

@@ -0,0 +1,14 @@
+"""
+Configurations for the MISP module
+"""
+from src.config import CustomBaseSettings
+
+class MISPConfig(CustomBaseSettings):
+	MISP_KEY_FILE: str = ""
+	MISP_OUTPUT_FILE: str = ""
+	ALLOWED_TLP: list[str] = ["tlp:clear", "tlp:white", "tlp:green"]
+	IGNORED_TLP: list[str] = ["tlp:red", "tlp:amber+strict", "tlp:amber"]
+	UNBOUND_CERT_DIR: str = ""
+
+
+settings = MISPConfig()

src/misp/constants.py

@@ -0,0 +1,7 @@
+"""
+Constants and error codes for <this module>
+
+Constants:
+ - List: Description
+ - Consts: Description
+"""

src/misp/dependencies.py

@@ -0,0 +1,11 @@
+"""
+Router dependencies for <this module>
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/misp/exceptions.py

@@ -0,0 +1,7 @@
+"""
+Module specific exceptions for <this module>
+
+Exceptions:
+ - List: Description
+ - Exceptions: Description
+"""

src/misp/models.py

@@ -0,0 +1,18 @@
+"""
+Database models for the misp module
+"""
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.ext.mutable import MutableList
+
+from src.database import Base
+
+
+class Domain(Base):
+	__tablename__ = "domain"
+
+	id = Column(Integer, primary_key=True)
+	domain = Column(String, index=True)
+	events = Column(MutableList.as_mutable(ARRAY(Integer)), default=[])
+	always_allowed = Column(Boolean, default=False)
+	ignored_events = Column(MutableList.as_mutable(ARRAY(Integer)), default=[])

src/misp/router.py

@@ -0,0 +1,123 @@
+"""
+Router endpoints for <this module>
+
+Endpoints:
+ - List: Description
+ - Endpoints: Description
+"""
+from fastapi import APIRouter, HTTPException, Request, BackgroundTasks
+from sqlalchemy.sql import and_
+
+from src.auth.service import authed_dependency
+from src.database import db_dependency
+
+from src.misp.models import Domain
+from src.misp.schemas import MispUpdatePutRequest, MispUpdatePutResponse
+
+router = APIRouter(
+	tags=["misp"],
+	prefix="/misp",
+)
+
+
+@router.put("/manual_update", response_model=MispUpdatePutResponse)
+async def manual_misp_update(request: Request, update_request: MispUpdatePutRequest, background_tasks: BackgroundTasks):
+	published_time = update_request.published_timestamp
+	misp_handler = request.app.misp_handler
+
+	background_tasks.add_task(misp_handler.full_update_once, p_time=published_time)
+
+	if not published_time:
+		published_time = "default"
+	return {"time": published_time, "state": "Starting"}
+
+
+@router.get("/domain/blocked/{domain}")
+async def domain_blocked(domain: str, db: db_dependency):
+	same_elements = and_(
+		Domain.events.contains(Domain.ignored_events),
+		Domain.events.contained_by(Domain.ignored_events)
+	)
+
+	domain_model = (db.query(Domain).filter(Domain.domain == domain,
+	                                        ~Domain.always_allowed,
+	                                        ~same_elements
+	                                        ).first())
+
+	return {"is_blocked": bool(domain_model)}
+
+
+@router.get("/domain/search")
+async def domain_search(domain: str, db: db_dependency):
+	domain = domain.replace("*", "%")
+
+	same_elements = and_(
+		Domain.events.contains(Domain.ignored_events),
+		Domain.events.contained_by(Domain.ignored_events)
+	)
+
+	query = db.query(Domain, same_elements.label("is_ignored)")).filter(Domain.domain.ilike(domain))
+	results = query.all()
+
+	def domain_status(item_details):
+		if item_details[0].always_allowed:
+			return "allowed"
+		elif item_details[1]:
+			return "ignored"
+		else:
+			return "blocked"
+
+	return {item[0].domain: domain_status(item) for item in results}
+
+@router.patch("/domain/always_allowed/{domain}")
+async def always_allowed(db: db_dependency, domain: str, allow: bool):
+	domain_model = db.query(Domain).filter(Domain.domain == domain).first()
+
+	if domain_model:
+		domain_model.always_allowed = allow
+	else:
+		domain_model = Domain(domain=domain, events=[], always_allowed=allow)
+
+	db.add(domain_model)
+	db.commit()
+
+
+@router.patch("/domain/events/{domain}/ignore")
+async def event_ignore(domain: str, db: db_dependency, event: int):
+	domain_model = db.query(Domain).filter(Domain.domain == domain).first()
+	if not domain_model:
+		raise HTTPException(status_code=404, detail="Domain Not Found")
+
+	if event not in domain_model.events:
+		raise HTTPException(status_code=404, detail="Event Not Found")
+
+	ignored_events = domain_model.ignored_events or []
+
+	if event in ignored_events:
+		return {"status": "Event Ignored"}
+
+	domain_model.ignored_events = ignored_events + [event]
+
+	db.commit()
+
+	return {"status": "Event Ignored"}
+
+
+@router.patch("/domain/events/{domain}/reinstate")
+async def event_reinstate(domain: str, db: db_dependency, event: int):
+	domain_model = db.query(Domain).filter(Domain.domain == domain).first()
+	if not domain_model:
+		raise HTTPException(status_code=404, detail="Domain Not Found")
+
+	ignored_events = domain_model.ignored_events
+
+	if not ignored_events or event not in ignored_events:
+		return {"status": "Event Ignored"}
+
+	ignored_events.remove(event)
+	domain_model.ignored_events = ignored_events
+
+	db.add(domain_model)
+	db.commit()
+
+	return {"status": "Event Un-ignored"}

src/misp/schemas.py

@@ -0,0 +1,20 @@
+"""
+Pydantic models for <this module>
+
+Models:
+ - List: Description
+ - Models: Description
+"""
+from typing import Optional
+from pydantic import Field
+
+from src.schemas import CustomBaseModel
+
+
+class MispUpdatePutRequest(CustomBaseModel):
+	published_timestamp: Optional[str] = Field(default=None, description="Timestamp for how far back to check for published timestamps")
+
+
+class MispUpdatePutResponse(CustomBaseModel):
+	time: str
+	state: str

src/misp/service.py

@@ -0,0 +1,268 @@
+"""
+Module specific business logic for the MISP module
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""
+import os
+import json
+import threading
+
+from typing import Any, Optional
+from sqlalchemy.sql import and_
+from pymisp import PyMISP
+from unbound_console import RemoteControl
+
+from src.config import settings as global_settings
+from src.misp.config import settings as misp_settings
+
+from src.database import get_db
+from src.misp.models import Domain
+from src.prometheus import prometheus
+
+
+class MISPHandler:
+	def __init__(self) -> None:
+		self.CONFIG_HEADER: str = """server:
+	interface: 0.0.0.0@853
+	interface: ::0@853
+	interface: 0.0.0.0@443
+	interface: ::0@443
+	interface: 0.0.0.0@53
+	interface: ::0@53
+	tls-service-key: "/etc/letsencrypt/live/dns.sr2.uk/privkey.pem"
+	tls-service-pem: "/etc/letsencrypt/live/dns.sr2.uk/fullchain.pem"
+	tls-port: 853
+	https-port: 443
+	access-control: 0.0.0.0/0 allow
+	access-control: ::/0 allow
+	do-ip4: yes
+	do-ip6: yes
+	do-udp: yes
+	do-tcp: yes
+	prefetch: yes
+"""
+		self.scheduled_published_timestamp = "7d"
+		self.domains: dict[str, set[int]] = {}
+		self.timer: Optional[threading.Thread] = None
+		self._stop_event: threading.Event = threading.Event()
+		self.is_fetching: bool = False
+		self.dev_dump: bool = True
+		self.timer_time: int = 30
+
+		self.pymisp = PyMISP(
+			url="https://misp.civicert.org",
+			key=open(misp_settings.MISP_KEY_FILE, "r").read().strip(),
+		)
+
+		prometheus.TIMER_STATE.state("stopped")
+		prometheus.UNBOUND_RELOAD_RESPONSE.state("not reloaded")
+
+		self.update_metrics()
+
+	async def start_timer(self) -> None:
+		if self.timer is not None and self.timer.is_alive():
+			print("Timer already started")
+			return
+
+		self._stop_event.clear()
+
+		self.timer = threading.Thread(target=self.run_with_timer, daemon=True)
+		if self.timer:
+			self.timer.start()
+
+	def run_with_timer(self) -> None:
+		prometheus.TIMER_STATE.state("running")
+		try:
+			while not self._stop_event.wait(self.timer_time):
+				self.full_update_once()
+		finally:
+			prometheus.TIMER_STATE.state("stopped")
+			self.timer = None
+			self._stop_event.clear()
+
+	def stop_timer(self) -> None:
+		if self.timer is None:
+			return
+
+		prometheus.TIMER_STATE.state("stopping")
+		self._stop_event.set()
+		self.timer.join()
+		self.timer = None
+
+	@staticmethod
+	def is_event_tlp_ok(event: dict[str, Any]) -> bool:
+		for tag in event.get("Tag", []):
+			if tag["name"] in misp_settings.ALLOWED_TLP:
+				return True
+		return False
+
+	def is_tlp_ok(self, attribute: dict[str, Any]) -> bool:
+		for tag in attribute.get("Tag", []):
+			if tag["name"] in misp_settings.IGNORED_TLP:
+				return False
+			if tag["name"] in misp_settings.ALLOWED_TLP:
+				return True
+		return self.is_event_tlp_ok(attribute["Event"])
+
+	def add_domains_to_db(self) -> None:
+		prometheus.MISP_STATE.state("updating")
+		db = next(get_db())
+
+		for domain, events in self.domains.items():
+			domain_model = (db.query(Domain).filter(Domain.domain == domain).first())
+			if domain_model:
+				existing_events = set(domain_model.events)
+				existing_events.update(events)
+				domain_model.events = list(existing_events)
+			else:
+				domain_model = Domain(domain=domain, events=list(events), always_allowed=False, ignored_events=[])
+
+			db.add(domain_model)
+		db.commit()
+		prometheus.MISP_STATE.state("idle")
+
+	def get_domains(self, type_: str, page: int, p_time: Optional[str]) -> tuple[dict[str, set[int]], int]:
+		if p_time is None:
+			p_time = self.scheduled_published_timestamp
+		# pymisp.search with these parameters returns a dict. Other params may return a different structure so inspection flags a type error.
+		# noinspection PyTypeChecker
+		data: dict[str, Any] = self.pymisp.search(
+			controller="attributes",
+			type_attribute=type_,
+			page=page,
+			limit=1000,
+			publish_timestamp=p_time,
+			published=True,
+			to_ids=True,
+			deleted="0",
+			include_event_tags=True
+		)
+
+		if self.dev_dump:
+			with open("data_dump.json", "w", encoding="utf-8") as dump_file:
+				json.dump(data, dump_file, indent=4)
+			self.dev_dump = False
+
+		domains = {}
+
+		for attribute in data["Attribute"]:
+			if not self.is_tlp_ok(attribute):
+				continue
+			domain = attribute["value"]
+			event_id = int(attribute["Event"]["id"])
+			if domain not in domains:
+				domains[domain] = {event_id}
+			else:
+				domains[domain].add(event_id)
+
+		return domains, len(data["Attribute"])
+
+	def produce_domain_set(self, p_time: Optional[str]) -> None:
+		self.is_fetching = True
+		prometheus.MISP_STATE.state("fetching")
+		for type_ in ["domain", "hostname"]:
+			page = 1
+			while True:
+				page_domains, page_size = self.get_domains(type_, page, p_time)
+				if not page_size:
+					break
+
+				for key, value in page_domains.items():
+					if key in self.domains:
+						self.domains[key].update(value)
+					else:
+						self.domains[key] = value
+
+				print(type_, page, len(page_domains), len(self.domains.keys()))
+				page += 1
+				if global_settings.ENVIRONMENT.is_debug:
+					break
+		self.is_fetching = False
+		prometheus.MISP_STATE.state("idle")
+
+	def save_conf(self) -> None:
+		db = next(get_db())
+
+		same_elements = and_(
+			Domain.events.contains(Domain.ignored_events),
+			Domain.events.contained_by(Domain.ignored_events)
+		)
+
+		results = (db.query(Domain)
+		           .filter(
+						~Domain.always_allowed,
+						~same_elements
+					)
+		           .all()
+		           )
+
+		blocked_domains = [item.domain for item in results]
+
+		with open(misp_settings.MISP_OUTPUT_FILE, "w") as conf:
+			conf.write(self.CONFIG_HEADER)
+			for domain in blocked_domains:
+				conf.write(f'    local-zone: "{domain}" always_nxdomain\n')
+			conf.write('    local-zone: "stats.invalid." static\n')
+			conf.write(f'    local-data: "stats.invalid. TXT \'{len(self.domains.keys())} domains in blocklist"\n')
+
+	@staticmethod
+	def send_unbound_command(command) -> str:
+		rc = RemoteControl(
+			host=misp_settings.UNBOUND_HOST,
+			port=misp_settings.UNBOUND_PORT,
+			server_cert=os.path.join(misp_settings.UNBOUND_CERT_DIR, "unbound_server.pem"),
+			client_cert=os.path.join(misp_settings.UNBOUND_CERT_DIR, "unbound_control.pem"),
+			client_key=os.path.join(misp_settings.UNBOUND_CERT_DIR, "unbound_control.key"),
+		)
+
+		return rc.send_command(cmd=command)
+
+	def reload_service(self) -> None:
+		prometheus.MISP_STATE.state("reloading service")
+		response = self.send_unbound_command("reload")
+		if response:
+			if response == "0":
+				prometheus.UNBOUND_RELOAD_RESPONSE.state("success")
+			elif response == "1":
+				prometheus.UNBOUND_RELOAD_RESPONSE.state("error")
+		else:
+			prometheus.UNBOUND_RELOAD_RESPONSE.state("no response")
+		prometheus.MISP_STATE.state("idle")
+
+	@staticmethod
+	def update_metrics() -> None:
+		db = next(get_db())
+		same_elements = and_(
+			Domain.events.contains(Domain.ignored_events),
+			Domain.events.contained_by(Domain.ignored_events)
+		)
+
+		blocked_count = (db.query(Domain)
+		           .filter(
+						~Domain.always_allowed,
+						~same_elements
+					)
+		           .count()
+		           )
+		prometheus.blocked_domain_count.set(blocked_count)
+
+	@prometheus.last_update_length.time()
+	def full_update_once(self, p_time: Optional[str] = None) -> None:
+		if self.is_fetching:
+			return
+		self.produce_domain_set(p_time)
+		self.add_domains_to_db()
+		self.save_conf()
+
+		if not global_settings.ENVIRONMENT.is_debug:
+			self.reload_service()
+
+		prometheus.last_update_complete_time.set_to_current_time()
+		self.update_metrics()
+		print("Done")

src/misp/utils.py

@@ -0,0 +1,11 @@
+"""
+Non-business logic reusable functions and classes for <this module>
+
+Classes:
+ - List: Description
+ - Classes: Description
+
+Functions:
+ - List: Description
+ - Functions: Description
+"""

src/models.py

@@ -0,0 +1,4 @@
+"""
+Global database models
+"""
+

src/prometheus/prometheus.py

@@ -0,0 +1,51 @@
+from prometheus_client import Enum, Gauge
+
+
+APP_STATE = Enum(
+	"app_state",
+	"Current state of the application",
+	states=["starting", "running", "error"],
+	namespace="app"
+)
+
+MISP_STATE = Enum(
+	"misp_state",
+	"Current state of the ",
+	states=["idle", "fetching", "updating", "reloading service", "error"],
+	namespace="misp"
+)
+
+TIMER_STATE = Enum(
+	"timer_state",
+	"Current state of the timer",
+	states=["running", "stopping", "stopped"],
+	namespace="misp",
+	subsystem="timer"
+)
+
+UNBOUND_RELOAD_RESPONSE = Enum(
+	"unbound_reload_response",
+	"Response from last unbound reload command",
+	states=["success", "error", "no response", "not reloaded"],
+	namespace="misp",
+	subsystem="unbound"
+)
+
+blocked_domain_count = Gauge(
+	"blocked_domain_count",
+	"Number of blocked domains",
+	namespace="misp",
+	subsystem="domains"
+)
+
+last_update_length = Gauge(
+	"last_update_time",
+	"How long the last update took",
+	namespace="misp",
+)
+
+last_update_complete_time = Gauge(
+	"last_update_complete_time",
+	"The time at which the last update was completed",
+	namespace="misp",
+)

src/schemas.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel
+
+
+class CustomBaseModel(BaseModel):
+	pass

src/utils.py

@@ -0,0 +1,98 @@
+import asyncio
+import logging
+import inspect
+
+from functools import wraps
+from traceback import format_exception
+from typing import Coroutine, Callable, Any
+
+from starlette.concurrency import run_in_threadpool
+
+NoArgsNoReturnFuncT = Callable[[], None]
+NoArgsNoReturnAsyncFuncT = Callable[[], Coroutine[Any, Any, None]]
+ExcArgNoReturnFuncT = Callable[[Exception], None]
+ExcArgNoReturnAsyncFuncT = Callable[[Exception], Coroutine[Any, Any, None]]
+NoArgsNoReturnAnyFuncT = NoArgsNoReturnFuncT | NoArgsNoReturnAsyncFuncT
+ExcArgNoReturnAnyFuncT = ExcArgNoReturnFuncT | ExcArgNoReturnAsyncFuncT
+NoArgsNoReturnDecorator = Callable[[NoArgsNoReturnAnyFuncT], NoArgsNoReturnAsyncFuncT]
+
+
+async def _handle_repeat_func(func: NoArgsNoReturnAnyFuncT) -> None:
+	if inspect.iscoroutinefunction(func):
+		await func()
+	else:
+		await run_in_threadpool(func)
+
+
+async def _handle_repeat_exc(
+		exc: Exception, on_exception: ExcArgNoReturnAnyFuncT | None
+) -> None:
+	if on_exception:
+		if inspect.iscoroutinefunction(on_exception):
+			await on_exception(exc)
+		else:
+			await run_in_threadpool(on_exception, exc)
+
+
+def repeat_every(
+		*,
+		seconds: float,
+		wait_first: float | None = None,
+		max_repetitions: int | None = None,
+		on_complete: NoArgsNoReturnAnyFuncT | None = None,
+		on_exception: ExcArgNoReturnAnyFuncT | None = None,
+) -> NoArgsNoReturnDecorator:
+	"""
+	This function returns a decorator that modifies a function so it is periodically re-executed after its first call.
+
+	The function it decorates should accept no arguments and return nothing. If necessary, this can be accomplished
+	by using `functools.partial` or otherwise wrapping the target function prior to decoration.
+
+	Parameters
+	----------
+	seconds: float
+		The number of seconds to wait between repeated calls
+	wait_first: float (default None)
+		If not None, the function will wait for the given duration before the first call
+	max_repetitions: Optional[int] (default None)
+		The maximum number of times to call the repeated function. If `None`, the function is repeated forever.
+	on_complete: Optional[Callable[[], None]] (default None)
+		A function to call after the final repetition of the decorated function.
+	on_exception: Optional[Callable[[Exception], None]] (default None)
+		A function to call when an exception is raised by the decorated function.
+	"""
+
+	def decorator(func: NoArgsNoReturnAnyFuncT) -> NoArgsNoReturnAsyncFuncT:
+		"""
+		Converts the decorated function into a repeated, periodically-called version of itself.
+		"""
+
+		@wraps(func)
+		async def wrapped() -> None:
+			async def loop() -> None:
+				if wait_first is not None:
+					await asyncio.sleep(wait_first)
+
+				repetitions = 0
+				while max_repetitions is None or repetitions < max_repetitions:
+					try:
+						await _handle_repeat_func(func)
+
+					except Exception as exc:
+						formatted_exception = "".join(
+							format_exception(type(exc), exc, exc.__traceback__)
+						)
+						logging.error(formatted_exception)
+						await _handle_repeat_exc(exc, on_exception)
+
+					repetitions += 1
+					await asyncio.sleep(seconds)
+
+				if on_complete:
+					await _handle_repeat_func(on_complete)
+
+			asyncio.ensure_future(loop())
+
+		return wrapped
+
+	return decorator

template.env

@@ -0,0 +1,18 @@
+SECRET_KEY=""
+ENVIRONMENT=""
+
+OIDC_CONFIG=""
+OIDC_ISSUER=""
+OIDC_AUDIENCE=""
+CLIENT_ID=""
+
+DATABASE_NAME=""
+DATABASE_PORT=""
+DATABASE_HOSTNAME=""
+DATABASE_CREDENTIALS=""
+
+MISP_KEY_FILE=""
+MISP_OUTPUT_FILE=""
+ALLOWED_TLP="[]"
+IGNORED_TLP="[]"
+UNBOUND_CERT_DIR=""