commit ac39206625e808219d2d7cdaa9824f5f1a915843 Author: luxferre Date: Wed May 6 12:02:26 2026 +0100 init diff --git a/.alembic/README b/.alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/.alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/.alembic/env.py b/.alembic/env.py new file mode 100644 index 0000000..da69861 --- /dev/null +++ b/.alembic/env.py @@ -0,0 +1,80 @@ +from logging.config import fileConfig + +from sqlalchemy import create_engine +# noinspection PyUnusedImportsInspection +from sqlalchemy import pool + +from alembic import context + +from src.config import SQLALCHEMY_DATABASE_URI + +from src.database import Base +# noinspection PyUnusedImportsInspection +from src.misp.models import Domain + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = create_engine(SQLALCHEMY_DATABASE_URI.get_secret_value()) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/.alembic/script.py.mako b/.alembic/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/.alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/.alembic/versions/2026-04-14_add_events_array_column.py b/.alembic/versions/2026-04-14_add_events_array_column.py new file mode 100644 index 0000000..a8043ad --- /dev/null +++ b/.alembic/versions/2026-04-14_add_events_array_column.py @@ -0,0 +1,32 @@ +"""add events array column + +Revision ID: de3a912a492c +Revises: 85f864dba26f +Create Date: 2026-04-14 16:36:51.595658 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'de3a912a492c' +down_revision: Union[str, Sequence[str], None] = '85f864dba26f' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('domain', sa.Column('events', sa.ARRAY(sa.Integer()), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('domain', 'events') + # ### end Alembic commands ### diff --git a/.alembic/versions/2026-04-14_initial_database_model.py b/.alembic/versions/2026-04-14_initial_database_model.py new file mode 100644 index 0000000..3ed7783 --- /dev/null +++ b/.alembic/versions/2026-04-14_initial_database_model.py @@ -0,0 +1,38 @@ +"""initial database model + +Revision ID: 85f864dba26f +Revises: +Create Date: 2026-04-14 13:30:14.536202 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '85f864dba26f' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('domain', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('domain', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_domain_domain'), 'domain', ['domain'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_domain_domain'), table_name='domain') + op.drop_table('domain') + # ### end Alembic commands ### diff --git a/.alembic/versions/2026-04-15_add_allowlisting_columns.py b/.alembic/versions/2026-04-15_add_allowlisting_columns.py new file mode 100644 index 0000000..69c0e56 --- /dev/null +++ b/.alembic/versions/2026-04-15_add_allowlisting_columns.py @@ -0,0 +1,34 @@ +"""add allowlisting columns + +Revision ID: 4ac5a5cbc12f +Revises: de3a912a492c +Create Date: 2026-04-15 14:44:03.825138 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '4ac5a5cbc12f' +down_revision: Union[str, Sequence[str], None] = 'de3a912a492c' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('domain', sa.Column('always_allowed', sa.Boolean(), nullable=True)) + op.add_column('domain', sa.Column('ignored_events', postgresql.ARRAY(sa.Integer()), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('domain', 'ignored_events') + op.drop_column('domain', 'always_allowed') + # ### end Alembic commands ### diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a92ef6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,209 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +#pdm.lock +#pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +#pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Cursor +# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to +# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data +# refer to https://docs.cursor.com/context/ignore-files +.cursorignore +.cursorindexingignore + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +key diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..43a6c21 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.13-slim AS build-env + +COPY . /app +WORKDIR /app + +RUN pip install --no-cache-dir -r requirements.txt +RUN pip install "psycopg[binary,pool]" + +FROM gcr.io/distroless/python3-debian13 + +COPY --from=build-env /usr/local/lib /usr/local/lib +COPY --from=build-env /usr/local/bin /usr/local/bin + +COPY --from=build-env /app /app + +EXPOSE 8000 + +ENV PYTHONPATH=/usr/local/lib/python3.13/site-packages + +WORKDIR /app + +CMD ["/usr/local/bin/uvicorn", "src.main:app", "--host", "0.0.0.0"] diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..ffe7e34 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,149 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/.alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +file_template = %%(year)d-%%(month).2d-%%(day).2d_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = none + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0d8217f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +fastapi +pydantic +uvicorn +jinja2 +python-dotenv +requests +itsdangerous +starlette +pydantic-settings +authlib +httpx +types-authlib +python-jose +pytest +uvloop; sys_platform != 'win32' +sqlalchemy +httptools +psycopg +alembic +pymisp +prometheus-client +unbound_console diff --git a/src/_module_template/config.py b/src/_module_template/config.py new file mode 100644 index 0000000..4be170e --- /dev/null +++ b/src/_module_template/config.py @@ -0,0 +1,7 @@ +""" +Configurations for + +Configurations: + - List: Description + - Configs: Description +""" \ No newline at end of file diff --git a/src/_module_template/constants.py b/src/_module_template/constants.py new file mode 100644 index 0000000..e1df957 --- /dev/null +++ b/src/_module_template/constants.py @@ -0,0 +1,7 @@ +""" +Constants and error codes for + +Constants: + - List: Description + - Consts: Description +""" \ No newline at end of file diff --git a/src/_module_template/dependencies.py b/src/_module_template/dependencies.py new file mode 100644 index 0000000..7447aaf --- /dev/null +++ b/src/_module_template/dependencies.py @@ -0,0 +1,11 @@ +""" +Router dependencies for + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/_module_template/exceptions.py b/src/_module_template/exceptions.py new file mode 100644 index 0000000..5debbb4 --- /dev/null +++ b/src/_module_template/exceptions.py @@ -0,0 +1,7 @@ +""" +Module specific exceptions for + +Exceptions: + - List: Description + - Exceptions: Description +""" \ No newline at end of file diff --git a/src/_module_template/models.py b/src/_module_template/models.py new file mode 100644 index 0000000..6d2494b --- /dev/null +++ b/src/_module_template/models.py @@ -0,0 +1,7 @@ +""" +Database models for + +Models: + - List: Description + - Models: Description +""" \ No newline at end of file diff --git a/src/_module_template/router.py b/src/_module_template/router.py new file mode 100644 index 0000000..2eb8569 --- /dev/null +++ b/src/_module_template/router.py @@ -0,0 +1,13 @@ +""" +Router endpoints for + +Endpoints: + - List: Description + - Endpoints: Description +""" +from fastapi import APIRouter + + +_router = APIRouter( + tags=[""], +) \ No newline at end of file diff --git a/src/_module_template/schemas.py b/src/_module_template/schemas.py new file mode 100644 index 0000000..a074c75 --- /dev/null +++ b/src/_module_template/schemas.py @@ -0,0 +1,7 @@ +""" +Pydantic models for + +Models: + - List: Description + - Models: Description +""" \ No newline at end of file diff --git a/src/_module_template/service.py b/src/_module_template/service.py new file mode 100644 index 0000000..7365fa9 --- /dev/null +++ b/src/_module_template/service.py @@ -0,0 +1,11 @@ +""" +Module specific business logic for + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/_module_template/utils.py b/src/_module_template/utils.py new file mode 100644 index 0000000..f2da15a --- /dev/null +++ b/src/_module_template/utils.py @@ -0,0 +1,11 @@ +""" +Non-business logic reusable functions and classes for + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/api.py b/src/api.py new file mode 100644 index 0000000..27fdab8 --- /dev/null +++ b/src/api.py @@ -0,0 +1,21 @@ +""" +This module hooks the routers for the main endpoints into a single router for importing to the app. +""" +from fastapi import APIRouter + +from src.auth.router import router as auth_router +from src.misp.router import router as misp_router +from src.control.router import router as control_router + + +api_router = APIRouter() + +api_router.include_router(auth_router) +api_router.include_router(misp_router) +api_router.include_router(control_router) + + +@api_router.get("/healthcheck", include_in_schema=False) +def healthcheck(): + """Simple health check endpoint.""" + return {"status": "ok"} diff --git a/src/auth/config.py b/src/auth/config.py new file mode 100644 index 0000000..99803fd --- /dev/null +++ b/src/auth/config.py @@ -0,0 +1,13 @@ +""" +Configurations for auth module, import auth_settings +""" +from src.config import CustomBaseSettings + +class AuthConfig(CustomBaseSettings): + OIDC_CONFIG: str = "" + OIDC_ISSUER: str = "" + OIDC_AUDIENCE: str = "" + CLIENT_ID: str = "" + + +auth_settings = AuthConfig() diff --git a/src/auth/constants.py b/src/auth/constants.py new file mode 100644 index 0000000..1ab3fd6 --- /dev/null +++ b/src/auth/constants.py @@ -0,0 +1,7 @@ +""" +Constants and error codes for auth module + +Constants: + - List: Description + - Consts: Description +""" \ No newline at end of file diff --git a/src/auth/dependencies.py b/src/auth/dependencies.py new file mode 100644 index 0000000..0001fb8 --- /dev/null +++ b/src/auth/dependencies.py @@ -0,0 +1,11 @@ +""" +Router dependencies for auth module + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/auth/exceptions.py b/src/auth/exceptions.py new file mode 100644 index 0000000..3861aad --- /dev/null +++ b/src/auth/exceptions.py @@ -0,0 +1,7 @@ +""" +Module specific exceptions for auth module + +Exceptions: + - List: Description + - Exceptions: Description +""" \ No newline at end of file diff --git a/src/auth/models.py b/src/auth/models.py new file mode 100644 index 0000000..487b3d1 --- /dev/null +++ b/src/auth/models.py @@ -0,0 +1,7 @@ +""" +Database models for auth module + +Models: + - List: Description + - Models: Description +""" \ No newline at end of file diff --git a/src/auth/router.py b/src/auth/router.py new file mode 100644 index 0000000..5e8871d --- /dev/null +++ b/src/auth/router.py @@ -0,0 +1,11 @@ +""" +Router endpoints for auth module +Contains oauth registration + +Endpoints: +""" +from fastapi import APIRouter + +router = APIRouter( + tags=["auth"], +) \ No newline at end of file diff --git a/src/auth/schemas.py b/src/auth/schemas.py new file mode 100644 index 0000000..92afae5 --- /dev/null +++ b/src/auth/schemas.py @@ -0,0 +1,7 @@ +""" +Pydantic models for auth module + +Models: + - List: Description + - Models: Description +""" \ No newline at end of file diff --git a/src/auth/service.py b/src/auth/service.py new file mode 100644 index 0000000..28f1c9c --- /dev/null +++ b/src/auth/service.py @@ -0,0 +1,63 @@ +""" +Module specific business logic for auth module + +Exports: + - claims_dependency + - authed_dependency +""" +import json + +from typing import Annotated +from authlib.jose import jwt +from urllib.request import urlopen + +from fastapi import Depends, HTTPException +from fastapi.security import OpenIdConnect +from authlib.jose.rfc7517.jwk import JsonWebKey +from authlib.jose.rfc7517.key_set import KeySet +from authlib.oauth2.rfc7523.validator import JWTBearerToken + +from src.auth.config import auth_settings + + +oidc = OpenIdConnect(openIdConnectUrl=auth_settings.OIDC_CONFIG) +oidc_dependency = Annotated[str, Depends(oidc)] + + +async def get_current_user(oidc_auth_string: oidc_dependency) -> JWTBearerToken: + config_url = urlopen(auth_settings.OIDC_CONFIG) + config = json.loads(config_url.read()) + jwks_uri = config["jwks_uri"] + key_response = urlopen(jwks_uri) + jwk_keys: KeySet = JsonWebKey.import_key_set(json.loads(key_response.read())) + + claims_options = { + "exp": {"essential": True}, + "aud": {"essential": True, "value": "account"}, + "iss": {"essential": True, "value": auth_settings.OIDC_ISSUER}, + } + + claims: JWTBearerToken = jwt.decode( + oidc_auth_string.replace("Bearer ", ""), + jwk_keys, + claims_options=claims_options, + claims_cls=JWTBearerToken, + ) + + claims.validate() + + return claims + + +claims_dependency = Annotated[JWTBearerToken, Depends(get_current_user)] + + +async def is_authed_user(claims: claims_dependency) -> bool: + authed_users: list[str] = ["chris@sr2.uk"] + user_email = claims.get("email", None) + if not user_email or user_email not in authed_users: + raise HTTPException(status_code=403, detail="Not authenticated") + return claims.get("email") in authed_users + + +authed_dependency = Annotated[bool, Depends(is_authed_user)] diff --git a/src/auth/utils.py b/src/auth/utils.py new file mode 100644 index 0000000..e913437 --- /dev/null +++ b/src/auth/utils.py @@ -0,0 +1,11 @@ +""" +Non-business logic reusable functions and classes for auth module + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..763fff5 --- /dev/null +++ b/src/config.py @@ -0,0 +1,53 @@ +""" +Global configurations: import settings, app_configs + +Classes: + - CustomBaseSettings - Base class to be used by all modules for loading configs +""" + +from typing import Any +from urllib import parse +from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import SecretStr + +from src.constants import Environment + + +class CustomBaseSettings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="ignore" + ) + + +class Config(CustomBaseSettings): + APP_VERSION: str = "0.1" + ENVIRONMENT: Environment = Environment.PRODUCTION + SECRET_KEY: SecretStr = "" + + CORS_ORIGINS: list[str] = ["*"] + CORS_ORIGINS_REGEX: str | None = None + CORS_HEADERS: list[str] = ["*"] + + DATABASE_NAME: str = "dns-exp" + DATABASE_PORT: str = "5432" + DATABASE_HOSTNAME: str = "localhost" + DATABASE_CREDENTIALS: SecretStr = "" + +settings = Config() + +DATABASE_NAME = settings.DATABASE_NAME +DATABASE_PORT = settings.DATABASE_PORT +DATABASE_HOSTNAME = settings.DATABASE_HOSTNAME +DATABASE_CREDENTIALS = settings.DATABASE_CREDENTIALS.get_secret_value() +# this will support special chars for credentials +_DATABASE_CREDENTIAL_USER, _DATABASE_CREDENTIAL_PASSWORD = str(DATABASE_CREDENTIALS).split(":") +_QUOTED_DATABASE_PASSWORD = parse.quote_plus(str(_DATABASE_CREDENTIAL_PASSWORD)) + +SQLALCHEMY_DATABASE_URI = SecretStr(f"postgresql+psycopg://{_DATABASE_CREDENTIAL_USER}:{_QUOTED_DATABASE_PASSWORD}@{DATABASE_HOSTNAME}:{DATABASE_PORT}/{DATABASE_NAME}") + +app_configs: dict[str, Any] = {"title": "App API"} +if settings.ENVIRONMENT.is_deployed: + app_configs["root_path"] = f"/v{settings.APP_VERSION}" + +if not settings.ENVIRONMENT.is_debug: + app_configs["openapi_url"] = None # hide docs diff --git a/src/constants.py b/src/constants.py new file mode 100644 index 0000000..ab33afb --- /dev/null +++ b/src/constants.py @@ -0,0 +1,36 @@ +""" +Global constants + +Classes: + - Environment(StrEnum): LOCAL, TESTING, STAGING, PRODUCTION +""" +from enum import StrEnum, auto + + +class Environment(StrEnum): + """ + Enumeration of environments. + + Attributes: + LOCAL (str): Application is running locally + TESTING (str): Application is running in testing mode + STAGING (str): Application is running in staging mode (ie not testing) + PRODUCTION (str): Application is running in production mode + """ + + LOCAL = auto() + TESTING = auto() + STAGING = auto() + PRODUCTION = auto() + + @property + def is_debug(self): + return self in (self.LOCAL, self.STAGING, self.TESTING) + + @property + def is_testing(self): + return self == self.TESTING + + @property + def is_deployed(self) -> bool: + return self in (self.STAGING, self.PRODUCTION) diff --git a/src/control/config.py b/src/control/config.py new file mode 100644 index 0000000..4be170e --- /dev/null +++ b/src/control/config.py @@ -0,0 +1,7 @@ +""" +Configurations for + +Configurations: + - List: Description + - Configs: Description +""" \ No newline at end of file diff --git a/src/control/constants.py b/src/control/constants.py new file mode 100644 index 0000000..4875d28 --- /dev/null +++ b/src/control/constants.py @@ -0,0 +1,13 @@ +""" +Constants and error codes for + +Constants: + - List: Description + - Consts: Description +""" +from enum import StrEnum, auto + + +class TimerState(StrEnum): + STARTING = auto() + STOPPING = auto() diff --git a/src/control/dependencies.py b/src/control/dependencies.py new file mode 100644 index 0000000..7447aaf --- /dev/null +++ b/src/control/dependencies.py @@ -0,0 +1,11 @@ +""" +Router dependencies for + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/control/exceptions.py b/src/control/exceptions.py new file mode 100644 index 0000000..5debbb4 --- /dev/null +++ b/src/control/exceptions.py @@ -0,0 +1,7 @@ +""" +Module specific exceptions for + +Exceptions: + - List: Description + - Exceptions: Description +""" \ No newline at end of file diff --git a/src/control/models.py b/src/control/models.py new file mode 100644 index 0000000..6d2494b --- /dev/null +++ b/src/control/models.py @@ -0,0 +1,7 @@ +""" +Database models for + +Models: + - List: Description + - Models: Description +""" \ No newline at end of file diff --git a/src/control/router.py b/src/control/router.py new file mode 100644 index 0000000..ae3dcee --- /dev/null +++ b/src/control/router.py @@ -0,0 +1,34 @@ +""" +Router endpoints for the control module + +Endpoints: + - List: Description + - Endpoints: Description +""" +from fastapi import APIRouter, Request +from starlette import status + +from src.auth.service import authed_dependency + +from src.control.schemas import ControlTimerPutResponse + +router = APIRouter( + tags=["control"], + prefix="/control", +) + + +@router.put("/start_timer", status_code=status.HTTP_202_ACCEPTED, response_model=ControlTimerPutResponse) +async def start_timer(request: Request): + misp_handler = request.app.misp_handler + + await misp_handler.start_timer() + return {"state": "starting"} + + +@router.put("/stop_timer", status_code=status.HTTP_202_ACCEPTED, response_model=ControlTimerPutResponse) +async def stop_timer(request: Request): + misp_handler = request.app.misp_handler + + misp_handler.stop_timer() + return {"state": "stopping"} diff --git a/src/control/schemas.py b/src/control/schemas.py new file mode 100644 index 0000000..8677b02 --- /dev/null +++ b/src/control/schemas.py @@ -0,0 +1,14 @@ +""" +Pydantic models for + +Models: + - List: Description + - Models: Description +""" +from src.schemas import CustomBaseModel + +from src.control.constants import TimerState + + +class ControlTimerPutResponse(CustomBaseModel): + state: TimerState diff --git a/src/control/service.py b/src/control/service.py new file mode 100644 index 0000000..7365fa9 --- /dev/null +++ b/src/control/service.py @@ -0,0 +1,11 @@ +""" +Module specific business logic for + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/control/utils.py b/src/control/utils.py new file mode 100644 index 0000000..f2da15a --- /dev/null +++ b/src/control/utils.py @@ -0,0 +1,11 @@ +""" +Non-business logic reusable functions and classes for + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/database.py b/src/database.py new file mode 100644 index 0000000..9b4a199 --- /dev/null +++ b/src/database.py @@ -0,0 +1,31 @@ +""" +Database connections and init + +Exports: + - db_dependency + - Base (sqlalchemy base model) + - get_db +""" +from typing import Annotated +from sqlalchemy import create_engine +from sqlalchemy.orm import declarative_base, sessionmaker, Session + +from fastapi import Depends + +from src.config import SQLALCHEMY_DATABASE_URI + +engine = create_engine(SQLALCHEMY_DATABASE_URI.get_secret_value()) + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +def get_db(): + with SessionLocal.begin() as db: + try: + yield db + finally: + db.rollback() # Anything not explicitly commited is rolled back + db.close() + + +db_dependency = Annotated[Session, Depends(get_db)] +Base = declarative_base() diff --git a/src/exceptions.py b/src/exceptions.py new file mode 100644 index 0000000..b18e221 --- /dev/null +++ b/src/exceptions.py @@ -0,0 +1,3 @@ +""" +Global exceptions +""" \ No newline at end of file diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..c85985e --- /dev/null +++ b/src/main.py @@ -0,0 +1,67 @@ +""" +Application root file: Inits the FastAPI application +""" +from contextlib import asynccontextmanager +from typing import AsyncGenerator +from prometheus_client import make_asgi_app + +from fastapi import FastAPI +from starlette.middleware.sessions import SessionMiddleware +from starlette.middleware.cors import CORSMiddleware + +from src.config import settings +from src.api import api_router +from src.prometheus import prometheus + +from src.auth.config import auth_settings +from src.misp.service import MISPHandler + +# TODO: Create Pydantic request/response schemas + + +@asynccontextmanager +async def lifespan(_application: FastAPI) -> AsyncGenerator: + # Startup + yield + # Shutdown + _application.misp_handler.stop_timer() + + +if settings.ENVIRONMENT.is_deployed: + # Do this only on prod + pass + + +app = FastAPI( + swagger_ui_init_oauth={ + "clientId": auth_settings.CLIENT_ID, + "usePkceWithAuthorizationCodeGrant": True, + "scopes": "openid profile email", + } +) + + +metrics_app = make_asgi_app() +app.mount("/metrics", metrics_app) + +prometheus.APP_STATE.state("starting") + +# Type inspection disabled for middleware injection. +# Known bug in FastAPI type checking: https://github.com/astral-sh/ty/issues/1635 +# noinspection PyTypeChecker +app.add_middleware(SessionMiddleware, secret_key=settings.SECRET_KEY.get_secret_value()) +# noinspection PyTypeChecker +app.add_middleware( + CORSMiddleware, + allow_origins=settings.CORS_ORIGINS, + allow_origin_regex=settings.CORS_ORIGINS_REGEX, + allow_credentials=True, + allow_methods=("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"), + allow_headers=settings.CORS_HEADERS, +) + +app.include_router(api_router) + +app.misp_handler = MISPHandler() + +prometheus.APP_STATE.state("running") diff --git a/src/misp/config.py b/src/misp/config.py new file mode 100644 index 0000000..e0dfc0e --- /dev/null +++ b/src/misp/config.py @@ -0,0 +1,14 @@ +""" +Configurations for the MISP module +""" +from src.config import CustomBaseSettings + +class MISPConfig(CustomBaseSettings): + MISP_KEY_FILE: str = "" + MISP_OUTPUT_FILE: str = "" + ALLOWED_TLP: list[str] = ["tlp:clear", "tlp:white", "tlp:green"] + IGNORED_TLP: list[str] = ["tlp:red", "tlp:amber+strict", "tlp:amber"] + UNBOUND_CERT_DIR: str = "" + + +settings = MISPConfig() diff --git a/src/misp/constants.py b/src/misp/constants.py new file mode 100644 index 0000000..e1df957 --- /dev/null +++ b/src/misp/constants.py @@ -0,0 +1,7 @@ +""" +Constants and error codes for + +Constants: + - List: Description + - Consts: Description +""" \ No newline at end of file diff --git a/src/misp/dependencies.py b/src/misp/dependencies.py new file mode 100644 index 0000000..7447aaf --- /dev/null +++ b/src/misp/dependencies.py @@ -0,0 +1,11 @@ +""" +Router dependencies for + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/misp/exceptions.py b/src/misp/exceptions.py new file mode 100644 index 0000000..5debbb4 --- /dev/null +++ b/src/misp/exceptions.py @@ -0,0 +1,7 @@ +""" +Module specific exceptions for + +Exceptions: + - List: Description + - Exceptions: Description +""" \ No newline at end of file diff --git a/src/misp/models.py b/src/misp/models.py new file mode 100644 index 0000000..b71c1df --- /dev/null +++ b/src/misp/models.py @@ -0,0 +1,18 @@ +""" +Database models for the misp module +""" +from sqlalchemy import Column, Integer, String, Boolean +from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy.ext.mutable import MutableList + +from src.database import Base + + +class Domain(Base): + __tablename__ = "domain" + + id = Column(Integer, primary_key=True) + domain = Column(String, index=True) + events = Column(MutableList.as_mutable(ARRAY(Integer)), default=[]) + always_allowed = Column(Boolean, default=False) + ignored_events = Column(MutableList.as_mutable(ARRAY(Integer)), default=[]) diff --git a/src/misp/router.py b/src/misp/router.py new file mode 100644 index 0000000..e985c25 --- /dev/null +++ b/src/misp/router.py @@ -0,0 +1,123 @@ +""" +Router endpoints for + +Endpoints: + - List: Description + - Endpoints: Description +""" +from fastapi import APIRouter, HTTPException, Request, BackgroundTasks +from sqlalchemy.sql import and_ + +from src.auth.service import authed_dependency +from src.database import db_dependency + +from src.misp.models import Domain +from src.misp.schemas import MispUpdatePutRequest, MispUpdatePutResponse + +router = APIRouter( + tags=["misp"], + prefix="/misp", +) + + +@router.put("/manual_update", response_model=MispUpdatePutResponse) +async def manual_misp_update(request: Request, update_request: MispUpdatePutRequest, background_tasks: BackgroundTasks): + published_time = update_request.published_timestamp + misp_handler = request.app.misp_handler + + background_tasks.add_task(misp_handler.full_update_once, p_time=published_time) + + if not published_time: + published_time = "default" + return {"time": published_time, "state": "Starting"} + + +@router.get("/domain/blocked/{domain}") +async def domain_blocked(domain: str, db: db_dependency): + same_elements = and_( + Domain.events.contains(Domain.ignored_events), + Domain.events.contained_by(Domain.ignored_events) + ) + + domain_model = (db.query(Domain).filter(Domain.domain == domain, + ~Domain.always_allowed, + ~same_elements + ).first()) + + return {"is_blocked": bool(domain_model)} + + +@router.get("/domain/search") +async def domain_search(domain: str, db: db_dependency): + domain = domain.replace("*", "%") + + same_elements = and_( + Domain.events.contains(Domain.ignored_events), + Domain.events.contained_by(Domain.ignored_events) + ) + + query = db.query(Domain, same_elements.label("is_ignored)")).filter(Domain.domain.ilike(domain)) + results = query.all() + + def domain_status(item_details): + if item_details[0].always_allowed: + return "allowed" + elif item_details[1]: + return "ignored" + else: + return "blocked" + + return {item[0].domain: domain_status(item) for item in results} + +@router.patch("/domain/always_allowed/{domain}") +async def always_allowed(db: db_dependency, domain: str, allow: bool): + domain_model = db.query(Domain).filter(Domain.domain == domain).first() + + if domain_model: + domain_model.always_allowed = allow + else: + domain_model = Domain(domain=domain, events=[], always_allowed=allow) + + db.add(domain_model) + db.commit() + + +@router.patch("/domain/events/{domain}/ignore") +async def event_ignore(domain: str, db: db_dependency, event: int): + domain_model = db.query(Domain).filter(Domain.domain == domain).first() + if not domain_model: + raise HTTPException(status_code=404, detail="Domain Not Found") + + if event not in domain_model.events: + raise HTTPException(status_code=404, detail="Event Not Found") + + ignored_events = domain_model.ignored_events or [] + + if event in ignored_events: + return {"status": "Event Ignored"} + + domain_model.ignored_events = ignored_events + [event] + + db.commit() + + return {"status": "Event Ignored"} + + +@router.patch("/domain/events/{domain}/reinstate") +async def event_reinstate(domain: str, db: db_dependency, event: int): + domain_model = db.query(Domain).filter(Domain.domain == domain).first() + if not domain_model: + raise HTTPException(status_code=404, detail="Domain Not Found") + + ignored_events = domain_model.ignored_events + + if not ignored_events or event not in ignored_events: + return {"status": "Event Ignored"} + + ignored_events.remove(event) + domain_model.ignored_events = ignored_events + + db.add(domain_model) + db.commit() + + return {"status": "Event Un-ignored"} diff --git a/src/misp/schemas.py b/src/misp/schemas.py new file mode 100644 index 0000000..6410d6d --- /dev/null +++ b/src/misp/schemas.py @@ -0,0 +1,20 @@ +""" +Pydantic models for + +Models: + - List: Description + - Models: Description +""" +from typing import Optional +from pydantic import Field + +from src.schemas import CustomBaseModel + + +class MispUpdatePutRequest(CustomBaseModel): + published_timestamp: Optional[str] = Field(default=None, description="Timestamp for how far back to check for published timestamps") + + +class MispUpdatePutResponse(CustomBaseModel): + time: str + state: str diff --git a/src/misp/service.py b/src/misp/service.py new file mode 100644 index 0000000..7a4f511 --- /dev/null +++ b/src/misp/service.py @@ -0,0 +1,268 @@ +""" +Module specific business logic for the MISP module + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" +import os +import json +import threading + +from typing import Any, Optional +from sqlalchemy.sql import and_ +from pymisp import PyMISP +from unbound_console import RemoteControl + +from src.config import settings as global_settings +from src.misp.config import settings as misp_settings + +from src.database import get_db +from src.misp.models import Domain +from src.prometheus import prometheus + + +class MISPHandler: + def __init__(self) -> None: + self.CONFIG_HEADER: str = """server: + interface: 0.0.0.0@853 + interface: ::0@853 + interface: 0.0.0.0@443 + interface: ::0@443 + interface: 0.0.0.0@53 + interface: ::0@53 + tls-service-key: "/etc/letsencrypt/live/dns.sr2.uk/privkey.pem" + tls-service-pem: "/etc/letsencrypt/live/dns.sr2.uk/fullchain.pem" + tls-port: 853 + https-port: 443 + access-control: 0.0.0.0/0 allow + access-control: ::/0 allow + do-ip4: yes + do-ip6: yes + do-udp: yes + do-tcp: yes + prefetch: yes +""" + self.scheduled_published_timestamp = "7d" + self.domains: dict[str, set[int]] = {} + self.timer: Optional[threading.Thread] = None + self._stop_event: threading.Event = threading.Event() + self.is_fetching: bool = False + self.dev_dump: bool = True + self.timer_time: int = 30 + + self.pymisp = PyMISP( + url="https://misp.civicert.org", + key=open(misp_settings.MISP_KEY_FILE, "r").read().strip(), + ) + + prometheus.TIMER_STATE.state("stopped") + prometheus.UNBOUND_RELOAD_RESPONSE.state("not reloaded") + + self.update_metrics() + + async def start_timer(self) -> None: + if self.timer is not None and self.timer.is_alive(): + print("Timer already started") + return + + self._stop_event.clear() + + self.timer = threading.Thread(target=self.run_with_timer, daemon=True) + if self.timer: + self.timer.start() + + def run_with_timer(self) -> None: + prometheus.TIMER_STATE.state("running") + try: + while not self._stop_event.wait(self.timer_time): + self.full_update_once() + finally: + prometheus.TIMER_STATE.state("stopped") + self.timer = None + self._stop_event.clear() + + def stop_timer(self) -> None: + if self.timer is None: + return + + prometheus.TIMER_STATE.state("stopping") + self._stop_event.set() + self.timer.join() + self.timer = None + + @staticmethod + def is_event_tlp_ok(event: dict[str, Any]) -> bool: + for tag in event.get("Tag", []): + if tag["name"] in misp_settings.ALLOWED_TLP: + return True + return False + + def is_tlp_ok(self, attribute: dict[str, Any]) -> bool: + for tag in attribute.get("Tag", []): + if tag["name"] in misp_settings.IGNORED_TLP: + return False + if tag["name"] in misp_settings.ALLOWED_TLP: + return True + return self.is_event_tlp_ok(attribute["Event"]) + + def add_domains_to_db(self) -> None: + prometheus.MISP_STATE.state("updating") + db = next(get_db()) + + for domain, events in self.domains.items(): + domain_model = (db.query(Domain).filter(Domain.domain == domain).first()) + if domain_model: + existing_events = set(domain_model.events) + existing_events.update(events) + domain_model.events = list(existing_events) + else: + domain_model = Domain(domain=domain, events=list(events), always_allowed=False, ignored_events=[]) + + db.add(domain_model) + db.commit() + prometheus.MISP_STATE.state("idle") + + def get_domains(self, type_: str, page: int, p_time: Optional[str]) -> tuple[dict[str, set[int]], int]: + if p_time is None: + p_time = self.scheduled_published_timestamp + # pymisp.search with these parameters returns a dict. Other params may return a different structure so inspection flags a type error. + # noinspection PyTypeChecker + data: dict[str, Any] = self.pymisp.search( + controller="attributes", + type_attribute=type_, + page=page, + limit=1000, + publish_timestamp=p_time, + published=True, + to_ids=True, + deleted="0", + include_event_tags=True + ) + + if self.dev_dump: + with open("data_dump.json", "w", encoding="utf-8") as dump_file: + json.dump(data, dump_file, indent=4) + self.dev_dump = False + + domains = {} + + for attribute in data["Attribute"]: + if not self.is_tlp_ok(attribute): + continue + domain = attribute["value"] + event_id = int(attribute["Event"]["id"]) + if domain not in domains: + domains[domain] = {event_id} + else: + domains[domain].add(event_id) + + return domains, len(data["Attribute"]) + + def produce_domain_set(self, p_time: Optional[str]) -> None: + self.is_fetching = True + prometheus.MISP_STATE.state("fetching") + for type_ in ["domain", "hostname"]: + page = 1 + while True: + page_domains, page_size = self.get_domains(type_, page, p_time) + if not page_size: + break + + for key, value in page_domains.items(): + if key in self.domains: + self.domains[key].update(value) + else: + self.domains[key] = value + + print(type_, page, len(page_domains), len(self.domains.keys())) + page += 1 + if global_settings.ENVIRONMENT.is_debug: + break + self.is_fetching = False + prometheus.MISP_STATE.state("idle") + + def save_conf(self) -> None: + db = next(get_db()) + + same_elements = and_( + Domain.events.contains(Domain.ignored_events), + Domain.events.contained_by(Domain.ignored_events) + ) + + results = (db.query(Domain) + .filter( + ~Domain.always_allowed, + ~same_elements + ) + .all() + ) + + blocked_domains = [item.domain for item in results] + + with open(misp_settings.MISP_OUTPUT_FILE, "w") as conf: + conf.write(self.CONFIG_HEADER) + for domain in blocked_domains: + conf.write(f' local-zone: "{domain}" always_nxdomain\n') + conf.write(' local-zone: "stats.invalid." static\n') + conf.write(f' local-data: "stats.invalid. TXT \'{len(self.domains.keys())} domains in blocklist"\n') + + @staticmethod + def send_unbound_command(command) -> str: + rc = RemoteControl( + host=misp_settings.UNBOUND_HOST, + port=misp_settings.UNBOUND_PORT, + server_cert=os.path.join(misp_settings.UNBOUND_CERT_DIR, "unbound_server.pem"), + client_cert=os.path.join(misp_settings.UNBOUND_CERT_DIR, "unbound_control.pem"), + client_key=os.path.join(misp_settings.UNBOUND_CERT_DIR, "unbound_control.key"), + ) + + return rc.send_command(cmd=command) + + def reload_service(self) -> None: + prometheus.MISP_STATE.state("reloading service") + response = self.send_unbound_command("reload") + if response: + if response == "0": + prometheus.UNBOUND_RELOAD_RESPONSE.state("success") + elif response == "1": + prometheus.UNBOUND_RELOAD_RESPONSE.state("error") + else: + prometheus.UNBOUND_RELOAD_RESPONSE.state("no response") + prometheus.MISP_STATE.state("idle") + + @staticmethod + def update_metrics() -> None: + db = next(get_db()) + same_elements = and_( + Domain.events.contains(Domain.ignored_events), + Domain.events.contained_by(Domain.ignored_events) + ) + + blocked_count = (db.query(Domain) + .filter( + ~Domain.always_allowed, + ~same_elements + ) + .count() + ) + prometheus.blocked_domain_count.set(blocked_count) + + @prometheus.last_update_length.time() + def full_update_once(self, p_time: Optional[str] = None) -> None: + if self.is_fetching: + return + self.produce_domain_set(p_time) + self.add_domains_to_db() + self.save_conf() + + if not global_settings.ENVIRONMENT.is_debug: + self.reload_service() + + prometheus.last_update_complete_time.set_to_current_time() + self.update_metrics() + print("Done") diff --git a/src/misp/utils.py b/src/misp/utils.py new file mode 100644 index 0000000..f2da15a --- /dev/null +++ b/src/misp/utils.py @@ -0,0 +1,11 @@ +""" +Non-business logic reusable functions and classes for + +Classes: + - List: Description + - Classes: Description + +Functions: + - List: Description + - Functions: Description +""" \ No newline at end of file diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..fa198e4 --- /dev/null +++ b/src/models.py @@ -0,0 +1,4 @@ +""" +Global database models +""" + diff --git a/src/prometheus/__init__.py b/src/prometheus/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/prometheus/prometheus.py b/src/prometheus/prometheus.py new file mode 100644 index 0000000..ce0492a --- /dev/null +++ b/src/prometheus/prometheus.py @@ -0,0 +1,51 @@ +from prometheus_client import Enum, Gauge + + +APP_STATE = Enum( + "app_state", + "Current state of the application", + states=["starting", "running", "error"], + namespace="app" +) + +MISP_STATE = Enum( + "misp_state", + "Current state of the ", + states=["idle", "fetching", "updating", "reloading service", "error"], + namespace="misp" +) + +TIMER_STATE = Enum( + "timer_state", + "Current state of the timer", + states=["running", "stopping", "stopped"], + namespace="misp", + subsystem="timer" +) + +UNBOUND_RELOAD_RESPONSE = Enum( + "unbound_reload_response", + "Response from last unbound reload command", + states=["success", "error", "no response", "not reloaded"], + namespace="misp", + subsystem="unbound" +) + +blocked_domain_count = Gauge( + "blocked_domain_count", + "Number of blocked domains", + namespace="misp", + subsystem="domains" +) + +last_update_length = Gauge( + "last_update_time", + "How long the last update took", + namespace="misp", +) + +last_update_complete_time = Gauge( + "last_update_complete_time", + "The time at which the last update was completed", + namespace="misp", +) diff --git a/src/schemas.py b/src/schemas.py new file mode 100644 index 0000000..e524f83 --- /dev/null +++ b/src/schemas.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + + +class CustomBaseModel(BaseModel): + pass diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..03d5466 --- /dev/null +++ b/src/utils.py @@ -0,0 +1,98 @@ +import asyncio +import logging +import inspect + +from functools import wraps +from traceback import format_exception +from typing import Coroutine, Callable, Any + +from starlette.concurrency import run_in_threadpool + +NoArgsNoReturnFuncT = Callable[[], None] +NoArgsNoReturnAsyncFuncT = Callable[[], Coroutine[Any, Any, None]] +ExcArgNoReturnFuncT = Callable[[Exception], None] +ExcArgNoReturnAsyncFuncT = Callable[[Exception], Coroutine[Any, Any, None]] +NoArgsNoReturnAnyFuncT = NoArgsNoReturnFuncT | NoArgsNoReturnAsyncFuncT +ExcArgNoReturnAnyFuncT = ExcArgNoReturnFuncT | ExcArgNoReturnAsyncFuncT +NoArgsNoReturnDecorator = Callable[[NoArgsNoReturnAnyFuncT], NoArgsNoReturnAsyncFuncT] + + +async def _handle_repeat_func(func: NoArgsNoReturnAnyFuncT) -> None: + if inspect.iscoroutinefunction(func): + await func() + else: + await run_in_threadpool(func) + + +async def _handle_repeat_exc( + exc: Exception, on_exception: ExcArgNoReturnAnyFuncT | None +) -> None: + if on_exception: + if inspect.iscoroutinefunction(on_exception): + await on_exception(exc) + else: + await run_in_threadpool(on_exception, exc) + + +def repeat_every( + *, + seconds: float, + wait_first: float | None = None, + max_repetitions: int | None = None, + on_complete: NoArgsNoReturnAnyFuncT | None = None, + on_exception: ExcArgNoReturnAnyFuncT | None = None, +) -> NoArgsNoReturnDecorator: + """ + This function returns a decorator that modifies a function so it is periodically re-executed after its first call. + + The function it decorates should accept no arguments and return nothing. If necessary, this can be accomplished + by using `functools.partial` or otherwise wrapping the target function prior to decoration. + + Parameters + ---------- + seconds: float + The number of seconds to wait between repeated calls + wait_first: float (default None) + If not None, the function will wait for the given duration before the first call + max_repetitions: Optional[int] (default None) + The maximum number of times to call the repeated function. If `None`, the function is repeated forever. + on_complete: Optional[Callable[[], None]] (default None) + A function to call after the final repetition of the decorated function. + on_exception: Optional[Callable[[Exception], None]] (default None) + A function to call when an exception is raised by the decorated function. + """ + + def decorator(func: NoArgsNoReturnAnyFuncT) -> NoArgsNoReturnAsyncFuncT: + """ + Converts the decorated function into a repeated, periodically-called version of itself. + """ + + @wraps(func) + async def wrapped() -> None: + async def loop() -> None: + if wait_first is not None: + await asyncio.sleep(wait_first) + + repetitions = 0 + while max_repetitions is None or repetitions < max_repetitions: + try: + await _handle_repeat_func(func) + + except Exception as exc: + formatted_exception = "".join( + format_exception(type(exc), exc, exc.__traceback__) + ) + logging.error(formatted_exception) + await _handle_repeat_exc(exc, on_exception) + + repetitions += 1 + await asyncio.sleep(seconds) + + if on_complete: + await _handle_repeat_func(on_complete) + + asyncio.ensure_future(loop()) + + return wrapped + + return decorator diff --git a/template.env b/template.env new file mode 100644 index 0000000..f39d526 --- /dev/null +++ b/template.env @@ -0,0 +1,18 @@ +SECRET_KEY="" +ENVIRONMENT="" + +OIDC_CONFIG="" +OIDC_ISSUER="" +OIDC_AUDIENCE="" +CLIENT_ID="" + +DATABASE_NAME="" +DATABASE_PORT="" +DATABASE_HOSTNAME="" +DATABASE_CREDENTIALS="" + +MISP_KEY_FILE="" +MISP_OUTPUT_FILE="" +ALLOWED_TLP="[]" +IGNORED_TLP="[]" +UNBOUND_CERT_DIR=""