ops/matrix-ops-bot
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add alertmanager as supported sender and update deps

Browse source
This commit is contained in:
Abel Luck 2023-11-07 15:14:56 +01:00
parent 05ffc640ed
commit 973e1fd789
18 changed files with 1682 additions and 1155 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -5,4 +5,6 @@ __pycache__
devstate
.env*
config.json
dev.data
dev.data
data
.direnv

7
.gitlab-ci.yml
View file

@ -11,15 +11,14 @@ variables:
REF_IMAGE: registry.gitlab.com/$CI_PROJECT_NAMESPACE/${CI_PROJECT_NAME}:$CI_COMMIT_REF_NAME
check:
image: python:3.10-bullseye
image: python:3.11-bullseye
stage: check
script:
- apt-get update
- apt-get install -y make libolm-dev
- pip install poetry
- apt-get install -y make libolm-dev python3-poetry
- poetry config virtualenvs.create false
- poetry install
- make ci
- make check
build-test:
image: docker:git

1
.python-version
View file

@ -1 +0,0 @@
3.10.5

4
Dockerfile
View file

@ -1,6 +1,6 @@
ARG PYTHON_VERSION=3.10
ARG PYTHON_VERSION=3.11
FROM docker.io/python:${PYTHON_VERSION}-alpine as builder
ARG LIBOLM_VERSION=3.2.10
ARG LIBOLM_VERSION=3.2.15
RUN apk add --no-cache \
make \
cmake \

23
Makefile
View file

@ -1,9 +1,21 @@
POETRY ?= poetry run
SRC := ops_bot
TESTS := tests
SHELL := $(shell which bash)
APP_VERSION := $(shell git rev-parse --short HEAD)
DOCKER ?= docker
docker-build:
DOCKER_BUILDKIT=1 $(DOCKER) build -f docker/Dockerfile \
--build-arg=$(APP_VERSION) \
-t matrix-ops-bot:latest \
.
fmt:
$(POETRY) black $(SRC)
$(POETRY) black $(TESTS)
$(POETRY) isort --profile black $(SRC)
$(POETRY) isort --profile black $(TESTS)
lint:
$(POETRY) flake8 $(SRC)
$(POETRY) bandit --silent --recursive $(SRC)
@ -13,9 +25,12 @@ types:
test:
$(POETRY) pytest $(TESTS)
ci: lint types test
freeze:
poetry export --without dev --format=requirements.txt --output requirements.frozen.txt
check:
$(MAKE) fmt
$(MAKE) lint
$(MAKE) types
$(MAKE) bandit
$(MAKE) test

1
README.md
View file

@ -9,6 +9,7 @@ Current supported webhooks:
* PagerDuty
* AWS SNS
* Gitlab
* Prometheus Alertmanager
## Usage

57
ops_bot/alertmanager.py Normal file
View file

@ -0,0 +1,57 @@
import logging
from typing import Any, Dict, List, Tuple
from fastapi import Request
from ops_bot.common import COLOR_ALARM, COLOR_OK, COLOR_UNKNOWN
from ops_bot.config import RoutingKey
def prometheus_alert_to_markdown(
alert_data: Dict, # type: ignore[type-arg]
) -> List[Tuple[str, str]]:
"""
Converts a prometheus alert json to markdown
"""
messages = []
known_labels = ["alertname", "instance", "job"]
for alert in alert_data["alerts"]:
title = (
alert["annotations"]["description"]
if hasattr(alert["annotations"], "description")
else alert["annotations"]["summary"]
)
severity = alert.get("severity", "critical")
if alert["status"] == "resolved":
color = COLOR_OK
elif severity == "critical":
color = COLOR_ALARM
else:
color = COLOR_UNKNOWN
status = alert["status"]
generatorURL = alert.get("generatorURL")
plain = f"{status}: {title}"
header_str = f"{status.upper()}[{status}]"
formatted = f"<strong><font color={color}>{header_str}</font></strong>: [{title}]({generatorURL})"
for label_name in known_labels:
try:
plain += "\n* **{0}**: {1}".format(
label_name.capitalize(), alert["labels"][label_name]
)
formatted += "\n* **{0}**: {1}".format(
label_name.capitalize(), alert["labels"][label_name]
)
except Exception as e:
logging.error("error parsing labels", exc_info=e)
pass
messages.append((plain, formatted))
return messages
async def parse_alertmanager_event(
route: RoutingKey,
payload: Any,
request: Request,
) -> List[Tuple[str, str]]:
return prometheus_alert_to_markdown(payload)

9
ops_bot/cli.py
View file

@ -4,7 +4,7 @@ from typing import Any
import click
from ops_bot.config import RoutingKey, load_config, save_config
from ops_bot.config import RoutingKey, hook_types, load_config, save_config
@click.group()
@ -28,20 +28,23 @@ def cli(ctx: Any, config_file: str) -> None:
)
@click.option("--room-id", help="The room ID to send the messages to")
@click.pass_obj
def add_hook(config_file: str, name: str, hook_type: str, room_id: str) -> None:
def add_hook(config_file: str, name: str, maybe_hook_type: str, room_id: str) -> None:
settings = load_config(config_file)
path_key = secrets.token_urlsafe(30)
secret_token = secrets.token_urlsafe(30)
if name in set([key.name for key in settings.routing_keys]):
print("Error: A hook with that name already exists")
sys.exit(1)
if maybe_hook_type not in hook_types:
print(f"Invalid hook type {maybe_hook_type}")
print("Must be one of ", ", ".join(hook_types))
settings.routing_keys.append(
RoutingKey(
name=name,
path_key=path_key,
secret_token=secret_token,
room_id=room_id,
hook_type=hook_type,
hook_type=maybe_hook_type, # type: ignore[arg-type]
)
)
save_config(settings)

24
ops_bot/config.py
View file

@ -1,30 +1,37 @@
import json
import logging
import os
import typing
from pathlib import Path
from typing import List, Literal
from pydantic import BaseSettings
from pydantic_settings import BaseSettings, SettingsConfigDict
from ops_bot.matrix import MatrixClientSettings
env_path = os.getenv("BOT_ENV_FILE")
HookType = Literal["gitlab", "pagerduty", "aws-sns", "alertmanager"]
hook_types = typing.get_args(HookType)
class RoutingKey(BaseSettings):
name: str
path_key: str
secret_token: str
room_id: str
hook_type: Literal["gitlab", "pagerduty", "aws-sns"]
hook_type: HookType
class BotSettings(BaseSettings):
model_config = SettingsConfigDict(
env_prefix="BOT_", env_file=env_path, env_file_encoding="utf-8"
)
routing_keys: List[RoutingKey]
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
matrix: MatrixClientSettings
class Config:
env_prefix = "BOT_"
case_sensitive = False
def get_rooms(self) -> List[str]:
return list(set([key.room_id for key in self.routing_keys]))
@ -35,9 +42,10 @@ def config_file_exists(filename: str) -> bool:
def load_config(filename: str = "config.json") -> BotSettings:
if config_file_exists(filename):
bot_settings = BotSettings.parse_file(filename)
with open(filename, "r") as f:
bot_settings = BotSettings.model_validate_json(f.read())
else:
bot_settings = BotSettings(_env_file=".env", _env_file_encoding="utf-8")
bot_settings = BotSettings() # type: ignore[call-arg]
logging.getLogger().setLevel(bot_settings.log_level)
return bot_settings

8
ops_bot/gitlab/hook.py
View file

@ -59,9 +59,9 @@ async def handle_event(x_gitlab_event: str, payload: Any) -> List[Tuple[str, str
**attr.asdict(subevt, recurse=False),
**{key: getattr(subevt, key) for key in subevt.event_properties},
"abort": abort,
**base_args, # type: ignore
**base_args,
}
args["templates"] = templates.proxy(args) # type: ignore
args["templates"] = templates.proxy(args)
html = tpl.render(**args)
if not html or aborted:
@ -87,4 +87,6 @@ async def parse_event(
route: RoutingKey, payload: Any, request: Request
) -> List[Tuple[str, str]]:
x_gitlab_event = request.headers.get("x-gitlab-event")
return await handle_event(x_gitlab_event, payload)
if x_gitlab_event:
return await handle_event(x_gitlab_event, payload)
return []

36
ops_bot/main.py
View file

@ -1,10 +1,12 @@
import asyncio
import logging
import os
from typing import Any, Dict, List, Optional, Protocol, Tuple, cast
import sys
from contextlib import asynccontextmanager
from typing import Any, AsyncIterator, Dict, List, Optional, Protocol, Tuple, cast
import json_logging
import uvicorn
from dotenv import load_dotenv
from fastapi import Depends, FastAPI, HTTPException, Request, status
from fastapi.security import (
HTTPAuthorizationCredentials,
@ -13,18 +15,11 @@ from fastapi.security import (
HTTPBearer,
)
from ops_bot import aws, pagerduty
from ops_bot import alertmanager, aws, pagerduty
from ops_bot.config import BotSettings, RoutingKey, load_config
from ops_bot.gitlab import hook as gitlab_hook
from ops_bot.matrix import MatrixClient
load_dotenv()
app = FastAPI()
bearer_security = HTTPBearer(auto_error=False)
basic_security = HTTPBasic(auto_error=False)
async def get_matrix_service(request: Request) -> MatrixClient:
"""A helper to fetch the matrix client from the app state"""
@ -37,21 +32,29 @@ async def matrix_main(matrix_client: MatrixClient) -> None:
await asyncio.gather(*workers)
@app.on_event("startup")
async def startup_event() -> None:
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
config_fname = os.environ.get("BOT_CONFIG_FILE", "config.yaml")
bot_settings = load_config(config_fname)
c = MatrixClient(settings=bot_settings.matrix, join_rooms=bot_settings.get_rooms())
app.state.matrix_client = c
app.state.bot_settings = bot_settings
asyncio.create_task(matrix_main(c))
@app.on_event("shutdown")
async def shutdown_event() -> None:
yield
await app.state.matrix_client.shutdown()
app = FastAPI(lifespan=lifespan)
bearer_security = HTTPBearer(auto_error=False)
basic_security = HTTPBasic(auto_error=False)
log = logging.getLogger("ops_bot")
log.addHandler(logging.StreamHandler(sys.stdout))
json_logging.init_fastapi(enable_json=True)
json_logging.init_request_instrument(app)
@app.get("/")
async def root() -> Dict[str, str]:
return {"message": "Hello World"}
@ -113,6 +116,7 @@ handlers: Dict[str, Tuple[Authorizer, ParseHandler]] = {
"gitlab": (gitlab_hook.authorize, gitlab_hook.parse_event),
"pagerduty": (bearer_token_authorizer, pagerduty.parse_pagerduty_event),
"aws-sns": (nop_authorizer, aws.parse_sns_event),
"alertmanager": (nop_authorizer, alertmanager.parse_alertmanager_event),
}

20
ops_bot/matrix.py
View file

@ -1,12 +1,14 @@
import json
import logging
import os
import pathlib
import sys
from typing import List, Optional, Protocol
from markdown import markdown
from nio import AsyncClient, AsyncClientConfig, LoginResponse
from pydantic import BaseModel, BaseSettings
from pydantic import BaseModel
from pydantic_settings import BaseSettings
class ClientCredentials(BaseModel):
@ -72,6 +74,9 @@ class MatrixClient:
self.greeting_sent = False
if self.store_path and not os.path.isdir(self.store_path):
os.mkdir(self.store_path)
async def start(self) -> None:
await self.login()
@ -95,9 +100,9 @@ class MatrixClient:
async def login_fresh(self) -> None:
self.client = AsyncClient(
self.settings.homeserver,
self.settings.user_id,
self.settings.store_path,
homeserver=self.settings.homeserver,
user=self.settings.user_id,
store_path=str(self.settings.store_path),
config=self.client_config,
ssl=self.settings.verify_ssl,
)
@ -117,11 +122,12 @@ class MatrixClient:
async def login_with_credentials(self) -> None:
credentials = self.credential_store.read()
self.client = AsyncClient(
credentials.homeserver,
credentials.user_id,
homeserver=credentials.homeserver,
user=credentials.user_id,
device_id=credentials.device_id,
store_path=self.store_path,
store_path=str(self.store_path),
config=self.client_config,
ssl=True,
)

2499
poetry.lock generated
View file

File diff suppressed because it is too large Load diff

19
pyproject.toml
View file

@ -5,23 +5,26 @@ description = ""
authors = ["Abel Luck <abel@guardianproject.info>"]
[tool.poetry.dependencies]
python = "^3.10"
matrix-nio = {extras = ["e2e"], version = "^0.19.0"}
fastapi = "^0.79.0"
uvicorn = "^0.18.2"
python = "^3.11"
matrix-nio = {extras = ["e2e"], version = "^0.22.1"}
fastapi = "^0.104.1"
uvicorn = "^0.24.0"
termcolor = "^1.1.0"
Markdown = "^3.4.1"
pydantic = {extras = ["dotenv"], version = "^1.9.1"}
commonmark = "^0.9.1"
Jinja2 = "^3.1.2"
mautrix = "^0.18.8"
mautrix = "^0.20.2"
click = "^8.1.3"
json-logging = "^1.3.0"
pydantic-settings = "^2.0.3"
prometheus-client = "^0.18.0"
prometheus-fastapi-instrumentator = "^6.1.0"
[tool.poetry.dev-dependencies]
pytest = "^7.2.0"
black = "^22.10.0"
isort = "^5.10.1"
mypy = "^0.991"
mypy = "^1.2.0"
bandit = "^1.7.4"
flake8 = "^6.0.0"
flake8-black = "^0.3.5"
@ -68,4 +71,4 @@ warn_unreachable = true
show_error_codes = true
# Explicit is better than implicit
no_implicit_optional = true
no_implicit_optional = true

33
shell.nix Normal file
View file

@ -0,0 +1,33 @@
{ system ? "x86_64-linux", pkgs ? import <nixpkgs> { inherit system; } }:
let
packages = [
pkgs.python311
pkgs.poetry
pkgs.zsh
];
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
pkgs.stdenv.cc.cc
# Add any missing library needed
# You can use the nix-index package to locate them, e.g. nix-locate -w --top-level --at-root /lib/libudev.so.1
];
# Put the venv on the repo, so direnv can access it
POETRY_VIRTUALENVS_IN_PROJECT = "true";
POETRY_VIRTUALENVS_PATH = "{project-dir}/.venv";
# Use python from path, so you can use a different version to the one bundled with poetry
POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON = "true";
in
pkgs.mkShell {
buildInputs = packages;
shellHook = ''
export SHELL=${pkgs.zsh}
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}"
export POETRY_VIRTUALENVS_IN_PROJECT="${POETRY_VIRTUALENVS_IN_PROJECT}"
export POETRY_VIRTUALENVS_PATH="${POETRY_VIRTUALENVS_PATH}"
export POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON="${POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON}"
export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
'';
}

57
tests/test_alertmanager.py Normal file
View file

@ -0,0 +1,57 @@
from ops_bot.alertmanager import prometheus_alert_to_markdown
import json
payload = json.loads(
"""
{
"receiver": "matrix",
"status": "firing",
"alerts": [
{
"status": "firing",
"labels": {
"alertname": "InstanceDown",
"environment": "monitoring.example.com",
"instance": "webserver.example.com",
"job": "node_exporter",
"severity": "critical"
},
"annotations": {
"description": "webserver.example.com of job node_exporter has been down for more than 5 minutes.",
"summary": "THIS IS A TEST Instance webserver.example.com down"
},
"startsAt": "2022-06-23T11:53:14.318Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://monitoring.example.com:9090",
"fingerprint": "9cd7837114d58797"
}
],
"groupLabels": {
"alertname": "InstanceDown"
},
"commonLabels": {
"alertname": "InstanceDown",
"environment": "monitoring.example.com",
"instance": "webserver.example.com",
"job": "node_exporter",
"severity": "critical"
},
"commonAnnotations": {
"description": "webserver.example.com of job node_exporter has been down for more than 5 minutes.",
"summary": "Instance webserver.example.com down"
},
"externalURL": "https://alert.example",
"version": "4",
"groupKey": "",
"truncatedAlerts": 0
}"""
)
def test_alertmanager():
r = prometheus_alert_to_markdown(payload)
assert len(r) == 1
plain, formatted = r[0]
assert "firing" in plain and "Instance webserver.example.com down" in plain
assert "firing" in formatted and "Instance webserver.example.com down" in formatted

18
tests/test_gitlab.py
View file

@ -1,4 +1,5 @@
import json
from ops_bot.gitlab import hook
from ops_bot.util.template import TemplateUtil
@ -148,16 +149,27 @@ issue_open_payload_raw = """
}
}"""
issue_open_payload = json.loads(issue_open_payload_raw)
def test_templates():
# print(gitlab.messages._loader.list_templates())
tpl = hook.messages["test"]
args = issue_open_payload | {"util": TemplateUtil}
args["templates"] = hook.templates.proxy(args)
assert tpl.render(**args) == "<strong data-mautrix-exclude-plaintext>[<a data-mautrix-exclude-plaintext href=\"http://example.com/gitlabhq/gitlab-test\">gitlabhq/gitlab-test</a>]</strong> <a data-mautrix-exclude-plaintext href=\"\">root</a>"
assert (
tpl.render(**args)
== '<strong data-mautrix-exclude-plaintext>[<a data-mautrix-exclude-plaintext href="http://example.com/gitlabhq/gitlab-test">gitlabhq/gitlab-test</a>]</strong> <a data-mautrix-exclude-plaintext href="">root</a>'
)
async def test_hook():
r = await hook.handle_event("Issue Hook", issue_open_payload)
assert r[0]
assert r[0][0] == "[gitlabhq/gitlab-test] root opened [issue #23](http://example.com/diaspora/issues/23): New API: create/update/delete file\n \n> Create new API for manipulations with repository\nAPI"
assert r[0][1] == "<strong data-mautrix-exclude-plaintext>[<a data-mautrix-exclude-plaintext href=\"http://example.com/gitlabhq/gitlab-test\">gitlabhq/gitlab-test</a>]</strong> <a data-mautrix-exclude-plaintext href=\"http://example.com/root\">root</a>\n opened <a href=\"http://example.com/diaspora/issues/23\" >issue #23</a>: New API: create/update/delete file<br/>\n <blockquote><p>Create new API for manipulations with repository</p>\n</blockquote>\n <span data-mx-color=\"#000000\"\n data-mx-bg-color=\"#ffffff\"\n title=\"API related issues\"\n >&nbsp;API&nbsp;</span>"
assert (
r[0][0]
== "[gitlabhq/gitlab-test] root opened [issue #23](http://example.com/diaspora/issues/23): New API: create/update/delete file\n \n> Create new API for manipulations with repository\nAPI"
)
assert (
r[0][1]
== '<strong data-mautrix-exclude-plaintext>[<a data-mautrix-exclude-plaintext href="http://example.com/gitlabhq/gitlab-test">gitlabhq/gitlab-test</a>]</strong> <a data-mautrix-exclude-plaintext href="http://example.com/root">root</a>\n opened <a href="http://example.com/diaspora/issues/23" >issue #23</a>: New API: create/update/delete file<br/>\n <blockquote><p>Create new API for manipulations with repository</p>\n</blockquote>\n <span data-mx-color="#000000"\n data-mx-bg-color="#ffffff"\n title="API related issues"\n >&nbsp;API&nbsp;</span>'
)

17
tests/test_ops_bot.py
View file

@ -1,11 +1,12 @@
import json
from ops_bot import __version__
from ops_bot import aws
from ops_bot import __version__, aws
def test_version() -> None:
assert __version__ == "0.1.0"
sns_subscribtion_unsubscribe = """{
"Type" : "UnsubscribeConfirmation",
"MessageId" : "47138184-6831-46b8-8f7c-afc488602d7d",
@ -44,19 +45,25 @@ sns_notification = """{
"UnsubscribeURL" : "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:123456789012:MyTopic:c9135db0-26c4-47ec-8998-413945fb5a96"
}"""
async def test_aws_sns_notification() -> None:
r = await aws.parse_sns_event(None, json.loads(sns_notification), None)
assert r[0][0] == "My First Message\nHello world!"
assert r[0][1] == "<strong><font color=#dc3545>My First Message</font></strong>\n<p>Hello world!</p>"
assert (
r[0][1]
== "<strong><font color=#dc3545>My First Message</font></strong>\n<p>Hello world!</p>"
)
async def test_aws_sns_subscribe() -> None:
r = await aws.parse_sns_event(None, json.loads(sns_subscribtion_confirm), None)
print(r)
expected = 'You have chosen to subscribe to the topic arn:aws:sns:us-west-2:123456789012:MyTopic.\nTo confirm the subscription, visit the SubscribeURL included in this message.\n\nhttps://sns.us-west-2.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-west-2:123456789012:MyTopic&Token=2336412f37...'
expected = "You have chosen to subscribe to the topic arn:aws:sns:us-west-2:123456789012:MyTopic.\nTo confirm the subscription, visit the SubscribeURL included in this message.\n\nhttps://sns.us-west-2.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-west-2:123456789012:MyTopic&Token=2336412f37..."
assert r[0] == (expected, expected)
async def test_aws_sns_unsubscribe() -> None:
r = await aws.parse_sns_event(None, json.loads(sns_subscribtion_unsubscribe), None)
print(r)
expected = 'You have chosen to deactivate subscription arn:aws:sns:us-west-2:123456789012:MyTopic:2bcfbf39-05c3-41de-beaa-fcfcc21c8f55.\nTo cancel this operation and restore the subscription, visit the SubscribeURL included in this message.\n\nhttps://sns.us-west-2.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-west-2:123456789012:MyTopic&Token=2336412f37fb6...'
expected = "You have chosen to deactivate subscription arn:aws:sns:us-west-2:123456789012:MyTopic:2bcfbf39-05c3-41de-beaa-fcfcc21c8f55.\nTo cancel this operation and restore the subscription, visit the SubscribeURL included in this message.\n\nhttps://sns.us-west-2.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-west-2:123456789012:MyTopic&Token=2336412f37fb6..."
assert r[0] == (expected, expected)