Move all the work to the worker task and add json logging

tailscalesd/main.py

@@ -1,28 +1,23 @@
 import asyncio
 import logging
 import os
+import sys
 from ipaddress import ip_address
 from typing import Dict, List
 
 import httpx
 import uvicorn
+
+import json_logging
 from fastapi import FastAPI
-from pydantic import SecretStr, Field
+from pydantic import Field, SecretStr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 env_path = os.getenv("TAILSCALESD_ENV_FILE")
 debug = os.getenv("TAILSCALESD_DEBUG", False)
-
-handler = logging.StreamHandler()
-handler.setFormatter(
-    logging.Formatter(style="{", fmt="[{name}:{filename}] {levelname} - {message}")
-)
-
 log = logging.getLogger("tailscalesd")
 log.setLevel(logging.DEBUG if debug else logging.INFO)
-log.addHandler(handler)
-
-CACHE_DEVICES = []
+log.addHandler(logging.StreamHandler(sys.stdout))
 
 
 def filter_ipv6(addresses):
@@ -43,25 +38,27 @@ class Settings(BaseSettings):
 settings = Settings()  # type: ignore[call-arg]
 app = FastAPI()
 
+json_logging.init_fastapi(enable_json=True)
+json_logging.init_request_instrument(app)
 
-async def tailscale_poll():
-    global CACHE_DEVICES
-    log.debug("Starting polling")
-    while True:
-        async with httpx.AsyncClient() as client:
+CACHE_SD = []
+
+
+async def tailscale_devices() -> List:
+    async with httpx.AsyncClient() as client:
+        try:
             # https://github.com/tailscale/tailscale/blob/main/api.md#tailnet-devices-get
             r = await client.get(
                 f"https://api.tailscale.com/api/v2/tailnet/{settings.tailnet}/devices",
                 auth=(settings.api_key.get_secret_value(), ""),
             )
-            CACHE_DEVICES = r.json()["devices"]
-
-        await asyncio.sleep(settings.interval)
-
-
-@app.on_event("startup")
-async def start_tailscale_poll():
-    asyncio.create_task(tailscale_poll())
+            return r.json()["devices"]
+        except Exception as e:
+            log.error(
+                f"Polling tailscale devices failed!",
+                exc_info=e,
+            )
+            return []
 
 
 def group_by_type(input_list):
@@ -130,8 +127,10 @@ async def matrix_sd(devices) -> List:
         try:
             workers = await matrix_node_sd(device)
         except Exception as e:
-            log.error(f"Failed parsing matrix node sd for device={device['hostname']}")
-            log.error(e)
+            log.error(
+                f"Failed parsing matrix node sd for device={device['hostname']}",
+                exc_info=e,
+            )
             workers = []
         targets = matrix_workers_to_sd(device, workers)
         if targets:
@@ -151,11 +150,30 @@ def plain_devices_sd(devices) -> List:
     return sd
 
 
+async def poll_sd():
+    global CACHE_SD
+    while True:
+        try:
+            devices = await tailscale_devices()
+            device_targets = plain_devices_sd(devices)
+            matrix_targets = await matrix_sd(devices)
+            CACHE_SD = matrix_targets + device_targets
+            await asyncio.sleep(settings.interval)
+        except Exception as e:
+            log.error(
+                f"service discovery poller failed",
+                exc_info=e,
+            )
+
+
+@app.on_event("startup")
+async def start_sd():
+    asyncio.create_task(poll_sd())
+
+
 @app.get("/")
 async def sd():
-    device_targets = plain_devices_sd(CACHE_DEVICES)
-    matrix_targets = await matrix_sd(CACHE_DEVICES)
-    return matrix_targets + device_targets
+    return CACHE_SD
 
 
 def main():