anynews/republisher
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

implement job runner and scheduler

Browse source
This commit is contained in:
Abel Luck 2026-03-30 14:02:39 +02:00
parent 328a70ff9b
commit 2b2a3f1cc0
11 changed files with 1572 additions and 284 deletions
Download patch file Download diff file Expand all files Collapse all files

131
repub/web.py
View file

@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import hashlib
from collections.abc import AsyncGenerator, Awaitable, Callable
from pathlib import Path
from typing import TypedDict, cast
from urllib.parse import urlparse
@ -15,8 +16,16 @@ from peewee import IntegrityError
from quart import Quart, Response, request, url_for
from repub.datastar import RefreshBroker, render_stream
from repub.jobs import (
JobRuntime,
load_dashboard_view,
load_execution_log_view,
load_runs_view,
)
from repub.model import (
Job,
create_source,
delete_job_source,
initialize_database,
load_source_form,
load_sources,
@ -25,7 +34,7 @@ from repub.model import (
)
from repub.pages import (
create_source_page,
dashboard_page,
dashboard_page_with_data,
edit_source_page,
execution_logs_page,
runs_page,
@ -35,6 +44,8 @@ from repub.pages import (
from repub.pages.sources import PANGEA_CONTENT_FORMATS, PANGEA_CONTENT_TYPES
REFRESH_BROKER_KEY = "repub.refresh_broker"
JOB_RUNTIME_KEY = "repub.job_runtime"
DEFAULT_LOG_DIR = Path("out/logs")
RenderFunction = Callable[[], Awaitable[Renderable]]
@ -83,7 +94,12 @@ def _render_shim_page(*, stylesheet_href: str, datastar_src: str) -> tuple[str,
def create_app() -> Quart:
app = Quart(__name__)
app.config["REPUB_DB_PATH"] = str(initialize_database())
app.config.setdefault("REPUB_LOG_DIR", DEFAULT_LOG_DIR)
app.config.setdefault("REPUB_JOB_WORKER_DURATION_SECONDS", 20.0)
app.config.setdefault("REPUB_JOB_WORKER_STATS_INTERVAL_SECONDS", 1.0)
app.config.setdefault("REPUB_JOB_WORKER_FAILURE_PROBABILITY", 0.3)
app.extensions[REFRESH_BROKER_KEY] = RefreshBroker()
app.extensions[JOB_RUNTIME_KEY] = None
@app.get("/")
@app.get("/sources")
@ -112,7 +128,7 @@ def create_app() -> Quart:
@app.post("/")
async def dashboard_patch() -> DatastarResponse:
return _page_patch_response(app, render_dashboard)
return _page_patch_response(app, lambda: render_dashboard(app))
@app.post("/sources")
async def sources_patch() -> DatastarResponse:
@ -147,6 +163,7 @@ def create_app() -> Quart:
{"_formError": "Slug must be unique.", "_formSuccess": ""}
)
)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return DatastarResponse(SSE.redirect("/sources"))
@ -171,20 +188,58 @@ def create_app() -> Quart:
{"_formError": "Source does not exist.", "_formSuccess": ""}
)
)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return DatastarResponse(SSE.redirect("/sources"))
@app.post("/runs")
async def runs_patch() -> DatastarResponse:
return _page_patch_response(app, render_runs)
return _page_patch_response(app, lambda: render_runs(app))
@app.post("/actions/jobs/<int:job_id>/run-now")
async def run_job_now_action(job_id: int) -> Response:
get_job_runtime(app).run_job_now(job_id, reason="manual")
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/jobs/<int:job_id>/toggle-enabled")
async def toggle_job_enabled_action(job_id: int) -> Response:
job = Job.get_or_none(id=job_id)
if job is not None:
get_job_runtime(app).set_job_enabled(job_id, enabled=not job.enabled)
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/jobs/<int:job_id>/delete")
async def delete_job_action(job_id: int) -> Response:
delete_job_source(job_id)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/executions/<int:execution_id>/cancel")
async def cancel_execution_action(execution_id: int) -> Response:
get_job_runtime(app).request_execution_cancel(execution_id)
trigger_refresh(app)
return Response(status=204)
@app.post("/job/<int:job_id>/execution/<int:execution_id>/logs")
async def logs_patch(job_id: int, execution_id: int) -> DatastarResponse:
async def render() -> Renderable:
return await render_execution_logs(job_id=job_id, execution_id=execution_id)
return await render_execution_logs(
app, job_id=job_id, execution_id=execution_id
)
return _page_patch_response(app, render)
@app.before_serving
async def start_runtime() -> None:
get_job_runtime(app).start()
@app.after_serving
async def stop_runtime() -> None:
get_job_runtime(app).shutdown()
return app
@ -192,12 +247,39 @@ def get_refresh_broker(app: Quart) -> RefreshBroker:
return cast(RefreshBroker, app.extensions[REFRESH_BROKER_KEY])
def get_job_runtime(app: Quart) -> JobRuntime:
runtime = cast(JobRuntime | None, app.extensions.get(JOB_RUNTIME_KEY))
if runtime is None:
runtime = JobRuntime(
log_dir=app.config["REPUB_LOG_DIR"],
worker_duration_seconds=float(
app.config["REPUB_JOB_WORKER_DURATION_SECONDS"]
),
worker_stats_interval_seconds=float(
app.config["REPUB_JOB_WORKER_STATS_INTERVAL_SECONDS"]
),
worker_failure_probability=float(
app.config["REPUB_JOB_WORKER_FAILURE_PROBABILITY"]
),
refresh_callback=lambda: trigger_refresh(app),
)
app.extensions[JOB_RUNTIME_KEY] = runtime
return runtime
def trigger_refresh(app: Quart, event: object = "refresh-event") -> None:
get_refresh_broker(app).publish(event)
async def render_dashboard() -> Renderable:
return dashboard_page()
async def render_dashboard(app: Quart | None = None) -> Renderable:
if app is None:
return dashboard_page_with_data()
view = load_dashboard_view(log_dir=app.config["REPUB_LOG_DIR"])
return dashboard_page_with_data(
snapshot=cast(dict[str, str], view["snapshot"]),
running_executions=cast(tuple[dict[str, object], ...], view["running"]),
)
async def render_sources(app: Quart | None = None) -> Renderable:
@ -221,12 +303,41 @@ async def render_edit_source(slug: str) -> Renderable:
)
async def render_runs() -> Renderable:
return runs_page()
async def render_runs(app: Quart | None = None) -> Renderable:
if app is None:
return runs_page()
view = load_runs_view(log_dir=app.config["REPUB_LOG_DIR"])
return runs_page(
running_executions=cast(tuple[dict[str, object], ...], view["running"]),
upcoming_jobs=cast(tuple[dict[str, object], ...], view["upcoming"]),
completed_executions=cast(tuple[dict[str, object], ...], view["completed"]),
)
async def render_execution_logs(*, job_id: int, execution_id: int) -> Renderable:
return execution_logs_page(job_id=job_id, execution_id=execution_id)
async def render_execution_logs(
app: Quart | None = None, *, job_id: int, execution_id: int
) -> Renderable:
if app is None:
return execution_logs_page(job_id=job_id, execution_id=execution_id)
log_view = load_execution_log_view(
log_dir=app.config["REPUB_LOG_DIR"],
job_id=job_id,
execution_id=execution_id,
)
return execution_logs_page(
job_id=job_id,
execution_id=execution_id,
log_view={
"title": log_view.title,
"description": log_view.description,
"status_label": log_view.status_label,
"status_tone": log_view.status_tone,
"log_text": log_view.log_text,
"error_message": log_view.error_message,
},
)
def _page_patch_response(app: Quart, render: RenderFunction) -> DatastarResponse: