anynews/republisher
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement tab-scoped runs pagination state

Browse source
This commit is contained in:
Abel Luck 2026-03-31 12:12:36 +02:00
parent dce67ea9e3
commit c834c3c254
6 changed files with 444 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

145
repub/web.py
View file

@ -3,6 +3,8 @@ from __future__ import annotations
import asyncio
import hashlib
from collections.abc import AsyncGenerator, Awaitable, Callable
from contextlib import suppress
from datetime import timedelta
from pathlib import Path
from typing import TypedDict, cast
from urllib.parse import urlparse
@ -13,16 +15,9 @@ from datastar_py.quart import DatastarResponse, read_signals
from datastar_py.sse import DatastarEvent
from htpy import Renderable
from peewee import IntegrityError
from quart import (
Quart,
Response,
has_request_context,
request,
send_from_directory,
url_for,
)
from quart import Quart, Response, request, send_from_directory, url_for
from repub.datastar import RefreshBroker, render_stream
from repub.datastar import RefreshBroker, TabStateStore, render_stream
from repub.jobs import (
COMPLETED_EXECUTION_PAGE_SIZE,
JobRuntime,
@ -58,10 +53,14 @@ from repub.pages.sources import PANGEA_CONTENT_FORMATS, PANGEA_CONTENT_TYPES
REFRESH_BROKER_KEY = "repub.refresh_broker"
JOB_RUNTIME_KEY = "repub.job_runtime"
TAB_STATE_STORE_KEY = "repub.tab_state_store"
TAB_STATE_CLEANER_TASK_KEY = "repub.tab_state_cleaner_task"
DEFAULT_LOG_DIR = Path("out/logs")
DEFAULT_FEEDS_DIR = Path("out/feeds")
RUNS_TAB_STATE_KEY = "runs"
TAB_STATE_CLEAN_INTERVAL = timedelta(seconds=10)
RenderFunction = Callable[[], Awaitable[Renderable]]
PatchRenderFunction = Callable[[str | None], Awaitable[Renderable]]
class SourceFormData(TypedDict):
@ -144,6 +143,8 @@ def create_app(*, dev_mode: bool = False) -> Quart:
app.config["REPUB_DEV_MODE"] = dev_mode
app.extensions[REFRESH_BROKER_KEY] = RefreshBroker()
app.extensions[JOB_RUNTIME_KEY] = None
app.extensions[TAB_STATE_STORE_KEY] = TabStateStore()
app.extensions[TAB_STATE_CLEANER_TASK_KEY] = None
@app.get("/feeds/<path:feed_path>")
async def published_feed(feed_path: str) -> Response:
@ -202,23 +203,27 @@ def create_app(*, dev_mode: bool = False) -> Quart:
@app.post("/")
async def dashboard_patch() -> DatastarResponse:
return _page_patch_response(app, lambda: render_dashboard(app))
return await _page_patch_response(app, lambda _tab_id: render_dashboard(app))
@app.post("/sources")
async def sources_patch() -> DatastarResponse:
return _page_patch_response(app, lambda: render_sources(app))
return await _page_patch_response(app, lambda _tab_id: render_sources(app))
@app.post("/sources/create")
async def create_source_patch() -> DatastarResponse:
return _page_patch_response(app, lambda: render_create_source(app))
return await _page_patch_response(
app, lambda _tab_id: render_create_source(app)
)
@app.post("/sources/<string:slug>/edit")
async def edit_source_patch(slug: str) -> DatastarResponse:
return _page_patch_response(app, lambda: render_edit_source(slug, app))
return await _page_patch_response(
app, lambda _tab_id: render_edit_source(slug, app)
)
@app.post("/settings")
async def settings_patch() -> DatastarResponse:
return _page_patch_response(app, lambda: render_settings(app))
return await _page_patch_response(app, lambda _tab_id: render_settings(app))
@app.post("/actions/sources/create")
async def create_source_action() -> DatastarResponse:
@ -293,7 +298,24 @@ def create_app(*, dev_mode: bool = False) -> Quart:
@app.post("/runs")
async def runs_patch() -> DatastarResponse:
return _page_patch_response(app, lambda: render_runs(app))
return await _page_patch_response(
app,
lambda tab_id: render_runs(app, tab_id=tab_id),
)
@app.post("/actions/runs/completed-page/<int:page>")
async def set_completed_runs_page_action(page: int) -> Response:
signals = await _read_optional_signals()
tab_id = _read_tab_id(signals)
if tab_id is None:
return Response(status=400)
get_tab_state_store(app).update_page_state(
tab_id,
RUNS_TAB_STATE_KEY,
lambda state: {**state, "completed_page": max(1, page)},
)
trigger_refresh(app, tab_id=tab_id)
return Response(status=204)
@app.post("/actions/jobs/<int:job_id>/run-now")
async def run_job_now_action(job_id: int) -> Response:
@ -351,14 +373,24 @@ def create_app(*, dev_mode: bool = False) -> Quart:
app, job_id=job_id, execution_id=execution_id
)
return _page_patch_response(app, render)
return await _page_patch_response(app, lambda _tab_id: render())
@app.before_serving
async def start_runtime() -> None:
get_job_runtime(app).start()
app.extensions[TAB_STATE_CLEANER_TASK_KEY] = asyncio.create_task(
_clean_tab_state_periodically(app)
)
@app.after_serving
async def stop_runtime() -> None:
cleaner = cast(
asyncio.Task[None] | None, app.extensions.get(TAB_STATE_CLEANER_TASK_KEY)
)
if cleaner is not None:
cleaner.cancel()
with suppress(asyncio.CancelledError):
await cleaner
get_job_runtime(app).shutdown()
return app
@ -368,6 +400,10 @@ def get_refresh_broker(app: Quart) -> RefreshBroker:
return cast(RefreshBroker, app.extensions[REFRESH_BROKER_KEY])
def get_tab_state_store(app: Quart) -> TabStateStore:
return cast(TabStateStore, app.extensions[TAB_STATE_STORE_KEY])
def get_job_runtime(app: Quart) -> JobRuntime:
runtime = cast(JobRuntime | None, app.extensions.get(JOB_RUNTIME_KEY))
if runtime is None:
@ -379,8 +415,10 @@ def get_job_runtime(app: Quart) -> JobRuntime:
return runtime
def trigger_refresh(app: Quart, event: object = "refresh-event") -> None:
get_refresh_broker(app).publish(event)
def trigger_refresh(
app: Quart, event: object = "refresh-event", *, tab_id: str | None = None
) -> None:
get_refresh_broker(app).publish(event, tab_id=tab_id)
async def render_dashboard(app: Quart | None = None) -> Renderable:
@ -431,18 +469,17 @@ async def render_edit_source(slug: str, app: Quart | None = None) -> Renderable:
)
async def render_runs(app: Quart | None = None) -> Renderable:
async def render_runs(
app: Quart | None = None, *, tab_id: str | None = None
) -> Renderable:
if app is None:
return runs_page()
completed_page = (
max(1, request.args.get("completed_page", 1, type=int) or 1)
if has_request_context()
else 1
)
tab_state = get_tab_state_store(app).get_page_state(tab_id, RUNS_TAB_STATE_KEY)
resolved_completed_page = max(1, _read_int(tab_state.get("completed_page"), 1))
view = load_runs_view(
log_dir=app.config["REPUB_LOG_DIR"],
completed_page=completed_page,
completed_page=resolved_completed_page,
completed_page_size=COMPLETED_EXECUTION_PAGE_SIZE,
)
return runs_page(
@ -494,24 +531,36 @@ async def render_execution_logs(
)
def _page_patch_response(app: Quart, render: RenderFunction) -> DatastarResponse:
queue = get_refresh_broker(app).subscribe()
async def _page_patch_response(
app: Quart, render: PatchRenderFunction
) -> DatastarResponse:
signals = await _read_optional_signals()
tab_id = _read_tab_id(signals)
if tab_id is not None:
get_tab_state_store(app).connect(tab_id)
queue = get_refresh_broker(app).subscribe(tab_id=tab_id)
stream = render_stream(
queue,
render=render,
render=lambda: render(tab_id),
last_event_id=request.headers.get("last-event-id"),
)
return DatastarResponse(_unsubscribe_on_close(queue, stream, app))
return DatastarResponse(_unsubscribe_on_close(queue, stream, app, tab_id=tab_id))
async def _unsubscribe_on_close(
queue: object, stream: AsyncGenerator[DatastarEvent, None], app: Quart
queue: object,
stream: AsyncGenerator[DatastarEvent, None],
app: Quart,
*,
tab_id: str | None,
) -> AsyncGenerator[DatastarEvent, None]:
try:
async for event in stream:
yield event
finally:
get_refresh_broker(app).unsubscribe(cast(asyncio.Queue[object], queue))
if tab_id is not None:
get_tab_state_store(app).disconnect(tab_id)
def _load_sidebar_counts(app: Quart) -> dict[str, int]:
@ -523,6 +572,29 @@ def _load_sidebar_counts(app: Quart) -> dict[str, int]:
}
async def _clean_tab_state_periodically(app: Quart) -> None:
while True:
await asyncio.sleep(TAB_STATE_CLEAN_INTERVAL.total_seconds())
get_tab_state_store(app).cleanup_stale()
async def _read_optional_signals() -> dict[str, object] | None:
content_type = request.headers.get("Content-Type", "")
if request.content_length in (None, 0) and "application/json" not in content_type:
return None
try:
return cast(dict[str, object] | None, await read_signals())
except Exception:
return None
def _read_tab_id(signals: dict[str, object] | None) -> str | None:
if signals is None:
return None
tab_id = _read_string(signals, "tabid")
return tab_id or None
def validate_source_form(
signals: dict[str, object] | None,
*,
@ -669,6 +741,17 @@ def _parse_int(value: str) -> int | None:
return None
def _read_int(value: object, default: int) -> int:
if isinstance(value, bool):
return int(value)
if isinstance(value, int):
return value
if isinstance(value, str):
parsed = _parse_int(value)
return default if parsed is None else parsed
return default
def _is_valid_url(value: str) -> bool:
parsed = urlparse(value)
return parsed.scheme in {"http", "https"} and parsed.netloc != ""