anynews/republisher
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add publisher dashboard routes
All checks were successful
buildbot/nix-eval Build done.
Details
buildbot/nix-build Build done.
Details
buildbot/nix-effects Build done.
Details

Browse source
This commit is contained in:
Abel Luck 2026-06-02 10:18:59 +02:00
parent 96551c2788
commit e4a5246ab3
31 changed files with 1603 additions and 516 deletions
Download patch file Download diff file Expand all files Collapse all files

8
README.md
View file

@ -17,7 +17,7 @@ The Republisher currently accepts the following source input types:
## Usage
Sync dependencies and start the admin UI:
Sync dependencies and start the web UI:
```sh
uv sync --all-groups
@ -50,13 +50,17 @@ In trusted-header mode, nginx must overwrite the `X-Republisher-*` identity head
Once the UI is running:
1. Open `http://127.0.0.1:8080/`.
1. Open `http://127.0.0.1:8080/admin` for the admin UI.
2. Create a source. Feed sources take a feed URL. Pangea sources take a domain plus category configuration.
3. Open `Settings` and set `Feed URL` to the public origin that serves mirrored feeds, for example `https://mirror.example`.
4. Configure the job schedule and any spider arguments.
5. Use `Run now` to trigger an immediate crawl, or leave the job enabled for scheduled runs.
6. Watch running jobs and logs live from the Runs pages.
Publisher-facing status is available at `http://127.0.0.1:8080/publisher`.
Set `REPUBLISHER_READER_APP_URL` or pass `--reader-app-url` to add a publisher
dashboard link to the AnyNews reader application that consumes the mirrored feeds.
Operational notes:
- The default database path is `republisher.db`. Set `REPUBLISHER_DB_PATH` to use a different SQLite file.

65
repub/components.py
View file

@ -98,27 +98,27 @@ def admin_sidebar(
h.nav(class_="mt-8 space-y-2")[
nav_link(
label="Dashboard",
href="/",
active=current_path == "/",
href="/admin",
active=current_path == "/admin",
badge="Live",
),
nav_link(
label="Sources",
href="/sources",
active=current_path.startswith("/sources"),
href="/admin/sources",
active=current_path.startswith("/admin/sources"),
badge=str(source_count),
),
nav_link(
label="Runs",
href="/runs",
active=current_path.startswith("/runs")
or current_path.startswith("/job/"),
href="/admin/runs",
active=current_path.startswith("/admin/runs")
or current_path.startswith("/admin/job/"),
badge=str(running_count),
),
nav_link(
label="Settings",
href="/settings",
active=current_path.startswith("/settings"),
href="/admin/settings",
active=current_path.startswith("/admin/settings"),
badge="App",
),
],
@ -148,11 +148,18 @@ def header_secondary_link(*, href: str, label: str) -> Renderable:
)[label]
def muted_action_link(*, href: str, label: str) -> Renderable:
return h.a(
href=href,
class_=_button_classes(tone="muted", emphasis="soft"),
)[label]
def muted_action_link(
*, href: str, label: str, target: str | None = None, rel: str | None = None
) -> Renderable:
attributes = {
"href": href,
"class": _button_classes(tone="muted", emphasis="soft"),
}
if target is not None:
attributes["target"] = target
if rel is not None:
attributes["rel"] = rel
return h.a(attributes)[label]
def inline_link(*, href: str, label: str, tone: str = "default") -> Renderable:
@ -225,6 +232,15 @@ def app_shell(
]
def publisher_shell(*, current_path: str, content: Node) -> Renderable:
del current_path
return h.main(id="morph", class_="min-h-screen")[
h.div(class_="px-4 py-4 sm:px-4 lg:px-5 lg:py-4")[
h.div(class_="mx-auto max-w-7xl space-y-4")[content]
],
]
def page_shell(
*,
current_path: str,
@ -269,7 +285,7 @@ def section_card(*, content: Node) -> Renderable:
def table_section(
*,
eyebrow: str | None = None,
title: str,
title: str | None,
subtitle: str | None = None,
empty_message: str,
headers: tuple[str, ...],
@ -315,8 +331,16 @@ def table_section(
)[empty_message]
]
has_heading = (
eyebrow is not None
or title is not None
or subtitle is not None
or actions is not None
)
return h.section[
h.div(
has_heading
and h.div(
class_="flex flex-col gap-2.5 sm:flex-row sm:items-end sm:justify-between"
)[
h.div[
@ -324,13 +348,18 @@ def table_section(
and h.p(
class_="text-xs font-semibold uppercase tracking-[0.22em] text-amber-600"
)[eyebrow],
h.h2(class_="mt-1 text-xl font-semibold text-slate-950")[title],
title
and h.h2(class_="mt-1 text-xl font-semibold text-slate-950")[title],
subtitle and h.p(class_="mt-1 text-sm text-slate-600")[subtitle],
],
actions,
],
h.div(
class_="mt-3 overflow-hidden rounded-2xl bg-white shadow-sm ring-1 ring-slate-200"
class_=(
"overflow-hidden rounded-2xl bg-white shadow-sm ring-1 ring-slate-200"
if not has_heading
else "mt-3 overflow-hidden rounded-2xl bg-white shadow-sm ring-1 ring-slate-200"
)
)[
h.div(class_="overflow-x-auto")[
h.table(

28
repub/entrypoint.py
View file

@ -63,6 +63,16 @@ def parse_args(argv: list[str] | None = None) -> tuple[str, argparse.Namespace]:
action="store_true",
help="Serve published feeds from /feeds for local development",
)
serve_parser.add_argument(
"--reload",
action="store_true",
help="Reload the web UI when source files change",
)
serve_parser.add_argument(
"--reader-app-url",
default=os.environ.get("REPUBLISHER_READER_APP_URL", ""),
help="URL of the AnyNews reader application linked from the publisher UI",
)
crawl_parser = subparsers.add_parser("crawl", help="Run the feed crawler once")
crawl_parser.add_argument(
@ -144,16 +154,18 @@ def _install_signal_handlers(stop_event: asyncio.Event) -> None:
signal.signal(signum, request_stop)
async def _serve_app(*, host: str, port: int, dev_mode: bool) -> None:
async def _serve_app(
*, host: str, port: int, dev_mode: bool, reload: bool, reader_app_url: str | None
) -> None:
stop_event = asyncio.Event()
_install_signal_handlers(stop_event)
app = create_app(dev_mode=dev_mode)
app = create_app(dev_mode=dev_mode, reader_app_url=reader_app_url)
app.extensions[SHUTDOWN_EVENT_KEY] = stop_event
config = HypercornConfig()
config.bind = [f"{host}:{port}"]
config.use_reloader = False
config.use_reloader = reload
config.accesslog = "-"
config.errorlog = "-"
@ -203,7 +215,15 @@ def entrypoint(argv: list[str] | None = None) -> int:
return 2
with suppress(KeyboardInterrupt):
asyncio.run(_serve_app(host=args.host, port=port, dev_mode=bool(args.dev_mode)))
asyncio.run(
_serve_app(
host=args.host,
port=port,
dev_mode=bool(args.dev_mode),
reload=bool(args.reload),
reader_app_url=args.reader_app_url,
)
)
return 0

76
repub/jobs.py
View file

@ -668,6 +668,7 @@ def load_runs_view(
now: datetime | None = None,
completed_page: int = 1,
completed_page_size: int = COMPLETED_EXECUTION_PAGE_SIZE,
path_prefix: str = "",
) -> RunsView:
reference_time = now or datetime.now(UTC)
resolved_log_dir = Path(log_dir)
@ -727,6 +728,7 @@ def load_runs_view(
execution,
resolved_log_dir,
reference_time,
path_prefix=path_prefix,
queued_follow_up=queued_by_job.get(
_job_id(cast(Job, execution.job))
),
@ -739,6 +741,7 @@ def load_runs_view(
reference_time,
position=position,
total_count=len(queued_executions),
path_prefix=path_prefix,
)
for position, execution in enumerate(queued_executions, start=1)
),
@ -748,12 +751,16 @@ def load_runs_view(
running_by_job.get(job.id),
queued_by_job.get(job.id),
reference_time,
path_prefix=path_prefix,
)
for job in jobs
),
"completed": tuple(
_project_completed_execution(
execution, resolved_log_dir, reference_time
execution,
resolved_log_dir,
reference_time,
path_prefix=path_prefix,
)
for execution in completed_executions
),
@ -801,10 +808,14 @@ def clear_completed_executions(*, log_dir: str | Path) -> int:
def load_dashboard_view(
*, log_dir: str | Path, now: datetime | None = None
*, log_dir: str | Path, now: datetime | None = None, path_prefix: str = ""
) -> dict[str, object]:
reference_time = now or datetime.now(UTC)
runs_view = load_runs_view(log_dir=log_dir, now=reference_time)
runs_view = load_runs_view(
log_dir=log_dir,
now=reference_time,
path_prefix=path_prefix,
)
output_dir = Path(log_dir).parent
running_by_job_id = {
int(cast(int, execution["job_id"])): execution
@ -842,6 +853,7 @@ def load_dashboard_view(
running_execution=running_by_job_id.get(_job_id(cast(Job, job))),
queued_execution=queued_by_job_id.get(_job_id(cast(Job, job))),
upcoming_job=upcoming_by_job_id.get(_job_id(cast(Job, job))),
path_prefix=path_prefix,
)
for job in jobs
),
@ -855,9 +867,9 @@ def load_dashboard_view(
def load_execution_log_view(
*, log_dir: str | Path, job_id: int, execution_id: int
*, log_dir: str | Path, job_id: int, execution_id: int, path_prefix: str = ""
) -> ExecutionLogView:
route = f"/job/{job_id}/execution/{execution_id}/logs"
route = _path(path_prefix, f"/job/{job_id}/execution/{execution_id}/logs")
with database.reader():
execution_primary_key = getattr(JobExecution, "_meta").primary_key
execution = (
@ -924,11 +936,16 @@ def _scheduler_job_id(job_id: int) -> str:
return f"{SCHEDULER_JOB_PREFIX}{job_id}"
def _path(prefix: str, path: str) -> str:
return f"{prefix.rstrip('/')}{path}" if prefix else path
def _project_running_execution(
execution: JobExecution,
log_dir: Path,
reference_time: datetime,
*,
path_prefix: str,
queued_follow_up: JobExecution | None = None,
) -> dict[str, object]:
job = cast(Job, execution.job)
@ -957,13 +974,16 @@ def _project_running_execution(
if execution.stop_requested_at
else "streaming stats from worker"
),
"log_href": f"/job/{job_id}/execution/{execution_id}/logs",
"log_href": _path(path_prefix, f"/job/{job_id}/execution/{execution_id}/logs"),
"log_exists": artifacts.log_path.exists(),
"cancel_label": "Cancel" if queued_follow_up is not None else "Stop",
"cancel_post_path": (
f"/actions/queued-executions/{_execution_id(queued_follow_up)}/cancel"
_path(
path_prefix,
f"/actions/queued-executions/{_execution_id(queued_follow_up)}/cancel",
)
if queued_follow_up is not None
else f"/actions/executions/{execution_id}/cancel"
else _path(path_prefix, f"/actions/executions/{execution_id}/cancel")
),
}
@ -974,6 +994,7 @@ def _project_queued_execution(
*,
position: int,
total_count: int,
path_prefix: str,
) -> dict[str, object]:
job = cast(Job, execution.job)
queued_at = _coerce_datetime(cast(datetime | str, execution.created_at))
@ -990,19 +1011,28 @@ def _project_queued_execution(
"status_tone": "idle",
"run_label": "Queued",
"run_disabled": True,
"run_post_path": f"/actions/jobs/{_job_id(job)}/run-now",
"cancel_post_path": (f"/actions/queued-executions/{execution_id}/cancel"),
"run_post_path": _path(path_prefix, f"/actions/jobs/{_job_id(job)}/run-now"),
"cancel_post_path": _path(
path_prefix,
f"/actions/queued-executions/{execution_id}/cancel",
),
"move_up_disabled": position == 1,
"move_up_post_path": (
None
if position == 1
else f"/actions/queued-executions/{execution_id}/move-up"
else _path(
path_prefix,
f"/actions/queued-executions/{execution_id}/move-up",
)
),
"move_down_disabled": position == total_count,
"move_down_post_path": (
None
if position == total_count
else f"/actions/queued-executions/{execution_id}/move-down"
else _path(
path_prefix,
f"/actions/queued-executions/{execution_id}/move-down",
)
),
}
@ -1012,6 +1042,8 @@ def _project_upcoming_job(
running_execution: JobExecution | None,
queued_execution: JobExecution | None,
reference_time: datetime,
*,
path_prefix: str,
) -> dict[str, object]:
job_id = _job_id(job)
trigger = _job_trigger(job)
@ -1051,14 +1083,21 @@ def _project_upcoming_job(
"run_reason": run_reason,
"toggle_label": "Disable" if job.enabled else "Enable",
"toggle_enabled": not job.enabled,
"run_post_path": f"/actions/jobs/{job_id}/run-now",
"toggle_post_path": f"/actions/jobs/{job_id}/toggle-enabled",
"delete_post_path": f"/actions/jobs/{job_id}/delete",
"run_post_path": _path(path_prefix, f"/actions/jobs/{job_id}/run-now"),
"toggle_post_path": _path(
path_prefix,
f"/actions/jobs/{job_id}/toggle-enabled",
),
"delete_post_path": _path(path_prefix, f"/actions/jobs/{job_id}/delete"),
}
def _project_completed_execution(
execution: JobExecution, log_dir: Path, reference_time: datetime
execution: JobExecution,
log_dir: Path,
reference_time: datetime,
*,
path_prefix: str,
) -> dict[str, object]:
job = cast(Job, execution.job)
job_id = _job_id(job)
@ -1100,7 +1139,7 @@ def _project_completed_execution(
else "Worker exited with failure"
)
),
"log_href": f"/job/{job_id}/execution/{execution_id}/logs",
"log_href": _path(path_prefix, f"/job/{job_id}/execution/{execution_id}/logs"),
"log_exists": artifacts.log_path.exists(),
}
@ -1113,6 +1152,7 @@ def _project_source_feed(
running_execution: dict[str, object] | None = None,
queued_execution: dict[str, object] | None = None,
upcoming_job: dict[str, object] | None = None,
path_prefix: str,
) -> dict[str, object]:
source = cast(Source, job.source)
source_slug = str(source.slug)
@ -1163,7 +1203,7 @@ def _project_source_feed(
"run_post_path": (
str(upcoming_job["run_post_path"])
if upcoming_job is not None
else f"/actions/jobs/{_job_id(job)}/run-now"
else _path(path_prefix, f"/actions/jobs/{_job_id(job)}/run-now")
),
"artifact_footprint": _format_bytes(_directory_size(source_dir)),
}

52
repub/pages/dashboard.py
View file

@ -9,7 +9,6 @@ from htpy import Node, Renderable
from repub.components import (
action_button,
app_shell,
header_action_link,
inline_link,
muted_action_link,
stat_card,
@ -19,7 +18,9 @@ from repub.components import (
from repub.pages.runs import live_work_section, relative_time_formatter_script
def dashboard_header() -> Renderable:
def dashboard_header(
*, path_prefix: str = "/admin", reader_app_url: str | None = None
) -> Renderable:
return h.section[
h.div(
class_="flex flex-col gap-4 sm:flex-row sm:items-start sm:justify-between"
@ -30,8 +31,20 @@ def dashboard_header() -> Renderable:
],
],
h.div(class_="flex flex-wrap gap-2")[
header_action_link(href="/sources/create", label="Create source"),
muted_action_link(href="/sources", label="View sources"),
(
muted_action_link(
href=reader_app_url,
label="Open AnyNews",
target="_blank",
rel="noopener noreferrer",
)
if reader_app_url is not None
else None
),
muted_action_link(
href=f"{path_prefix}/publisher",
label="Publisher View",
),
],
]
]
@ -124,9 +137,6 @@ def _source_feed_row(source_feed: Mapping[str, object]) -> tuple[Node, ...]:
),
last_updated,
next_run,
h.p(class_="font-medium text-slate-900")[
str(source_feed["artifact_footprint"])
],
action_button(
label="Run now",
disabled=bool(source_feed["run_disabled"]),
@ -136,12 +146,15 @@ def _source_feed_row(source_feed: Mapping[str, object]) -> tuple[Node, ...]:
def published_feeds_table(
*, source_feeds: tuple[Mapping[str, object], ...] | None = None
*,
source_feeds: tuple[Mapping[str, object], ...] | None = None,
manage_sources_href: str | None = "/admin/sources",
show_heading: bool = True,
) -> Renderable:
rows = tuple(_source_feed_row(source_feed) for source_feed in (source_feeds or ()))
return table_section(
eyebrow="Published feeds",
title="Published feeds",
eyebrow="Published feeds" if show_heading else None,
title="Published feeds" if show_heading else None,
empty_message="No feeds have been published yet.",
headers=(
"Source",
@ -149,11 +162,14 @@ def published_feeds_table(
"Status",
"Last updated",
"Next run",
"Disk usage",
"Actions",
),
rows=rows,
actions=muted_action_link(href="/sources", label="Manage sources"),
actions=(
muted_action_link(href=manage_sources_href, label="Manage sources")
if manage_sources_href is not None
else None
),
)
@ -163,26 +179,32 @@ def dashboard_page() -> Renderable:
def dashboard_page_with_data(
*,
current_path: str = "/admin",
path_prefix: str = "/admin",
snapshot: Mapping[str, str] | None = None,
running_executions: tuple[Mapping[str, object], ...] | None = None,
queued_executions: tuple[Mapping[str, object], ...] | None = None,
source_feeds: tuple[Mapping[str, object], ...] | None = None,
reader_app_url: str | None = None,
) -> Renderable:
running_items = running_executions or ()
queued_items = queued_executions or ()
source_items = source_feeds or ()
return app_shell(
current_path="/",
current_path=current_path,
source_count=len(source_items),
running_count=len(running_items),
content=(
dashboard_header(),
dashboard_header(path_prefix=path_prefix, reader_app_url=reader_app_url),
operational_snapshot(snapshot=snapshot),
live_work_section(
running_executions=running_items,
queued_executions=queued_items,
),
published_feeds_table(source_feeds=source_items),
published_feeds_table(
source_feeds=source_items,
manage_sources_href=f"{path_prefix}/sources",
),
relative_time_formatter_script(),
),
)

50
repub/pages/publisher.py
View file

@ -1,19 +1,57 @@
from __future__ import annotations
from collections.abc import Mapping
import htpy as h
from htpy import Renderable
from repub.components import app_shell
from repub.components import publisher_shell
from repub.pages.dashboard import published_feeds_table
from repub.pages.runs import live_work_section, relative_time_formatter_script
def publisher_page(*, current_path: str) -> Renderable:
return app_shell(
def publisher_page(
*,
current_path: str,
source_feeds: tuple[Mapping[str, object], ...] | None = None,
running_executions: tuple[Mapping[str, object], ...] | None = None,
queued_executions: tuple[Mapping[str, object], ...] | None = None,
reader_app_url: str | None = None,
) -> Renderable:
return publisher_shell(
current_path=current_path,
content=(
h.section[
h.h1(class_="text-3xl font-semibold tracking-tight text-slate-950")[
"Hello publishers"
]
h.div(
class_="flex flex-col gap-3 sm:flex-row sm:items-end sm:justify-between"
)[
h.div[
h.p(
class_="text-xs font-semibold uppercase tracking-[0.22em] text-amber-600"
)["Republisher"],
h.h1(
class_="mt-1 text-3xl font-semibold tracking-tight text-slate-950"
)["Published feeds"],
],
reader_app_url
and h.a(
href=reader_app_url,
target="_blank",
rel="noopener noreferrer",
class_="inline-flex shrink-0 items-center justify-center rounded-full bg-amber-400 px-4 py-2.5 text-sm font-semibold text-slate-950 transition hover:bg-amber-300",
)["Open AnyNews"],
],
],
published_feeds_table(
source_feeds=source_feeds,
manage_sources_href=None,
show_heading=False,
),
live_work_section(
running_executions=running_executions,
queued_executions=queued_executions,
show_row_actions=False,
),
relative_time_formatter_script(),
),
)

73
repub/pages/runs.py
View file

@ -190,7 +190,9 @@ def _queue_row_attrs(execution: Mapping[str, object]) -> dict[str, str]:
}
def _running_row(execution: Mapping[str, object]) -> tuple[Node, ...]:
def _running_row(
execution: Mapping[str, object], *, show_row_actions: bool = True
) -> tuple[Node, ...]:
started_at = _maybe_text(execution, "started_at_iso")
started_at_label: Node = h.p(class_="truncate")[_text(execution, "started_at")]
if started_at is not None:
@ -203,7 +205,7 @@ def _running_row(execution: Mapping[str, object]) -> tuple[Node, ...]:
class_="truncate",
)[_text(execution, "started_at")]
return (
cells = (
_live_status_cell(
execution_id=_text(execution, "execution_id"),
status=_text(execution, "status"),
@ -222,6 +224,11 @@ def _running_row(execution: Mapping[str, object]) -> tuple[Node, ...]:
h.p(class_="font-medium text-slate-900")[_text(execution, "stats")],
h.p(class_="mt-0.5 text-xs text-slate-500")[_text(execution, "worker")],
],
)
if not show_row_actions:
return cells
return (
*cells,
h.div(class_="flex flex-wrap items-center gap-2")[
inline_link(
href=_text(execution, "log_href"),
@ -237,7 +244,9 @@ def _running_row(execution: Mapping[str, object]) -> tuple[Node, ...]:
)
def _queued_row(execution: Mapping[str, object]) -> tuple[Node, ...]:
def _queued_row(
execution: Mapping[str, object], *, show_row_actions: bool = True
) -> tuple[Node, ...]:
queued_at = _maybe_text(execution, "queued_at_iso")
queued_label: Node = h.p(class_="truncate")[_text(execution, "queued_at")]
if queued_at is not None:
@ -250,7 +259,7 @@ def _queued_row(execution: Mapping[str, object]) -> tuple[Node, ...]:
class_="truncate",
)[_text(execution, "queued_at")]
return (
cells = (
_live_status_cell(
execution_id=_text(execution, "execution_id"),
status="Queued",
@ -270,6 +279,11 @@ def _queued_row(execution: Mapping[str, object]) -> tuple[Node, ...]:
],
h.p(class_="mt-0.5 text-xs text-slate-500")["waiting for capacity"],
],
)
if not show_row_actions:
return cells
return (
*cells,
h.div(class_="flex flex-wrap items-center gap-2")[
action_button(
label=_queue_icon("up"),
@ -362,8 +376,8 @@ def _completed_row(execution: Mapping[str, object]) -> tuple[Node, ...]:
)
def _completed_page_action_path(page: int) -> str:
return f"/actions/runs/completed-page/{page}"
def _completed_page_action_path(page: int, *, path_prefix: str = "/admin") -> str:
return f"{path_prefix}/actions/runs/completed-page/{page}"
def _pagination_button(
@ -372,9 +386,12 @@ def _pagination_button(
page: int,
current: bool = False,
class_name: str,
path_prefix: str = "/admin",
) -> Renderable:
attributes = {
"data-on:pointerdown": f"@post('{_completed_page_action_path(page)}')",
"data-on:pointerdown": (
f"@post('{_completed_page_action_path(page, path_prefix=path_prefix)}')"
),
}
if current:
attributes["aria-current"] = "page"
@ -391,6 +408,7 @@ def _completed_history_pagination(
completed_page_size: int,
completed_total_count: int,
completed_total_pages: int,
path_prefix: str = "/admin",
) -> Renderable | None:
if completed_total_count <= completed_page_size:
return None
@ -410,6 +428,7 @@ def _completed_history_pagination(
_pagination_button(
label="Previous",
page=max(1, completed_page - 1),
path_prefix=path_prefix,
class_name=(
"relative inline-flex items-center rounded-xl border border-slate-200 "
"bg-white px-4 py-2 text-sm font-medium text-slate-700 hover:bg-stone-50"
@ -418,6 +437,7 @@ def _completed_history_pagination(
_pagination_button(
label="Next",
page=min(completed_total_pages, completed_page + 1),
path_prefix=path_prefix,
class_name=(
"relative ml-3 inline-flex items-center rounded-xl border border-slate-200 "
"bg-white px-4 py-2 text-sm font-medium text-slate-700 hover:bg-stone-50"
@ -443,6 +463,7 @@ def _completed_history_pagination(
label=str(page_number),
page=page_number,
current=page_number == completed_page,
path_prefix=path_prefix,
class_name=(
"relative z-10 inline-flex items-center bg-amber-500 px-4 py-2 text-sm font-semibold text-slate-950"
if page_number == completed_page
@ -463,12 +484,14 @@ def _completed_history_section(
completed_page_size: int,
completed_total_count: int,
completed_total_pages: int,
path_prefix: str = "/admin",
) -> Renderable:
pagination = _completed_history_pagination(
completed_page=completed_page,
completed_page_size=completed_page_size,
completed_total_count=completed_total_count,
completed_total_pages=completed_total_pages,
path_prefix=path_prefix,
)
return h.section[
table_section(
@ -486,7 +509,7 @@ def _completed_history_section(
action_button(
label="Clear history",
tone="danger",
post_path="/actions/completed-executions/clear",
post_path=f"{path_prefix}/actions/completed-executions/clear",
)
if completed_total_count > 0
else None
@ -501,11 +524,18 @@ def live_work_section(
running_executions: tuple[Mapping[str, object], ...] | None = None,
queued_executions: tuple[Mapping[str, object], ...] | None = None,
actions: Node | None = None,
show_row_actions: bool = True,
) -> Renderable:
running_items = running_executions or ()
queued_items = queued_executions or ()
running_rows = tuple(_running_row(execution) for execution in running_items)
queued_rows = tuple(_queued_row(execution) for execution in queued_items)
running_rows = tuple(
_running_row(execution, show_row_actions=show_row_actions)
for execution in running_items
)
queued_rows = tuple(
_queued_row(execution, show_row_actions=show_row_actions)
for execution in queued_items
)
live_rows = running_rows + queued_rows
live_row_attrs = tuple(
_queue_row_attrs(execution) for execution in running_items + queued_items
@ -515,10 +545,9 @@ def live_work_section(
title="Running jobs",
empty_message="No jobs are running or queued.",
headers=(
"State",
"Source",
"Details",
"Actions",
("State", "Source", "Details", "Actions")
if show_row_actions
else ("State", "Source", "Details")
),
rows=live_rows,
row_attrs=live_row_attrs,
@ -585,6 +614,7 @@ def runs_page(
completed_total_count: int | None = None,
completed_total_pages: int | None = None,
source_count: int = 0,
path_prefix: str = "/admin",
) -> Renderable:
upcoming_items = upcoming_jobs or ()
completed_items = completed_executions or ()
@ -598,10 +628,13 @@ def runs_page(
)
return page_shell(
current_path="/runs",
current_path=f"{path_prefix}/runs",
eyebrow="Execution control",
title="Runs",
actions=muted_action_link(href="/sources", label="Back to sources"),
actions=muted_action_link(
href=f"{path_prefix}/sources",
label="Back to sources",
),
source_count=source_count,
running_count=len(running_executions or ()),
content=(
@ -629,6 +662,7 @@ def runs_page(
completed_page_size=completed_page_size,
completed_total_count=resolved_completed_total_count,
completed_total_pages=resolved_completed_total_pages,
path_prefix=path_prefix,
),
relative_time_formatter_script(),
),
@ -640,6 +674,7 @@ def execution_logs_page(
job_id: int,
execution_id: int,
log_view: Mapping[str, object] | None = None,
path_prefix: str = "/admin",
) -> Renderable:
if log_view is None:
log_view = {
@ -664,10 +699,10 @@ def execution_logs_page(
)
return page_shell(
current_path=f"/job/{job_id}/execution/{execution_id}/logs",
current_path=f"{path_prefix}/job/{job_id}/execution/{execution_id}/logs",
eyebrow="Execution log",
title=_text(log_view, "title"),
actions=muted_action_link(href="/runs", label="Back to runs"),
actions=muted_action_link(href=f"{path_prefix}/runs", label="Back to runs"),
content=(
section_card(
content=(
@ -677,7 +712,7 @@ def execution_logs_page(
class_="text-xs font-semibold uppercase tracking-[0.22em] text-amber-600"
)["Route"],
h.h2(class_="mt-2 text-xl font-semibold text-slate-950")[
f"/job/{job_id}/execution/{execution_id}/logs"
f"{path_prefix}/job/{job_id}/execution/{execution_id}/logs"
],
],
status_badge(

10
repub/pages/settings.py
View file

@ -23,12 +23,13 @@ def _value(settings: Mapping[str, object] | None, key: str, default: str = "") -
def settings_page(
*,
settings: Mapping[str, object] | None = None,
action_path: str = "/actions/settings",
action_path: str = "/admin/actions/settings",
source_count: int = 0,
running_count: int = 0,
path_prefix: str = "/admin",
) -> Renderable:
return page_shell(
current_path="/settings",
current_path=f"{path_prefix}/settings",
eyebrow="Configuration",
title="Settings",
description="Global runtime controls for the republisher.",
@ -85,7 +86,10 @@ def settings_page(
),
],
h.div(class_="flex flex-wrap justify-end gap-3 pt-2")[
muted_action_link(href="/", label="Back to dashboard"),
muted_action_link(
href=path_prefix,
label="Back to dashboard",
),
action_button(
label="Save settings",
tone="dark",

5
repub/pages/shim.py
View file

@ -3,7 +3,7 @@ from __future__ import annotations
import htpy as h
from htpy import Node, Renderable
from repub.components import app_shell
from repub.components import app_shell, publisher_shell
ON_LOAD_JS = (
"@post(window.location.pathname + "
@ -17,6 +17,7 @@ TAB_ID_JS = "self.crypto.randomUUID().substring(0,8)"
def shim_page(
*, datastar_src: str, current_path: str, head: Node | None = None
) -> Renderable:
shell = app_shell if current_path.startswith("/admin") else publisher_shell
return h.html(lang="en")[
h.head[
h.meta(charset="UTF-8"),
@ -33,7 +34,7 @@ def shim_page(
}
),
h.noscript["Your browser does not support JavaScript!"],
app_shell(
shell(
current_path=current_path,
content=(
h.section[

60
repub/pages/sources.py
View file

@ -55,7 +55,9 @@ def _checked(source: Mapping[str, object] | None, key: str, default: bool) -> bo
return bool(value)
def _source_row(source: Mapping[str, object]) -> tuple[Node, ...]:
def _source_row(
source: Mapping[str, object], *, path_prefix: str = "/admin"
) -> tuple[Node, ...]:
return (
h.div[
h.div(class_="font-semibold text-slate-950")[str(source["name"])],
@ -78,28 +80,37 @@ def _source_row(source: Mapping[str, object]) -> tuple[Node, ...]:
],
h.div(class_="flex flex-nowrap items-center gap-3 whitespace-nowrap")[
inline_link(
href=f"/sources/{source['slug']}/edit", label="Edit", tone="amber"
href=f"{path_prefix}/sources/{source['slug']}/edit",
label="Edit",
tone="amber",
),
action_button(
label="Delete",
tone="danger",
post_path=f"/actions/sources/{source['slug']}/delete",
post_path=f"{path_prefix}/actions/sources/{source['slug']}/delete",
),
],
)
def sources_table(
*, sources: tuple[Mapping[str, object], ...] | None = None
*,
sources: tuple[Mapping[str, object], ...] | None = None,
path_prefix: str = "/admin",
) -> Renderable:
rows = tuple(_source_row(source) for source in (sources or ()))
rows = tuple(
_source_row(source, path_prefix=path_prefix) for source in (sources or ())
)
return table_section(
eyebrow="Inventory",
title="Sources",
empty_message="No sources yet.",
headers=("Source", "Type", "Upstream", "Schedule", "Job state", "Actions"),
rows=rows,
actions=header_action_link(href="/sources/create", label="Create source"),
actions=header_action_link(
href=f"{path_prefix}/sources/create",
label="Create source",
),
)
@ -107,15 +118,16 @@ def sources_page(
*,
sources: tuple[Mapping[str, object], ...] | None = None,
running_count: int = 0,
path_prefix: str = "/admin",
) -> Renderable:
source_items = sources or ()
return page_shell(
current_path="/sources",
current_path=f"{path_prefix}/sources",
eyebrow="Source management",
title="Sources",
source_count=len(source_items),
running_count=running_count,
content=sources_table(sources=source_items),
content=sources_table(sources=source_items, path_prefix=path_prefix),
)
@ -124,6 +136,7 @@ def source_form(
mode: str,
action_path: str,
source: Mapping[str, object] | None = None,
path_prefix: str = "/admin",
) -> Renderable:
source_type = _value(source, "source_type", "pangea")
slug = _value(source, "slug")
@ -397,7 +410,7 @@ def source_form(
h.div(
class_="flex flex-wrap justify-end gap-3 border-t border-slate-200 pt-6"
)[
muted_action_link(href="/sources", label="Cancel"),
muted_action_link(href=f"{path_prefix}/sources", label="Cancel"),
action_button(
label=submit_label,
tone="dark",
@ -412,22 +425,27 @@ def source_form(
def create_source_page(
*,
action_path: str = "/actions/sources/create",
action_path: str = "/admin/actions/sources/create",
source_count: int = 0,
running_count: int = 0,
path_prefix: str = "/admin",
) -> Renderable:
actions = (
muted_action_link(href="/sources", label="Back to sources"),
header_action_link(href="/runs", label="View runs"),
muted_action_link(href=f"{path_prefix}/sources", label="Back to sources"),
header_action_link(href=f"{path_prefix}/runs", label="View runs"),
)
return page_shell(
current_path="/sources/create",
current_path=f"{path_prefix}/sources/create",
eyebrow="Source creation",
title="Create source",
actions=actions,
source_count=source_count,
running_count=running_count,
content=source_form(mode="create", action_path=action_path),
content=source_form(
mode="create",
action_path=action_path,
path_prefix=path_prefix,
),
)
@ -438,17 +456,23 @@ def edit_source_page(
action_path: str,
source_count: int = 0,
running_count: int = 0,
path_prefix: str = "/admin",
) -> Renderable:
actions = (
muted_action_link(href="/sources", label="Back to sources"),
header_action_link(href="/runs", label="View runs"),
muted_action_link(href=f"{path_prefix}/sources", label="Back to sources"),
header_action_link(href=f"{path_prefix}/runs", label="View runs"),
)
return page_shell(
current_path=f"/sources/{slug}/edit",
current_path=f"{path_prefix}/sources/{slug}/edit",
eyebrow="Source editing",
title="Edit source",
actions=actions,
source_count=source_count,
running_count=running_count,
content=source_form(mode="edit", action_path=action_path, source=source),
content=source_form(
mode="edit",
action_path=action_path,
source=source,
path_prefix=path_prefix,
),
)

33
repub/web/__init__.py Normal file
View file

@ -0,0 +1,33 @@
from repub.web.app import (
SHUTDOWN_EVENT_KEY,
create_app,
get_job_runtime,
get_refresh_broker,
get_tab_state_store,
render_create_source,
render_dashboard,
render_edit_source,
render_execution_logs,
render_publisher,
render_runs,
render_settings,
render_sources,
versioned_static_asset_href,
)
__all__ = [
"SHUTDOWN_EVENT_KEY",
"create_app",
"get_job_runtime",
"get_refresh_broker",
"get_tab_state_store",
"render_create_source",
"render_dashboard",
"render_edit_source",
"render_execution_logs",
"render_publisher",
"render_runs",
"render_settings",
"render_sources",
"versioned_static_asset_href",
]

24
repub/web/admin/__init__.py Normal file
View file

@ -0,0 +1,24 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from quart import Quart
from repub.web.admin.actions import register_admin_actions
from repub.web.admin.pages.dashboard import register_dashboard_routes
from repub.web.admin.pages.logs import register_log_routes
from repub.web.admin.pages.runs import register_runs_routes
from repub.web.admin.pages.settings import register_settings_routes
from repub.web.admin.pages.sources import register_source_routes
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_admin_routes(app: Quart, *, admin_required: RouteGuard) -> None:
register_dashboard_routes(app, admin_required=admin_required)
register_source_routes(app, admin_required=admin_required)
register_runs_routes(app, admin_required=admin_required)
register_settings_routes(app, admin_required=admin_required)
register_log_routes(app, admin_required=admin_required)
register_admin_actions(app, admin_required=admin_required)

181
repub/web/admin/actions.py Normal file
View file

@ -0,0 +1,181 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any, cast
from datastar_py import ServerSentEventGenerator as SSE
from datastar_py.quart import DatastarResponse, read_signals
from peewee import IntegrityError
from quart import Quart, Response
from repub.jobs import clear_completed_executions
from repub.model import (
create_source,
delete_job_source,
delete_source,
load_job_enabled,
save_setting,
source_slug_exists,
update_source,
)
from repub.web.app import (
RUNS_TAB_STATE_KEY,
_read_optional_signals,
_read_tab_id,
get_job_runtime,
get_tab_state_store,
run_job_now_response,
trigger_refresh,
validate_settings_form,
validate_source_form,
)
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_admin_actions(app: Quart, *, admin_required: RouteGuard) -> None:
@app.post("/admin/actions/sources/create")
@admin_required
async def admin_create_source_action() -> DatastarResponse:
signals = cast(dict[str, object], await read_signals())
source, error = validate_source_form(
signals,
slug_exists=source_slug_exists,
)
if error is not None:
return DatastarResponse(
SSE.patch_signals({"_formError": error, "_formSuccess": ""})
)
assert source is not None
try:
create_source(**source)
except IntegrityError:
return DatastarResponse(
SSE.patch_signals(
{"_formError": "Slug must be unique.", "_formSuccess": ""}
)
)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return DatastarResponse(SSE.redirect("/admin/sources"))
@app.post("/admin/actions/sources/<string:slug>/edit")
@admin_required
async def admin_edit_source_action(slug: str) -> DatastarResponse:
signals = cast(dict[str, object], await read_signals())
source, error = validate_source_form(
signals,
slug_exists=lambda candidate: candidate != slug
and source_slug_exists(candidate),
immutable_slug=slug,
)
if error is not None:
return DatastarResponse(
SSE.patch_signals({"_formError": error, "_formSuccess": ""})
)
assert source is not None
if update_source(slug, **source) is None:
return DatastarResponse(
SSE.patch_signals(
{"_formError": "Source does not exist.", "_formSuccess": ""}
)
)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return DatastarResponse(SSE.redirect("/admin/sources"))
@app.post("/admin/actions/sources/<string:slug>/delete")
@admin_required
async def admin_delete_source_action(slug: str) -> Response:
delete_source(slug)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return Response(status=204)
@app.post("/admin/actions/settings")
@admin_required
async def admin_update_settings_action() -> DatastarResponse:
signals = cast(dict[str, object], await read_signals())
settings, error = validate_settings_form(signals)
if error is not None:
return DatastarResponse(
SSE.patch_signals({"_formError": error, "_formSuccess": ""})
)
assert settings is not None
save_setting("max_concurrent_jobs", settings["max_concurrent_jobs"])
save_setting("feed_url", settings["feed_url"])
trigger_refresh(app)
return DatastarResponse(SSE.redirect("/admin/settings"))
@app.post("/admin/actions/runs/completed-page/<int:page>")
@admin_required
async def admin_set_completed_runs_page_action(page: int) -> Response:
signals = await _read_optional_signals()
tab_id = _read_tab_id(signals)
if tab_id is None:
return Response(status=400)
get_tab_state_store(app).update_page_state(
tab_id,
RUNS_TAB_STATE_KEY,
lambda state: {**state, "completed_page": max(1, page)},
)
trigger_refresh(app, tab_id=tab_id)
return Response(status=204)
@app.post("/admin/actions/jobs/<int:job_id>/run-now")
@admin_required
async def admin_run_job_now_action(job_id: int) -> Response:
return run_job_now_response(app, job_id)
@app.post("/admin/actions/jobs/<int:job_id>/toggle-enabled")
@admin_required
async def admin_toggle_job_enabled_action(job_id: int) -> Response:
enabled = load_job_enabled(job_id)
if enabled is not None:
get_job_runtime(app).set_job_enabled(job_id, enabled=not enabled)
trigger_refresh(app)
return Response(status=204)
@app.post("/admin/actions/jobs/<int:job_id>/delete")
@admin_required
async def admin_delete_job_action(job_id: int) -> Response:
delete_job_source(job_id)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return Response(status=204)
@app.post("/admin/actions/executions/<int:execution_id>/cancel")
@admin_required
async def admin_cancel_execution_action(execution_id: int) -> Response:
get_job_runtime(app).request_execution_cancel(execution_id)
trigger_refresh(app)
return Response(status=204)
@app.post("/admin/actions/queued-executions/<int:execution_id>/cancel")
@admin_required
async def admin_cancel_queued_execution_action(execution_id: int) -> Response:
get_job_runtime(app).cancel_queued_execution(execution_id)
trigger_refresh(app)
return Response(status=204)
@app.post("/admin/actions/queued-executions/<int:execution_id>/move-up")
@admin_required
async def admin_move_queued_execution_up_action(execution_id: int) -> Response:
get_job_runtime(app).move_queued_execution(execution_id, direction="up")
return Response(status=204)
@app.post("/admin/actions/queued-executions/<int:execution_id>/move-down")
@admin_required
async def admin_move_queued_execution_down_action(execution_id: int) -> Response:
get_job_runtime(app).move_queued_execution(execution_id, direction="down")
return Response(status=204)
@app.post("/admin/actions/completed-executions/clear")
@admin_required
async def admin_clear_completed_executions_action() -> Response:
clear_completed_executions(log_dir=app.config["REPUB_LOG_DIR"])
trigger_refresh(app)
return Response(status=204)

1
repub/web/admin/pages/__init__.py Normal file
View file

@ -0,0 +1 @@
from __future__ import annotations

26
repub/web/admin/pages/dashboard.py Normal file
View file

@ -0,0 +1,26 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from datastar_py.quart import DatastarResponse
from quart import Quart, Response
from repub.web.app import _page_patch_response, _shim_page_response, render_dashboard
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_dashboard_routes(app: Quart, *, admin_required: RouteGuard) -> None:
@app.get("/admin")
@admin_required
async def admin_dashboard_home() -> Response:
return _shim_page_response(current_path="/admin", static_prefix="/admin")
@app.post("/admin")
@admin_required
async def admin_dashboard_patch() -> DatastarResponse:
return await _page_patch_response(
app,
lambda _tab_id: render_dashboard(app, path_prefix="/admin"),
)

42
repub/web/admin/pages/logs.py Normal file
View file

@ -0,0 +1,42 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from datastar_py.quart import DatastarResponse
from htpy import Renderable
from quart import Quart, Response
from repub.web.app import (
_page_patch_response,
_shim_page_response,
render_execution_logs,
)
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_log_routes(app: Quart, *, admin_required: RouteGuard) -> None:
@app.get("/admin/job/<int:job_id>/execution/<int:execution_id>/logs")
@admin_required
async def admin_logs_home(job_id: int, execution_id: int) -> Response:
return _shim_page_response(
current_path=f"/admin/job/{job_id}/execution/{execution_id}/logs",
static_prefix="/admin",
)
@app.post("/admin/job/<int:job_id>/execution/<int:execution_id>/logs")
@admin_required
async def admin_logs_patch(
job_id: int,
execution_id: int,
) -> DatastarResponse:
async def render() -> Renderable:
return await render_execution_logs(
app,
job_id=job_id,
execution_id=execution_id,
path_prefix="/admin",
)
return await _page_patch_response(app, lambda _tab_id: render())

26
repub/web/admin/pages/runs.py Normal file
View file

@ -0,0 +1,26 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from datastar_py.quart import DatastarResponse
from quart import Quart, Response
from repub.web.app import _page_patch_response, _shim_page_response, render_runs
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_runs_routes(app: Quart, *, admin_required: RouteGuard) -> None:
@app.get("/admin/runs")
@admin_required
async def admin_runs_home() -> Response:
return _shim_page_response(current_path="/admin/runs", static_prefix="/admin")
@app.post("/admin/runs")
@admin_required
async def admin_runs_patch() -> DatastarResponse:
return await _page_patch_response(
app,
lambda tab_id: render_runs(app, tab_id=tab_id, path_prefix="/admin"),
)

26
repub/web/admin/pages/settings.py Normal file
View file

@ -0,0 +1,26 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from datastar_py.quart import DatastarResponse
from quart import Quart, Response
from repub.web.app import _page_patch_response, _shim_page_response, render_settings
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_settings_routes(app: Quart, *, admin_required: RouteGuard) -> None:
@app.get("/admin/settings")
@admin_required
async def admin_settings_home() -> Response:
return _shim_page_response(
current_path="/admin/settings",
static_prefix="/admin",
)
@app.post("/admin/settings")
@admin_required
async def admin_settings_patch() -> DatastarResponse:
return await _page_patch_response(app, lambda _tab_id: render_settings(app))

48
repub/web/admin/pages/sources.py Normal file
View file

@ -0,0 +1,48 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from datastar_py.quart import DatastarResponse
from quart import Quart, Response, request
from repub.web.app import (
_page_patch_response,
_shim_page_response,
render_create_source,
render_edit_source,
render_sources,
)
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_source_routes(app: Quart, *, admin_required: RouteGuard) -> None:
@app.get("/admin/sources")
@app.get("/admin/sources/create")
@app.get("/admin/sources/<string:slug>/edit")
@admin_required
async def admin_sources_shim(slug: str | None = None) -> Response:
del slug
return _shim_page_response(current_path=request.path, static_prefix="/admin")
@app.post("/admin/sources")
@admin_required
async def admin_sources_patch() -> DatastarResponse:
return await _page_patch_response(app, lambda _tab_id: render_sources(app))
@app.post("/admin/sources/create")
@admin_required
async def admin_create_source_patch() -> DatastarResponse:
return await _page_patch_response(
app,
lambda _tab_id: render_create_source(app),
)
@app.post("/admin/sources/<string:slug>/edit")
@admin_required
async def admin_edit_source_patch(slug: str) -> DatastarResponse:
return await _page_patch_response(
app,
lambda _tab_id: render_edit_source(slug, app),
)

377
repub/web.py → repub/web/app.py
View file

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import hashlib
import os
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
from contextlib import suppress
from datetime import timedelta
@ -11,12 +12,10 @@ from typing import Any, TypedDict, cast
from urllib.parse import urlparse
import htpy as h
from datastar_py import ServerSentEventGenerator as SSE
from datastar_py.quart import DatastarResponse, read_signals
from datastar_py.sse import DatastarEvent
from htpy import Renderable
from peewee import IntegrityError
from quart import Quart, Response, request, send_from_directory, url_for
from quart import Quart, Response, redirect, request
from repub.auth_headers import (
AUTH_MODE_DISABLED,
@ -29,23 +28,16 @@ from repub.datastar import RefreshBroker, TabStateStore, render_stream
from repub.jobs import (
COMPLETED_EXECUTION_PAGE_SIZE,
JobRuntime,
clear_completed_executions,
load_dashboard_view,
load_execution_log_view,
load_runs_view,
)
from repub.model import (
create_source,
delete_job_source,
delete_source,
initialize_database,
load_job_enabled,
load_settings_form,
load_source_form,
load_sources,
save_setting,
source_slug_exists,
update_source,
)
from repub.pages import (
create_source_page,
@ -67,6 +59,7 @@ TAB_STATE_CLEANER_TASK_KEY = "repub.tab_state_cleaner_task"
SHUTDOWN_EVENT_KEY = "repub.shutdown_event"
DEFAULT_LOG_DIR = Path("out/logs")
DEFAULT_FEEDS_DIR = Path("out/feeds")
READER_APP_URL_ENV = "REPUBLISHER_READER_APP_URL"
RUNS_TAB_STATE_KEY = "runs"
TAB_STATE_CLEAN_INTERVAL = timedelta(seconds=10)
@ -109,7 +102,7 @@ DEFAULT_PANGEA_CONTENT_FORMAT = "MOBILE_3"
DEFAULT_PANGEA_CONTENT_TYPE = "articles"
DEFAULT_PANGEA_MAX_ARTICLES = "10"
DEFAULT_PANGEA_OLDEST_ARTICLE = "3"
STATIC_DIR = Path(__file__).resolve().parent / "static"
STATIC_DIR = Path(__file__).resolve().parents[1] / "static"
CACHE_BUSTED_STATIC_ASSETS = frozenset({"app.css"})
CACHE_BUSTED_HASH_LENGTH = 12
@ -137,8 +130,8 @@ def versioned_static_asset_filename(filename: str) -> str:
return f"{asset_path.stem}-{truncated_hash}{asset_path.suffix}"
def versioned_static_asset_href(filename: str) -> str:
return f"/static/{versioned_static_asset_filename(filename)}"
def versioned_static_asset_href(filename: str, *, prefix: str = "/admin") -> str:
return f"{prefix}/static/{versioned_static_asset_filename(filename)}"
def _require_cache_busted_static_asset(filename: str) -> None:
@ -146,13 +139,18 @@ def _require_cache_busted_static_asset(filename: str) -> None:
raise ValueError(f"Unsupported cache-busted static asset: {filename}")
def create_app(*, dev_mode: bool = False) -> Quart:
app = Quart(__name__)
def create_app(*, dev_mode: bool = False, reader_app_url: str | None = None) -> Quart:
app = Quart(__name__, static_folder=None)
app.config["REPUB_DB_PATH"] = str(initialize_database())
app.config.setdefault("REPUB_LOG_DIR", DEFAULT_LOG_DIR)
app.config.setdefault("REPUB_FEEDS_DIR", DEFAULT_FEEDS_DIR)
app.config["REPUB_DEV_MODE"] = dev_mode
app.config["REPUB_AUTH_MODE"] = load_auth_mode()
app.config["REPUB_READER_APP_URL"] = _normalize_external_url(
os.environ.get(READER_APP_URL_ENV, "")
if reader_app_url is None
else reader_app_url
)
app.extensions[REFRESH_BROKER_KEY] = RefreshBroker()
app.extensions[JOB_RUNTIME_KEY] = None
app.extensions[TAB_STATE_STORE_KEY] = TabStateStore()
@ -161,270 +159,17 @@ def create_app(*, dev_mode: bool = False) -> Quart:
admin_required = _require_role(app, "admin")
publisher_required = _require_role(app, "publisher")
@app.get("/feeds/<path:feed_path>")
async def published_feed(feed_path: str) -> Response:
if not bool(app.config["REPUB_DEV_MODE"]):
return Response(status=404)
response = await send_from_directory(
str(Path(app.config["REPUB_FEEDS_DIR"])),
feed_path,
)
if Path(feed_path).suffix == ".rss":
response.mimetype = "application/rss+xml"
return response
@app.get("/static/<string:asset_name>-<string:asset_hash>.<string:extension>")
async def versioned_static_asset(
asset_name: str, asset_hash: str, extension: str
) -> Response:
logical_filename = f"{asset_name}.{extension}"
requested_filename = f"{asset_name}-{asset_hash}.{extension}"
if logical_filename in CACHE_BUSTED_STATIC_ASSETS:
response = await send_from_directory(str(STATIC_DIR), logical_filename)
response.cache_control.public = True
response.cache_control.max_age = 31536000
response.cache_control.immutable = True
return response
response = await send_from_directory(str(STATIC_DIR), requested_filename)
return response
@app.get("/publisher")
@publisher_required
async def publisher_home() -> Response:
return _shim_page_response(current_path="/publisher")
@app.get("/admin/publisher")
@admin_required
async def admin_publisher_home() -> Response:
return _shim_page_response(current_path="/admin/publisher")
@app.get("/")
@app.get("/sources")
@app.get("/sources/create")
@app.get("/sources/<string:slug>/edit")
@app.get("/runs")
@app.get("/settings")
@app.get("/job/<int:job_id>/execution/<int:execution_id>/logs")
@admin_required
async def page_shim(
slug: str | None = None,
job_id: int | None = None,
execution_id: int | None = None,
) -> Response:
del slug, job_id, execution_id
return _shim_page_response(current_path=request.path)
async def root_redirect() -> Response:
return cast(Response, redirect("/publisher"))
@app.post("/")
@admin_required
async def dashboard_patch() -> DatastarResponse:
return await _page_patch_response(app, lambda _tab_id: render_dashboard(app))
from repub.web.routes import register_routes
@app.post("/publisher")
@publisher_required
async def publisher_patch() -> DatastarResponse:
return await _page_patch_response(
app,
lambda _tab_id: render_publisher(current_path="/publisher"),
)
@app.post("/admin/publisher")
@admin_required
async def admin_publisher_patch() -> DatastarResponse:
return await _page_patch_response(
app,
lambda _tab_id: render_publisher(current_path="/admin/publisher"),
)
@app.post("/sources")
@admin_required
async def sources_patch() -> DatastarResponse:
return await _page_patch_response(app, lambda _tab_id: render_sources(app))
@app.post("/sources/create")
@admin_required
async def create_source_patch() -> DatastarResponse:
return await _page_patch_response(
app, lambda _tab_id: render_create_source(app)
)
@app.post("/sources/<string:slug>/edit")
@admin_required
async def edit_source_patch(slug: str) -> DatastarResponse:
return await _page_patch_response(
app, lambda _tab_id: render_edit_source(slug, app)
)
@app.post("/settings")
@admin_required
async def settings_patch() -> DatastarResponse:
return await _page_patch_response(app, lambda _tab_id: render_settings(app))
@app.post("/actions/sources/create")
@admin_required
async def create_source_action() -> DatastarResponse:
signals = cast(dict[str, object], await read_signals())
source, error = validate_source_form(
signals,
slug_exists=source_slug_exists,
)
if error is not None:
return DatastarResponse(
SSE.patch_signals({"_formError": error, "_formSuccess": ""})
)
assert source is not None
try:
create_source(**source)
except IntegrityError:
return DatastarResponse(
SSE.patch_signals(
{"_formError": "Slug must be unique.", "_formSuccess": ""}
)
)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return DatastarResponse(SSE.redirect("/sources"))
@app.post("/actions/sources/<string:slug>/edit")
@admin_required
async def edit_source_action(slug: str) -> DatastarResponse:
signals = cast(dict[str, object], await read_signals())
source, error = validate_source_form(
signals,
slug_exists=lambda candidate: candidate != slug
and source_slug_exists(candidate),
immutable_slug=slug,
)
if error is not None:
return DatastarResponse(
SSE.patch_signals({"_formError": error, "_formSuccess": ""})
)
assert source is not None
if update_source(slug, **source) is None:
return DatastarResponse(
SSE.patch_signals(
{"_formError": "Source does not exist.", "_formSuccess": ""}
)
)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return DatastarResponse(SSE.redirect("/sources"))
@app.post("/actions/sources/<string:slug>/delete")
@admin_required
async def delete_source_action(slug: str) -> Response:
delete_source(slug)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/settings")
@admin_required
async def update_settings_action() -> DatastarResponse:
signals = cast(dict[str, object], await read_signals())
settings, error = validate_settings_form(signals)
if error is not None:
return DatastarResponse(
SSE.patch_signals({"_formError": error, "_formSuccess": ""})
)
assert settings is not None
save_setting("max_concurrent_jobs", settings["max_concurrent_jobs"])
save_setting("feed_url", settings["feed_url"])
trigger_refresh(app)
return DatastarResponse(SSE.redirect("/settings"))
@app.post("/runs")
@admin_required
async def runs_patch() -> DatastarResponse:
return await _page_patch_response(
app,
lambda tab_id: render_runs(app, tab_id=tab_id),
)
@app.post("/actions/runs/completed-page/<int:page>")
@admin_required
async def set_completed_runs_page_action(page: int) -> Response:
signals = await _read_optional_signals()
tab_id = _read_tab_id(signals)
if tab_id is None:
return Response(status=400)
get_tab_state_store(app).update_page_state(
tab_id,
RUNS_TAB_STATE_KEY,
lambda state: {**state, "completed_page": max(1, page)},
)
trigger_refresh(app, tab_id=tab_id)
return Response(status=204)
@app.post("/actions/jobs/<int:job_id>/run-now")
@admin_required
async def run_job_now_action(job_id: int) -> Response:
get_job_runtime(app).run_job_now(job_id, reason="manual")
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/jobs/<int:job_id>/toggle-enabled")
@admin_required
async def toggle_job_enabled_action(job_id: int) -> Response:
enabled = load_job_enabled(job_id)
if enabled is not None:
get_job_runtime(app).set_job_enabled(job_id, enabled=not enabled)
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/jobs/<int:job_id>/delete")
@admin_required
async def delete_job_action(job_id: int) -> Response:
delete_job_source(job_id)
get_job_runtime(app).sync_jobs()
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/executions/<int:execution_id>/cancel")
@admin_required
async def cancel_execution_action(execution_id: int) -> Response:
get_job_runtime(app).request_execution_cancel(execution_id)
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/queued-executions/<int:execution_id>/cancel")
@admin_required
async def cancel_queued_execution_action(execution_id: int) -> Response:
get_job_runtime(app).cancel_queued_execution(execution_id)
trigger_refresh(app)
return Response(status=204)
@app.post("/actions/queued-executions/<int:execution_id>/move-up")
@admin_required
async def move_queued_execution_up_action(execution_id: int) -> Response:
get_job_runtime(app).move_queued_execution(execution_id, direction="up")
return Response(status=204)
@app.post("/actions/queued-executions/<int:execution_id>/move-down")
@admin_required
async def move_queued_execution_down_action(execution_id: int) -> Response:
get_job_runtime(app).move_queued_execution(execution_id, direction="down")
return Response(status=204)
@app.post("/actions/completed-executions/clear")
@admin_required
async def clear_completed_executions_action() -> Response:
clear_completed_executions(log_dir=app.config["REPUB_LOG_DIR"])
trigger_refresh(app)
return Response(status=204)
@app.post("/job/<int:job_id>/execution/<int:execution_id>/logs")
@admin_required
async def logs_patch(job_id: int, execution_id: int) -> DatastarResponse:
async def render() -> Renderable:
return await render_execution_logs(
app, job_id=job_id, execution_id=execution_id
)
return await _page_patch_response(app, lambda _tab_id: render())
register_routes(
app,
admin_required=admin_required,
publisher_required=publisher_required,
)
@app.before_serving
async def start_runtime() -> None:
@ -447,10 +192,20 @@ def create_app(*, dev_mode: bool = False) -> Quart:
return app
def _shim_page_response(*, current_path: str) -> Response:
def _normalize_external_url(value: str | None) -> str:
url = (value or "").strip()
if url == "":
return ""
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or parsed.netloc == "":
return ""
return url
def _shim_page_response(*, current_path: str, static_prefix: str) -> Response:
body, etag = _render_shim_page(
stylesheet_href=versioned_static_asset_href("app.css"),
datastar_src=url_for("static", filename="datastar@1.0.0-RC.8.js"),
stylesheet_href=versioned_static_asset_href("app.css", prefix=static_prefix),
datastar_src=f"{static_prefix}/static/datastar@1.0.0-RC.8.js",
current_path=current_path,
)
if request.if_none_match.contains(etag):
@ -525,21 +280,56 @@ def trigger_refresh(
get_refresh_broker(app).publish(event, tab_id=tab_id)
async def render_dashboard(app: Quart | None = None) -> Renderable:
def run_job_now_response(app: Quart, job_id: int) -> Response:
if load_job_enabled(job_id) is None:
return Response(status=204)
get_job_runtime(app).run_job_now(job_id, reason="manual")
trigger_refresh(app)
return Response(status=204)
async def render_dashboard(
app: Quart | None = None, *, path_prefix: str = "/admin"
) -> Renderable:
if app is None:
return dashboard_page_with_data()
view = load_dashboard_view(log_dir=app.config["REPUB_LOG_DIR"])
view = load_dashboard_view(
log_dir=app.config["REPUB_LOG_DIR"],
path_prefix=path_prefix,
)
return dashboard_page_with_data(
current_path=path_prefix,
path_prefix=path_prefix,
snapshot=cast(dict[str, str], view["snapshot"]),
running_executions=cast(tuple[dict[str, object], ...], view["running"]),
queued_executions=cast(tuple[dict[str, object], ...], view["queued"]),
source_feeds=cast(tuple[dict[str, object], ...], view["source_feeds"]),
reader_app_url=cast(str, app.config["REPUB_READER_APP_URL"]) or None,
)
async def render_publisher(*, current_path: str) -> Renderable:
return publisher_page(current_path=current_path)
async def render_publisher(
app: Quart | None = None,
*,
current_path: str,
path_prefix: str | None = None,
) -> Renderable:
if app is None:
return publisher_page(current_path=current_path)
resolved_path_prefix = path_prefix or current_path
view = load_dashboard_view(
log_dir=app.config["REPUB_LOG_DIR"],
path_prefix=resolved_path_prefix,
)
return publisher_page(
current_path=current_path,
source_feeds=cast(tuple[dict[str, object], ...], view["source_feeds"]),
running_executions=cast(tuple[dict[str, object], ...], view["running"]),
queued_executions=cast(tuple[dict[str, object], ...], view["queued"]),
reader_app_url=cast(str, app.config["REPUB_READER_APP_URL"]) or None,
)
async def render_sources(app: Quart | None = None) -> Renderable:
@ -570,16 +360,21 @@ async def render_edit_source(slug: str, app: Quart | None = None) -> Renderable:
source = load_source_form(slug)
if source is None:
return sources_page(sources=())
sidebar_counts = {} if app is None else _load_sidebar_counts(app)
return edit_source_page(
slug=slug,
source=source,
action_path=f"/actions/sources/{slug}/edit",
**({} if app is None else _load_sidebar_counts(app)),
action_path=f"/admin/actions/sources/{slug}/edit",
source_count=sidebar_counts.get("source_count", 0),
running_count=sidebar_counts.get("running_count", 0),
)
async def render_runs(
app: Quart | None = None, *, tab_id: str | None = None
app: Quart | None = None,
*,
tab_id: str | None = None,
path_prefix: str = "/admin",
) -> Renderable:
if app is None:
return runs_page()
@ -590,8 +385,10 @@ async def render_runs(
log_dir=app.config["REPUB_LOG_DIR"],
completed_page=resolved_completed_page,
completed_page_size=COMPLETED_EXECUTION_PAGE_SIZE,
path_prefix=path_prefix,
)
return runs_page(
path_prefix=path_prefix,
running_executions=cast(tuple[dict[str, object], ...], view["running"]),
queued_executions=cast(tuple[dict[str, object], ...], view["queued"]),
upcoming_jobs=cast(tuple[dict[str, object], ...], view["upcoming"]),
@ -616,7 +413,11 @@ async def render_settings(app: Quart | None = None) -> Renderable:
async def render_execution_logs(
app: Quart | None = None, *, job_id: int, execution_id: int
app: Quart | None = None,
*,
job_id: int,
execution_id: int,
path_prefix: str = "/admin",
) -> Renderable:
if app is None:
return execution_logs_page(job_id=job_id, execution_id=execution_id)
@ -625,10 +426,12 @@ async def render_execution_logs(
log_dir=app.config["REPUB_LOG_DIR"],
job_id=job_id,
execution_id=execution_id,
path_prefix=path_prefix,
)
return execution_logs_page(
job_id=job_id,
execution_id=execution_id,
path_prefix=path_prefix,
log_view={
"title": log_view.title,
"description": log_view.description,

29
repub/web/publisher/__init__.py Normal file
View file

@ -0,0 +1,29 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from quart import Quart
from repub.web.publisher.actions import register_publisher_actions
from repub.web.publisher.pages.dashboard import register_publisher_dashboard_routes
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_publisher_routes(
app: Quart,
*,
publisher_required: RouteGuard,
admin_required: RouteGuard,
) -> None:
register_publisher_dashboard_routes(
app,
publisher_required=publisher_required,
admin_required=admin_required,
)
register_publisher_actions(
app,
publisher_required=publisher_required,
admin_required=admin_required,
)

27
repub/web/publisher/actions.py Normal file
View file

@ -0,0 +1,27 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from quart import Quart, Response
from repub.web.app import run_job_now_response
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_publisher_actions(
app: Quart,
*,
publisher_required: RouteGuard,
admin_required: RouteGuard,
) -> None:
@app.post("/publisher/actions/jobs/<int:job_id>/run-now")
@publisher_required
async def publisher_run_job_now_action(job_id: int) -> Response:
return run_job_now_response(app, job_id)
@app.post("/admin/publisher/actions/jobs/<int:job_id>/run-now")
@admin_required
async def admin_publisher_run_job_now_action(job_id: int) -> Response:
return run_job_now_response(app, job_id)

1
repub/web/publisher/pages/__init__.py Normal file
View file

@ -0,0 +1 @@
from __future__ import annotations

57
repub/web/publisher/pages/dashboard.py Normal file
View file

@ -0,0 +1,57 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from datastar_py.quart import DatastarResponse
from quart import Quart, Response
from repub.web.app import _page_patch_response, _shim_page_response, render_publisher
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_publisher_dashboard_routes(
app: Quart,
*,
publisher_required: RouteGuard,
admin_required: RouteGuard,
) -> None:
@app.get("/publisher")
@publisher_required
async def publisher_home() -> Response:
return _shim_page_response(
current_path="/publisher", static_prefix="/publisher"
)
@app.post("/publisher")
@publisher_required
async def publisher_patch() -> DatastarResponse:
return await _page_patch_response(
app,
lambda _tab_id: render_publisher(
app,
current_path="/publisher",
path_prefix="/publisher",
),
)
@app.get("/admin/publisher")
@admin_required
async def admin_publisher_home() -> Response:
return _shim_page_response(
current_path="/admin/publisher",
static_prefix="/admin",
)
@app.post("/admin/publisher")
@admin_required
async def admin_publisher_patch() -> DatastarResponse:
return await _page_patch_response(
app,
lambda _tab_id: render_publisher(
app,
current_path="/admin/publisher",
path_prefix="/admin/publisher",
),
)

27
repub/web/routes.py Normal file
View file

@ -0,0 +1,27 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from quart import Quart
from repub.web.admin import register_admin_routes
from repub.web.publisher import register_publisher_routes
from repub.web.static import register_static_routes
RouteGuard = Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]
def register_routes(
app: Quart,
*,
admin_required: RouteGuard,
publisher_required: RouteGuard,
) -> None:
register_static_routes(app)
register_admin_routes(app, admin_required=admin_required)
register_publisher_routes(
app,
publisher_required=publisher_required,
admin_required=admin_required,
)

50
repub/web/static.py Normal file
View file

@ -0,0 +1,50 @@
from __future__ import annotations
from pathlib import Path
from quart import Quart, Response, send_from_directory
from repub.web.app import CACHE_BUSTED_STATIC_ASSETS, STATIC_DIR
def register_static_routes(app: Quart) -> None:
@app.get("/feeds/<path:feed_path>")
async def published_feed(feed_path: str) -> Response:
if not bool(app.config["REPUB_DEV_MODE"]):
return Response(status=404)
response = await send_from_directory(
str(Path(app.config["REPUB_FEEDS_DIR"])),
feed_path,
)
if Path(feed_path).suffix == ".rss":
response.mimetype = "application/rss+xml"
return response
@app.get("/admin/static/<path:filename>")
async def admin_static_asset(filename: str) -> Response:
return await _static_asset_response(filename)
@app.get("/publisher/static/<path:filename>")
async def publisher_static_asset(filename: str) -> Response:
return await _static_asset_response(filename)
async def _static_asset_response(filename: str) -> Response:
logical_filename = _cache_busted_logical_filename(filename)
if logical_filename is not None:
response = await send_from_directory(str(STATIC_DIR), logical_filename)
response.cache_control.public = True
response.cache_control.max_age = 31536000
response.cache_control.immutable = True
return response
return await send_from_directory(str(STATIC_DIR), filename)
def _cache_busted_logical_filename(filename: str) -> str | None:
for logical_filename in CACHE_BUSTED_STATIC_ASSETS:
logical_path = Path(logical_filename)
prefix = f"{logical_path.stem}-"
if filename.startswith(prefix) and filename.endswith(logical_path.suffix):
return logical_filename
return None

40
tests/test_entrypoint.py
View file

@ -39,6 +39,28 @@ def test_parse_args_supports_dev_mode_flag() -> None:
assert args.dev_mode is True
def test_parse_args_supports_reload_flag() -> None:
command, args = parse_args(["serve", "--reload"])
assert command == "serve"
assert args.reload is True
def test_parse_args_uses_reader_app_url_env_var(monkeypatch) -> None:
monkeypatch.setenv(
"REPUBLISHER_READER_APP_URL",
"https://s3.amazonaws.com/anynews/marti-noticias/index.html",
)
command, args = parse_args(["serve"])
assert command == "serve"
assert (
args.reader_app_url
== "https://s3.amazonaws.com/anynews/marti-noticias/index.html"
)
def test_parse_args_supports_cleanup_media_defaults() -> None:
command, args = parse_args(["cleanup-media"])
@ -169,8 +191,9 @@ def test_entrypoint_passes_dev_mode_to_create_app(monkeypatch) -> None:
def __init__(self) -> None:
self.extensions: dict[str, object] = {}
def fake_create_app(*, dev_mode: bool) -> StubApp:
def fake_create_app(*, dev_mode: bool, reader_app_url: str | None) -> StubApp:
recorded["dev_mode"] = dev_mode
recorded["reader_app_url"] = reader_app_url
return StubApp()
def fake_install_signal_handlers(stop_event: object) -> None:
@ -185,6 +208,7 @@ def test_entrypoint_passes_dev_mode_to_create_app(monkeypatch) -> None:
recorded["app"] = app
recorded["host"] = config.bind[0].split(":")[0]
recorded["port"] = int(config.bind[0].split(":")[1])
recorded["reload"] = config.use_reloader
recorded["shutdown_trigger"] = shutdown_trigger
shutdown_event = cast(Any, app.extensions["repub.shutdown_event"])
recorded["app_shutdown_event"] = shutdown_event
@ -198,12 +222,24 @@ def test_entrypoint_passes_dev_mode_to_create_app(monkeypatch) -> None:
monkeypatch.setattr("repub.entrypoint.hypercorn_serve", fake_hypercorn_serve)
exit_code = entrypoint(
["serve", "--dev-mode", "--host", "0.0.0.0", "--port", "9090"]
[
"serve",
"--dev-mode",
"--reload",
"--host",
"0.0.0.0",
"--port",
"9090",
"--reader-app-url",
"https://reader.example/index.html",
]
)
assert exit_code == 0
assert recorded["dev_mode"] is True
assert recorded["reader_app_url"] == "https://reader.example/index.html"
assert recorded["host"] == "0.0.0.0"
assert recorded["port"] == 9090
assert recorded["reload"] is True
assert recorded["stop_event"] is recorded["app_shutdown_event"]
assert callable(recorded["shutdown_trigger"])

118
tests/test_header_auth.py
View file

@ -29,10 +29,10 @@ def _configure_trusted_auth(monkeypatch, tmp_path: Path, name: str) -> None:
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(tmp_path / f"{name}.db"))
def _assert_datastar_shell(body: str) -> None:
def _assert_datastar_shell(body: str, *, static_prefix: str) -> None:
assert body.startswith("<!doctype html>")
assert 'id="js"' in body
assert 'src="/static/datastar@1.0.0-RC.8.js"' in body
assert f'src="{static_prefix}/static/datastar@1.0.0-RC.8.js"' in body
assert 'data-init="@post(window.location.pathname +' in body
assert '<main id="morph"' in body
assert "Connecting" in body
@ -62,7 +62,7 @@ def test_trusted_header_mode_rejects_admin_route_without_identity(
async def run() -> None:
client = create_app().test_client()
response = await client.get("/")
response = await client.get("/admin")
assert response.status_code == 401
@ -78,7 +78,7 @@ def test_trusted_header_mode_ignores_generic_forwarded_identity_headers(
client = create_app().test_client()
response = await client.get(
"/",
"/admin",
headers={
"X-Forwarded-User": "mallory",
"X-Forwarded-Email": "mallory@example.org",
@ -100,7 +100,7 @@ def test_trusted_header_mode_rejects_malformed_trusted_identity_headers(
client = create_app().test_client()
response = await client.get(
"/",
"/admin",
headers={
"X-Republisher-Auth-Role": "admin",
"X-Republisher-Auth-Provider": "gp",
@ -120,7 +120,7 @@ def test_trusted_header_mode_allows_admin_identity_on_admin_route(
async def run() -> None:
client = create_app().test_client()
response = await client.get("/", headers=_trusted_headers(role="admin"))
response = await client.get("/admin", headers=_trusted_headers(role="admin"))
assert response.status_code == 200
@ -135,7 +135,9 @@ def test_trusted_header_mode_rejects_publisher_identity_on_admin_route(
async def run() -> None:
client = create_app().test_client()
response = await client.get("/", headers=_trusted_headers(role="publisher"))
response = await client.get(
"/admin", headers=_trusted_headers(role="publisher")
)
assert response.status_code == 403
@ -152,7 +154,7 @@ def test_trusted_header_mode_rejects_admin_action_without_identity(
app.config["REPUB_LOG_DIR"] = tmp_path / "logs"
client = app.test_client()
response = await client.post("/actions/completed-executions/clear")
response = await client.post("/admin/actions/completed-executions/clear")
assert response.status_code == 401
@ -170,7 +172,7 @@ def test_trusted_header_mode_rejects_publisher_identity_on_admin_action(
client = app.test_client()
response = await client.post(
"/actions/completed-executions/clear",
"/admin/actions/completed-executions/clear",
headers=_trusted_headers(role="publisher"),
)
@ -194,12 +196,12 @@ def test_trusted_header_mode_allows_publisher_identity_on_publisher_route(
body = await response.get_data(as_text=True)
assert response.status_code == 200
_assert_datastar_shell(body)
_assert_datastar_shell(body, static_prefix="/publisher")
asyncio.run(run())
def test_trusted_header_mode_publisher_post_serves_hello_publishers_morph(
def test_trusted_header_mode_publisher_post_serves_publisher_dashboard_morph(
monkeypatch, tmp_path: Path
) -> None:
_configure_trusted_auth(monkeypatch, tmp_path, "publisher-post")
@ -220,7 +222,7 @@ def test_trusted_header_mode_publisher_post_serves_hello_publishers_morph(
assert raw_connection.headers["Content-Type"] == "text/event-stream"
assert b"event: datastar-patch-elements" in chunk
assert b'<main id="morph"' in chunk
assert b"Hello publishers" in chunk
assert b"Published feeds" in chunk
await connection.disconnect()
asyncio.run(run())
@ -258,12 +260,12 @@ def test_trusted_header_mode_allows_admin_identity_on_admin_publisher_alias(
body = await response.get_data(as_text=True)
assert response.status_code == 200
_assert_datastar_shell(body)
_assert_datastar_shell(body, static_prefix="/admin")
asyncio.run(run())
def test_trusted_header_mode_admin_publisher_post_serves_hello_publishers_morph(
def test_trusted_header_mode_admin_publisher_post_serves_publisher_dashboard_morph(
monkeypatch, tmp_path: Path
) -> None:
_configure_trusted_auth(monkeypatch, tmp_path, "admin-alias-post")
@ -284,7 +286,7 @@ def test_trusted_header_mode_admin_publisher_post_serves_hello_publishers_morph(
assert raw_connection.headers["Content-Type"] == "text/event-stream"
assert b"event: datastar-patch-elements" in chunk
assert b'<main id="morph"' in chunk
assert b"Hello publishers" in chunk
assert b"Published feeds" in chunk
await connection.disconnect()
asyncio.run(run())
@ -308,7 +310,79 @@ def test_trusted_header_mode_rejects_publisher_identity_on_admin_publisher_alias
asyncio.run(run())
def test_trusted_header_mode_keeps_static_assets_public(
def test_trusted_header_mode_allows_publisher_identity_on_publisher_run_action(
monkeypatch, tmp_path: Path
) -> None:
_configure_trusted_auth(monkeypatch, tmp_path, "publisher-run-action")
async def run() -> None:
client = create_app().test_client()
response = await client.post(
"/publisher/actions/jobs/999/run-now",
headers=_trusted_headers(role="publisher"),
)
assert response.status_code == 204
asyncio.run(run())
def test_trusted_header_mode_rejects_admin_identity_on_publisher_run_action(
monkeypatch, tmp_path: Path
) -> None:
_configure_trusted_auth(monkeypatch, tmp_path, "publisher-run-action-admin")
async def run() -> None:
client = create_app().test_client()
response = await client.post(
"/publisher/actions/jobs/999/run-now",
headers=_trusted_headers(role="admin"),
)
assert response.status_code == 403
asyncio.run(run())
def test_trusted_header_mode_allows_admin_identity_on_admin_publisher_run_action(
monkeypatch, tmp_path: Path
) -> None:
_configure_trusted_auth(monkeypatch, tmp_path, "admin-publisher-run-action")
async def run() -> None:
client = create_app().test_client()
response = await client.post(
"/admin/publisher/actions/jobs/999/run-now",
headers=_trusted_headers(role="admin"),
)
assert response.status_code == 204
asyncio.run(run())
def test_trusted_header_mode_rejects_publisher_identity_on_admin_publisher_run_action(
monkeypatch, tmp_path: Path
) -> None:
_configure_trusted_auth(monkeypatch, tmp_path, "admin-publisher-run-publisher")
async def run() -> None:
client = create_app().test_client()
response = await client.post(
"/admin/publisher/actions/jobs/999/run-now",
headers=_trusted_headers(role="publisher"),
)
assert response.status_code == 403
asyncio.run(run())
def test_trusted_header_mode_keeps_section_static_assets_public(
monkeypatch, tmp_path: Path
) -> None:
_configure_trusted_auth(monkeypatch, tmp_path, "static-public")
@ -316,9 +390,17 @@ def test_trusted_header_mode_keeps_static_assets_public(
async def run() -> None:
client = create_app().test_client()
response = await client.get("/static/datastar@1.0.0-RC.8.js")
for path in (
"/admin/static/datastar@1.0.0-RC.8.js",
"/publisher/static/datastar@1.0.0-RC.8.js",
):
response = await client.get(path)
assert response.status_code == 200
assert response.status_code == 200
root_response = await client.get("/static/datastar@1.0.0-RC.8.js")
assert root_response.status_code == 404
asyncio.run(run())

87
tests/test_jobs.py
View file

@ -2,8 +2,9 @@ from __future__ import annotations
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import cast
from repub.jobs import load_runs_view
from repub.jobs import load_dashboard_view, load_runs_view
from repub.model import (
Job,
JobExecution,
@ -232,6 +233,90 @@ def test_load_runs_view_projects_queued_executions_in_fifo_order(
assert view["queued"][1]["move_down_disabled"] is True
def test_load_runs_view_projects_admin_prefixed_action_and_log_paths(
tmp_path: Path,
) -> None:
initialize_database(tmp_path / "jobs-admin-path-prefix.db")
source = create_source(
name="Admin prefixed source",
slug="admin-prefixed-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=True,
cron_minute="*/5",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url="https://example.com/admin-prefixed.xml",
)
with database.writer():
job = Job.get(Job.source == source)
running = JobExecution.create(
job=job,
running_status=JobExecutionStatus.RUNNING,
started_at=datetime(2026, 3, 30, 12, 0, tzinfo=UTC),
)
view = load_runs_view(
log_dir=tmp_path / "out" / "logs",
now=datetime(2026, 3, 30, 12, 30, tzinfo=UTC),
path_prefix="/admin",
)
assert (
view["running"][0]["log_href"]
== f"/admin/job/{job.id}/execution/{int(running.get_id())}/logs"
)
assert (
view["running"][0]["cancel_post_path"]
== f"/admin/actions/executions/{int(running.get_id())}/cancel"
)
assert view["upcoming"][0]["run_post_path"] == (
f"/admin/actions/jobs/{job.id}/run-now"
)
assert view["upcoming"][0]["toggle_post_path"] == (
f"/admin/actions/jobs/{job.id}/toggle-enabled"
)
assert view["upcoming"][0]["delete_post_path"] == (
f"/admin/actions/jobs/{job.id}/delete"
)
def test_load_dashboard_view_projects_publisher_run_action_but_root_feed_links(
tmp_path: Path,
) -> None:
initialize_database(tmp_path / "jobs-publisher-dashboard-prefix.db")
source = create_source(
name="Publisher prefixed source",
slug="publisher-prefixed-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=True,
cron_minute="*/5",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url="https://example.com/publisher-prefixed.xml",
)
with database.reader():
job = Job.get(Job.source == source)
view = load_dashboard_view(
log_dir=tmp_path / "out" / "logs",
now=datetime(2026, 3, 30, 12, 30, tzinfo=UTC),
path_prefix="/publisher",
)
source_feeds = cast(tuple[dict[str, object], ...], view["source_feeds"])
source_feed = source_feeds[0]
assert source_feed["feed_href"] == "/feeds/publisher-prefixed-source/feed.rss"
assert source_feed["run_post_path"] == (f"/publisher/actions/jobs/{job.id}/run-now")
def test_load_runs_view_keeps_queued_jobs_in_scheduled_jobs(
tmp_path: Path,
) -> None:

6
tests/test_scheduler_runtime.py
View file

@ -1048,7 +1048,7 @@ def test_render_runs_uses_database_backed_jobs_and_executions(
assert "Running jobs" in body
assert "Scheduled jobs" in body
assert "Completed job executions" in body
assert f"/job/{job.id}/execution/{execution.get_id()}/logs" in body
assert f"/admin/job/{job.id}/execution/{execution.get_id()}/logs" in body
assert "Succeeded" in body
assert "Run now" in body
@ -1138,7 +1138,7 @@ def test_delete_job_action_removes_source_job_and_execution_history(
running_status=JobExecutionStatus.SUCCEEDED,
)
response = await client.post(f"/actions/jobs/{job.id}/delete")
response = await client.post(f"/admin/actions/jobs/{job.id}/delete")
assert response.status_code == 204
assert (
@ -1185,7 +1185,7 @@ def test_delete_source_action_removes_source_job_and_execution_history(
running_status=JobExecutionStatus.SUCCEEDED,
)
response = await client.post("/actions/sources/delete-source-row/delete")
response = await client.post("/admin/actions/sources/delete-source-row/delete")
assert response.status_code == 204
assert (

466
tests/test_web.py
View file

@ -37,6 +37,7 @@ from repub.web import (
render_dashboard,
render_edit_source,
render_execution_logs,
render_publisher,
render_runs,
render_settings,
render_sources,
@ -55,7 +56,12 @@ def _db_writer(fn):
def test_web_routes_do_not_access_peewee_models_directly() -> None:
web_source = Path("repub/web.py").read_text(encoding="utf-8")
web_paths = (
tuple(sorted(Path("repub/web").rglob("*.py")))
if Path("repub/web").is_dir()
else (Path("repub/web.py"),)
)
web_source = "\n".join(path.read_text(encoding="utf-8") for path in web_paths)
assert (
re.search(
@ -111,7 +117,7 @@ def test_action_button_omits_post_handler_when_disabled() -> None:
action_button(
label="Queued",
disabled=True,
post_path="/actions/jobs/7/run-now",
post_path="/admin/actions/jobs/7/run-now",
)
)
@ -138,11 +144,13 @@ def test_action_button_supports_datastar_pointerdown_post() -> None:
action_button(
label="Delete",
tone="danger",
post_path="/actions/jobs/7/delete",
post_path="/admin/actions/jobs/7/delete",
)
)
assert 'data-on:pointerdown="@post(&#39;/actions/jobs/7/delete&#39;)"' in markup
assert (
'data-on:pointerdown="@post(&#39;/admin/actions/jobs/7/delete&#39;)"' in markup
)
def test_runs_page_renders_completed_execution_end_time_as_relative_hoverable_time() -> (
@ -163,7 +171,7 @@ def test_runs_page_renders_completed_execution_end_time_as_relative_hoverable_ti
"status_tone": "done",
"stats": "1 requests • 1 items • 1 bytes",
"summary": "Worker exited successfully",
"log_href": "/job/7/execution/42/logs",
"log_href": "/admin/job/7/execution/42/logs",
},
)
)
@ -195,7 +203,7 @@ def test_runs_page_renders_completed_execution_state_cell_with_duration_and_end_
"status_tone": "done",
"stats": "1 requests • 1 items • 1 bytes",
"summary": "Worker exited successfully",
"log_href": "/job/7/execution/42/logs",
"log_href": "/admin/job/7/execution/42/logs",
},
)
)
@ -234,8 +242,8 @@ def test_runs_page_renders_combined_running_jobs_table() -> None:
"status_tone": "idle",
"run_label": "Queued",
"run_disabled": True,
"run_post_path": "/actions/jobs/7/run-now",
"cancel_post_path": "/actions/queued-executions/42/cancel",
"run_post_path": "/admin/actions/jobs/7/run-now",
"cancel_post_path": "/admin/actions/queued-executions/42/cancel",
"move_up_disabled": True,
"move_up_post_path": None,
"move_down_disabled": True,
@ -249,7 +257,7 @@ def test_runs_page_renders_combined_running_jobs_table() -> None:
assert "queued-source" in body
assert ">Queued<" in body
assert "bg-amber-200 text-amber-950" in body
assert "/actions/queued-executions/42/cancel" in body
assert "/admin/actions/queued-executions/42/cancel" in body
def test_runs_page_renders_running_state_cell_with_duration_and_started_at() -> None:
@ -269,9 +277,9 @@ def test_runs_page_renders_running_state_cell_with_duration_and_started_at() ->
"status": "Running",
"stats": "1 requests • 1 items • 1 byte",
"worker": "streaming stats from worker",
"log_href": "/job/1/execution/11/logs",
"log_href": "/admin/job/1/execution/11/logs",
"cancel_label": "Stop",
"cancel_post_path": "/actions/executions/11/cancel",
"cancel_post_path": "/admin/actions/executions/11/cancel",
},
)
)
@ -316,9 +324,9 @@ def test_runs_page_moves_scheduled_jobs_state_column_to_second_position() -> Non
"run_disabled": False,
"run_reason": "Ready",
"toggle_label": "Disable",
"toggle_post_path": "/actions/jobs/7/toggle-enabled",
"run_post_path": "/actions/jobs/7/run-now",
"delete_post_path": "/actions/jobs/7/delete",
"toggle_post_path": "/admin/actions/jobs/7/toggle-enabled",
"run_post_path": "/admin/actions/jobs/7/run-now",
"delete_post_path": "/admin/actions/jobs/7/delete",
},
)
)
@ -371,7 +379,7 @@ def test_runs_page_renders_clear_completed_button_and_pagination() -> None:
"status_tone": "done",
"stats": "1 requests • 1 items • 1 bytes",
"summary": "Worker exited successfully",
"log_href": f"/job/7/execution/{index}/logs",
"log_href": f"/admin/job/7/execution/{index}/logs",
}
for index in range(1, 21)
)
@ -385,29 +393,41 @@ def test_runs_page_renders_clear_completed_button_and_pagination() -> None:
)
)
assert "/actions/completed-executions/clear" in body
assert "/admin/actions/completed-executions/clear" in body
assert ">Clear history<" in body
assert "Showing" in body
assert "21" in body
assert "@post(&#39;/actions/runs/completed-page/1&#39;)" in body
assert "@post(&#39;/actions/runs/completed-page/2&#39;)" in body
assert "@post(&#39;/admin/actions/runs/completed-page/1&#39;)" in body
assert "@post(&#39;/admin/actions/runs/completed-page/2&#39;)" in body
assert 'aria-current="page"' in body
def test_root_get_serves_datastar_shim() -> None:
def test_root_get_redirects_to_publisher() -> None:
async def run() -> None:
client = create_app().test_client()
response = await client.get("/")
assert response.status_code == 302
assert response.headers["Location"] == "/publisher"
asyncio.run(run())
def test_admin_get_serves_datastar_shim_with_admin_static_assets() -> None:
async def run() -> None:
client = create_app().test_client()
response = await client.get("/admin")
body = await response.get_data(as_text=True)
stylesheet_href = versioned_static_asset_href("app.css")
stylesheet_href = versioned_static_asset_href("app.css", prefix="/admin")
assert response.status_code == 200
assert response.headers["ETag"]
assert body.startswith("<!doctype html>")
assert f'<link rel="stylesheet" href="{stylesheet_href}">' in body
assert (
'<script id="js" defer type="module" src="/static/datastar@1.0.0-RC.8.js"></script>'
'<script id="js" defer type="module" src="/admin/static/datastar@1.0.0-RC.8.js"></script>'
in body
)
assert 'data-signals:tabid="self.crypto.randomUUID().substring(0,8)"' in body
@ -417,47 +437,110 @@ def test_root_get_serves_datastar_shim() -> None:
assert '<main id="morph"' in body
assert "lg:grid-cols-[14rem_minmax(0,1fr)]" in body
assert "lg:px-5 lg:py-4" in body
assert 'href="/sources"' in body
assert 'href="/runs"' in body
assert 'href="/settings"' in body
assert 'href="/admin/sources"' in body
assert 'href="/admin/runs"' in body
assert 'href="/admin/settings"' in body
assert "Connecting" in body
asyncio.run(run())
def test_publisher_get_serves_datastar_shim_with_publisher_static_assets() -> None:
async def run() -> None:
client = create_app().test_client()
response = await client.get("/publisher")
body = await response.get_data(as_text=True)
stylesheet_href = versioned_static_asset_href("app.css", prefix="/publisher")
assert response.status_code == 200
assert f'<link rel="stylesheet" href="{stylesheet_href}">' in body
assert (
'<script id="js" defer type="module" src="/publisher/static/datastar@1.0.0-RC.8.js"></script>'
in body
)
assert '<main id="morph"' in body
assert "Connecting" in body
asyncio.run(run())
def test_old_root_level_admin_routes_do_not_serve_admin_pages() -> None:
async def run() -> None:
client = create_app().test_client()
for path in (
"/sources",
"/sources/create",
"/runs",
"/settings",
"/job/1/execution/1/logs",
):
response = await client.get(path)
assert response.status_code == 404
asyncio.run(run())
def test_versioned_static_asset_href_uses_truncated_file_hash() -> None:
href = versioned_static_asset_href("app.css")
assert re.fullmatch(r"/static/app-[0-9a-f]{12}\.css", href)
assert re.fullmatch(r"/admin/static/app-[0-9a-f]{12}\.css", href)
def test_versioned_static_asset_route_serves_registered_css_file() -> None:
def test_versioned_static_asset_href_supports_publisher_prefix() -> None:
href = versioned_static_asset_href("app.css", prefix="/publisher")
assert re.fullmatch(r"/publisher/static/app-[0-9a-f]{12}\.css", href)
def test_section_static_asset_routes_serve_registered_css_file() -> None:
async def run() -> None:
client = create_app().test_client()
expected = (
Path(__file__).resolve().parents[1] / "repub" / "static" / "app.css"
).read_text(encoding="utf-8")
response = await client.get("/static/app-deadbeefcafe.css")
body = await response.get_data(as_text=True)
for path in (
"/admin/static/app-deadbeefcafe.css",
"/publisher/static/app-deadbeefcafe.css",
):
response = await client.get(path)
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert response.mimetype == "text/css"
assert body == expected
assert response.status_code == 200
assert response.mimetype == "text/css"
assert body == expected
asyncio.run(run())
def test_versioned_static_asset_route_preserves_existing_hyphenated_files() -> None:
def test_section_static_asset_routes_preserve_existing_hyphenated_files() -> None:
async def run() -> None:
client = create_app().test_client()
for path in (
"/admin/static/datastar@1.0.0-RC.8.js",
"/publisher/static/datastar@1.0.0-RC.8.js",
):
response = await client.get(path)
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert response.mimetype == "text/javascript"
assert body.startswith("// Datastar v1.0.0-RC.8")
asyncio.run(run())
def test_root_static_asset_route_no_longer_serves_app_assets() -> None:
async def run() -> None:
client = create_app().test_client()
response = await client.get("/static/datastar@1.0.0-RC.8.js")
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert response.mimetype == "text/javascript"
assert body.startswith("// Datastar v1.0.0-RC.8")
assert response.status_code == 404
asyncio.run(run())
@ -473,14 +556,14 @@ def test_create_app_bootstraps_default_database_path(
assert (tmp_path / "republisher.db").exists()
def test_root_get_honors_if_none_match() -> None:
def test_admin_get_honors_if_none_match() -> None:
async def run() -> None:
client = create_app().test_client()
initial = await client.get("/")
initial = await client.get("/admin")
etag = initial.headers["ETag"]
response = await client.get("/", headers={"If-None-Match": etag})
response = await client.get("/admin", headers={"If-None-Match": etag})
assert response.status_code == 304
assert response.headers["ETag"] == etag
@ -491,7 +574,7 @@ def test_root_get_honors_if_none_match() -> None:
def test_dashboard_post_serves_morph_component() -> None:
async def run() -> None:
client = create_app().test_client()
async with client.request("/?u=shim", method="POST") as connection:
async with client.request("/admin?u=shim", method="POST") as connection:
await connection.send_complete()
chunk = await asyncio.wait_for(connection.receive(), timeout=1)
raw_connection = cast(Any, connection)
@ -651,15 +734,46 @@ def test_render_dashboard_shows_dashboard_information_architecture(
assert "Operational snapshot" in body
assert "Running jobs" in body
assert "Published feeds" in body
assert 'href="/sources"' in body
assert 'href="/runs"' in body
assert "Create source" in body
assert 'href="/admin/sources"' in body
assert 'href="/admin/runs"' in body
assert 'href="/admin/publisher"' in body
assert "Publisher View" in body
assert "Create source" not in body
assert "View sources" not in body
assert "lg:grid-cols-[14rem_minmax(0,1fr)]" in body
assert "lg:px-5 lg:py-4" in body
asyncio.run(run())
def test_render_dashboard_shows_configured_reader_app_link(
monkeypatch, tmp_path: Path
) -> None:
db_path = tmp_path / "dashboard-reader-link.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
monkeypatch.setenv(
"REPUBLISHER_READER_APP_URL",
"https://s3.amazonaws.com/anynews/marti-noticias/index.html",
)
async def run() -> None:
app = create_app()
body = str(await render_dashboard(app))
link_match = re.search(
r'<a href="https://s3\.amazonaws\.com/anynews/marti-noticias/index\.html"[^>]*>Open AnyNews</a>',
body,
)
assert link_match is not None
link = link_match.group(0)
assert 'target="_blank"' in link
assert 'rel="noopener noreferrer"' in link
assert "bg-white" in link
assert "bg-amber-400" not in link
asyncio.run(run())
def test_render_dashboard_shows_empty_state_rows(monkeypatch, tmp_path: Path) -> None:
db_path = tmp_path / "dashboard-empty.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
@ -688,9 +802,9 @@ def test_dashboard_running_table_matches_runs_page_live_table_markup() -> None:
"status": "Running",
"stats": "1 requests • 1 items • 1 byte",
"worker": "streaming stats from worker",
"log_href": "/job/1/execution/11/logs",
"log_href": "/admin/job/1/execution/11/logs",
"cancel_label": "Stop",
"cancel_post_path": "/actions/executions/11/cancel",
"cancel_post_path": "/admin/actions/executions/11/cancel",
},
)
queued_executions = (
@ -706,8 +820,8 @@ def test_dashboard_running_table_matches_runs_page_live_table_markup() -> None:
"status_tone": "idle",
"run_label": "Queued",
"run_disabled": True,
"run_post_path": "/actions/jobs/2/run-now",
"cancel_post_path": "/actions/queued-executions/22/cancel",
"run_post_path": "/admin/actions/jobs/2/run-now",
"cancel_post_path": "/admin/actions/queued-executions/22/cancel",
"move_up_disabled": True,
"move_up_post_path": None,
"move_down_disabled": True,
@ -733,7 +847,7 @@ def test_dashboard_running_table_matches_runs_page_live_table_markup() -> None:
assert "running-source" in dashboard_body
assert "queued-source" in dashboard_body
assert "bg-sky-100 text-sky-800" in dashboard_body
assert "/job/1/execution/11/logs" in dashboard_body
assert "/admin/job/1/execution/11/logs" in dashboard_body
assert runs_body.count(">State<") >= 1
@ -829,7 +943,11 @@ def test_load_dashboard_view_lists_source_feed_artifacts(
source_feeds = cast(
tuple[dict[str, object], ...],
load_dashboard_view(log_dir=log_dir, now=reference_time)["source_feeds"],
load_dashboard_view(
log_dir=log_dir,
now=reference_time,
path_prefix="/admin",
)["source_feeds"],
)
assert source_feeds == (
@ -845,7 +963,7 @@ def test_load_dashboard_view_lists_source_feed_artifacts(
"next_run": "Not scheduled",
"next_run_at": None,
"run_disabled": False,
"run_post_path": f"/actions/jobs/{available_job.id}/run-now",
"run_post_path": f"/admin/actions/jobs/{available_job.id}/run-now",
"artifact_footprint": "3.0 KB",
},
{
@ -860,7 +978,7 @@ def test_load_dashboard_view_lists_source_feed_artifacts(
"next_run": "Not scheduled",
"next_run_at": None,
"run_disabled": False,
"run_post_path": f"/actions/jobs/{missing_job.id}/run-now",
"run_post_path": f"/admin/actions/jobs/{missing_job.id}/run-now",
"artifact_footprint": "0 B",
},
)
@ -991,19 +1109,164 @@ def test_render_dashboard_shows_source_feed_links_and_statuses(
assert "Never published" in body
assert "Next run" in body
assert ">Run now<" in body
assert f"/actions/jobs/{published_job.id}/run-now" in body
assert f"/actions/jobs/{missing_job.id}/run-now" in body
assert f"/admin/actions/jobs/{published_job.id}/run-now" in body
assert f"/admin/actions/jobs/{missing_job.id}/run-now" in body
assert "data-next-run-at" in body
asyncio.run(run())
def test_render_publisher_shows_published_feeds_with_publisher_actions(
monkeypatch, tmp_path: Path
) -> None:
db_path = tmp_path / "publisher-render.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
app = create_app()
app.config["REPUB_LOG_DIR"] = tmp_path / "out" / "logs"
source = create_source(
name="Publisher source",
slug="publisher-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=True,
cron_minute="*/5",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url="https://example.com/publisher.xml",
)
async def run() -> None:
feed_path = tmp_path / "out" / "feeds" / "publisher-source" / "feed.rss"
feed_path.parent.mkdir(parents=True)
feed_path.write_text("<rss/>\n", encoding="utf-8")
job = _db_reader(lambda: Job.get(Job.source == source))
body = str(await render_publisher(app, current_path="/publisher"))
assert body.count("Published feeds") == 1
assert "Publisher source" in body
assert 'href="/feeds/publisher-source/feed.rss"' in body
assert "Available" in body
assert "Next run" in body
assert "Disk usage" not in body
assert f"/publisher/actions/jobs/{job.id}/run-now" in body
assert f"/admin/actions/jobs/{job.id}/run-now" not in body
assert 'href="/admin/sources"' not in body
assert "Create source" not in body
assert "Manage sources" not in body
asyncio.run(run())
def test_render_publisher_shows_live_work_without_admin_controls(
monkeypatch, tmp_path: Path
) -> None:
db_path = tmp_path / "publisher-live-work.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
app = create_app()
app.config["REPUB_LOG_DIR"] = tmp_path / "out" / "logs"
running_source = create_source(
name="Publisher running source",
slug="publisher-running-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=True,
cron_minute="*/5",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url="https://example.com/running.xml",
)
queued_source = create_source(
name="Publisher queued source",
slug="publisher-queued-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=True,
cron_minute="*/5",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url="https://example.com/queued.xml",
)
running_job, running_execution, queued_execution = _db_writer(
lambda: (
Job.get(Job.source == running_source),
JobExecution.create(
job=Job.get(Job.source == running_source),
running_status=JobExecutionStatus.RUNNING,
started_at=datetime(2026, 3, 30, 12, 0, tzinfo=UTC),
),
JobExecution.create(
job=Job.get(Job.source == queued_source),
running_status=JobExecutionStatus.PENDING,
),
)
)
async def run() -> None:
body = str(await render_publisher(app, current_path="/publisher"))
assert body.index("Published feeds") < body.index("Live work")
assert "Running jobs" in body
assert "Publisher running source" in body
assert "Publisher queued source" in body
assert "Queue position #1" in body
assert f"/publisher/job/{running_job.id}/execution/" not in body
assert (
f"/publisher/actions/executions/{int(running_execution.get_id())}/cancel"
not in body
)
assert (
f"/publisher/actions/queued-executions/{int(queued_execution.get_id())}/cancel"
not in body
)
assert "View log" not in body
assert ">Stop<" not in body
assert ">Cancel<" not in body
assert body.count(">Actions<") == 1
asyncio.run(run())
def test_render_publisher_shows_configured_reader_app_link(
monkeypatch, tmp_path: Path
) -> None:
db_path = tmp_path / "publisher-reader-link.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
monkeypatch.setenv(
"REPUBLISHER_READER_APP_URL",
"https://s3.amazonaws.com/anynews/marti-noticias/index.html",
)
app = create_app()
async def run() -> None:
body = str(await render_publisher(app, current_path="/publisher"))
assert (
'href="https://s3.amazonaws.com/anynews/marti-noticias/index.html"' in body
)
assert 'target="_blank"' in body
assert "Open AnyNews" in body
asyncio.run(run())
def test_render_sources_shows_table_and_create_link() -> None:
async def run() -> None:
body = str(await render_sources())
assert ">Sources<" in body
assert 'href="/sources/create"' in body
assert 'href="/admin/sources/create"' in body
assert "No sources yet." in body
assert "guardian-feed" not in body
assert "podcast-audio" not in body
@ -1048,12 +1311,12 @@ def test_render_sources_shows_live_sidebar_badges(monkeypatch, tmp_path: Path) -
body = str(await render_sources(app))
assert re.search(
r'href="/sources"[^>]*>.*?<span>Sources</span>\s*<span[^>]*>2</span>',
r'href="/admin/sources"[^>]*>.*?<span>Sources</span>\s*<span[^>]*>2</span>',
body,
re.S,
)
assert re.search(
r'href="/runs"[^>]*>.*?<span>Runs</span>\s*<span[^>]*>0</span>',
r'href="/admin/runs"[^>]*>.*?<span>Runs</span>\s*<span[^>]*>0</span>',
body,
re.S,
)
@ -1086,12 +1349,12 @@ def test_render_dashboard_shows_live_sidebar_badges(
body = str(await render_dashboard(app))
assert re.search(
r'href="/sources"[^>]*>.*?<span>Sources</span>\s*<span[^>]*>1</span>',
r'href="/admin/sources"[^>]*>.*?<span>Sources</span>\s*<span[^>]*>1</span>',
body,
re.S,
)
assert re.search(
r'href="/runs"[^>]*>.*?<span>Runs</span>\s*<span[^>]*>0</span>',
r'href="/admin/runs"[^>]*>.*?<span>Runs</span>\s*<span[^>]*>0</span>',
body,
re.S,
)
@ -1125,7 +1388,7 @@ def test_render_sources_shows_delete_action_for_each_source(
assert "Delete" in body
assert "data-on:pointerdown" in body
assert "/actions/sources/delete-me/delete" in body
assert "/admin/actions/sources/delete-me/delete" in body
asyncio.run(run())
@ -1137,7 +1400,7 @@ def test_render_create_source_shows_dedicated_form_page() -> None:
assert ">Create source<" in body
assert "Source and job setup" in body
assert "data-signals__ifmissing" in body
assert "/actions/sources/create" in body
assert "/admin/actions/sources/create" in body
assert 'data-show="$sourceType === &#39;feed&#39;"' in body
assert 'data-show="$sourceType === &#39;pangea&#39;"' in body
assert "jobEnabled" in body
@ -1206,7 +1469,7 @@ def test_render_edit_source_shows_existing_values(monkeypatch, tmp_path: Path) -
body = str(await render_edit_source("kenya-health"))
assert "Edit source" in body
assert "/actions/sources/kenya-health/edit" in body
assert "/admin/actions/sources/kenya-health/edit" in body
assert "Kenya health desk" in body
assert "kenya-health" in body
assert 'id="source-slug"' in body
@ -1239,7 +1502,7 @@ def test_render_settings_shows_current_max_concurrent_jobs(
body = str(await render_settings(app))
assert ">Settings<" in body
assert "/actions/settings" in body
assert "/admin/actions/settings" in body
assert 'value="3"' in body
assert 'value="https://mirror.example"' in body
assert "Max concurrent jobs" in body
@ -1263,7 +1526,7 @@ def test_create_source_action_creates_pangea_source_and_job_in_database(
client = app.test_client()
response = await client.post(
"/actions/sources/create",
"/admin/actions/sources/create",
headers={"Datastar-Request": "true"},
json={
"sourceName": "Kenya health desk",
@ -1291,7 +1554,7 @@ def test_create_source_action_creates_pangea_source_and_job_in_database(
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert "window.location = '/sources'" in body
assert "window.location = '/admin/sources'" in body
source, pangea, job = _db_reader(
lambda: (
@ -1331,7 +1594,7 @@ def test_create_source_action_creates_feed_source_and_job_in_database(
client = app.test_client()
response = await client.post(
"/actions/sources/create",
"/admin/actions/sources/create",
headers={"Datastar-Request": "true"},
json={
"sourceName": "NASA feed",
@ -1351,7 +1614,7 @@ def test_create_source_action_creates_feed_source_and_job_in_database(
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert "window.location = '/sources'" in body
assert "window.location = '/admin/sources'" in body
source, feed, job = _db_reader(
lambda: (
@ -1409,7 +1672,7 @@ def test_edit_source_action_updates_existing_source_and_job_in_database(
client = app.test_client()
response = await client.post(
"/actions/sources/kenya-health/edit",
"/admin/actions/sources/kenya-health/edit",
headers={"Datastar-Request": "true"},
json={
"sourceName": "Kenya health desk nightly",
@ -1440,7 +1703,7 @@ def test_edit_source_action_updates_existing_source_and_job_in_database(
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert "window.location = '/sources'" in body
assert "window.location = '/admin/sources'" in body
source, pangea, job = _db_reader(
lambda: (
@ -1505,7 +1768,7 @@ def test_edit_source_action_rejects_slug_changes(monkeypatch, tmp_path: Path) ->
client = app.test_client()
response = await client.post(
"/actions/sources/kenya-health/edit",
"/admin/actions/sources/kenya-health/edit",
headers={"Datastar-Request": "true"},
json={
"sourceName": "Kenya health desk",
@ -1569,7 +1832,7 @@ def test_create_source_action_validates_duplicate_slug_and_pangea_type(
client = app.test_client()
response = await client.post(
"/actions/sources/create",
"/admin/actions/sources/create",
headers={"Datastar-Request": "true"},
json={
"sourceName": "Duplicate guardian",
@ -1619,7 +1882,7 @@ def test_settings_action_updates_max_concurrent_jobs(
client = app.test_client()
response = await client.post(
"/actions/settings",
"/admin/actions/settings",
headers={"Datastar-Request": "true"},
json={
"maxConcurrentJobs": "3",
@ -1629,7 +1892,7 @@ def test_settings_action_updates_max_concurrent_jobs(
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert "window.location = '/settings'" in body
assert "window.location = '/admin/settings'" in body
assert load_max_concurrent_jobs() == 3
assert load_settings_form()["feed_url"] == "https://mirror.example"
assert 'value="3"' in str(await render_settings(app))
@ -1648,7 +1911,7 @@ def test_settings_action_rejects_non_positive_max_concurrent_jobs(
client = app.test_client()
response = await client.post(
"/actions/settings",
"/admin/actions/settings",
headers={"Datastar-Request": "true"},
json={"maxConcurrentJobs": "0", "feedUrl": "https://mirror.example"},
)
@ -1670,7 +1933,7 @@ def test_settings_action_rejects_invalid_feed_url(monkeypatch, tmp_path: Path) -
client = app.test_client()
response = await client.post(
"/actions/settings",
"/admin/actions/settings",
headers={"Datastar-Request": "true"},
json={"maxConcurrentJobs": "2", "feedUrl": "mirror.example"},
)
@ -1722,7 +1985,7 @@ def test_render_runs_shows_running_scheduled_and_completed_tables(
assert "Scheduled jobs" in body
assert "Completed job executions" in body
assert "runs-render-source" in body
assert f"/job/{job.id}/execution/{execution.get_id()}/logs" in body
assert f"/admin/job/{job.id}/execution/{execution.get_id()}/logs" in body
assert "data-next-run-at" in body
assert "in " in body
@ -1797,7 +2060,7 @@ def test_runs_pagination_action_updates_only_the_current_tab(
)
async with client.request(
"/runs?u=shim",
"/admin/runs?u=shim",
method="POST",
headers={
"Datastar-Request": "true",
@ -1805,7 +2068,7 @@ def test_runs_pagination_action_updates_only_the_current_tab(
},
) as first_connection:
async with client.request(
"/runs?u=shim",
"/admin/runs?u=shim",
method="POST",
headers={
"Datastar-Request": "true",
@ -1825,7 +2088,7 @@ def test_runs_pagination_action_updates_only_the_current_tab(
).decode()
assert (
'href="/runs?completed_page=1" aria-current="page"'
'href="/admin/runs?completed_page=1" aria-current="page"'
not in first_body
)
assert (
@ -1840,7 +2103,7 @@ def test_runs_pagination_action_updates_only_the_current_tab(
) in second_body
response = await client.post(
"/actions/runs/completed-page/2",
"/admin/actions/runs/completed-page/2",
headers={"Datastar-Request": "true"},
json={"tabid": "tab-1"},
)
@ -1878,7 +2141,7 @@ def test_runs_patch_creates_and_cleans_up_tab_state(
client = app.test_client()
async with client.request(
"/runs?u=shim",
"/admin/runs?u=shim",
method="POST",
headers={
"Datastar-Request": "true",
@ -1955,7 +2218,7 @@ def test_render_runs_keeps_queued_execution_in_scheduled_jobs_table(
assert "scheduled-source" in body
assert ">Queued<" in body
assert (
f"/actions/queued-executions/{int(queued_execution.get_id())}/cancel"
f"/admin/actions/queued-executions/{int(queued_execution.get_id())}/cancel"
in body
)
assert "Ready" in body
@ -2004,9 +2267,12 @@ def test_render_runs_shows_cancel_button_for_running_row_with_queued_follow_up(
async def run() -> None:
body = str(await render_runs(app))
assert f"/job/{job.id}/execution/{int(running_execution.get_id())}/logs" in body
assert (
f"/actions/queued-executions/{int(pending_execution.get_id())}/cancel"
f"/admin/job/{job.id}/execution/{int(running_execution.get_id())}/logs"
in body
)
assert (
f"/admin/actions/queued-executions/{int(pending_execution.get_id())}/cancel"
in body
)
assert ">Cancel<" in body
@ -2031,9 +2297,9 @@ def test_render_runs_keeps_all_action_controls_visible_in_html_after_compaction(
"status": "Running",
"stats": "1 requests • 1 items • 1 byte",
"worker": "streaming stats from worker",
"log_href": "/job/1/execution/11/logs",
"log_href": "/admin/job/1/execution/11/logs",
"cancel_label": "Stop",
"cancel_post_path": "/actions/executions/11/cancel",
"cancel_post_path": "/admin/actions/executions/11/cancel",
},
),
queued_executions=(
@ -2049,8 +2315,8 @@ def test_render_runs_keeps_all_action_controls_visible_in_html_after_compaction(
"status_tone": "idle",
"run_label": "Queued",
"run_disabled": True,
"run_post_path": "/actions/jobs/2/run-now",
"cancel_post_path": "/actions/queued-executions/22/cancel",
"run_post_path": "/admin/actions/jobs/2/run-now",
"cancel_post_path": "/admin/actions/queued-executions/22/cancel",
"move_up_disabled": True,
"move_up_post_path": None,
"move_down_disabled": True,
@ -2070,9 +2336,9 @@ def test_render_runs_keeps_all_action_controls_visible_in_html_after_compaction(
"run_disabled": False,
"run_reason": "Ready",
"toggle_label": "Disable",
"toggle_post_path": "/actions/jobs/3/toggle-enabled",
"run_post_path": "/actions/jobs/3/run-now",
"delete_post_path": "/actions/jobs/3/delete",
"toggle_post_path": "/admin/actions/jobs/3/toggle-enabled",
"run_post_path": "/admin/actions/jobs/3/run-now",
"delete_post_path": "/admin/actions/jobs/3/delete",
},
),
completed_executions=(
@ -2087,7 +2353,7 @@ def test_render_runs_keeps_all_action_controls_visible_in_html_after_compaction(
"status_tone": "done",
"stats": "1 requests • 1 items • 1 byte",
"summary": "Worker exited successfully",
"log_href": "/job/4/execution/44/logs",
"log_href": "/admin/job/4/execution/44/logs",
},
),
)
@ -2098,7 +2364,7 @@ def test_render_runs_keeps_all_action_controls_visible_in_html_after_compaction(
assert ">Cancel<" in body
assert ">Run now<" in body
assert ">Disable<" in body
assert "/job/4/execution/44/logs" in body
assert "/admin/job/4/execution/44/logs" in body
def test_cancel_queued_execution_action_deletes_pending_row_without_touching_running_execution(
@ -2143,7 +2409,7 @@ def test_cancel_queued_execution_action_deletes_pending_row_without_touching_run
)
response = await client.post(
f"/actions/queued-executions/{int(pending_execution.get_id())}/cancel"
f"/admin/actions/queued-executions/{int(pending_execution.get_id())}/cancel"
)
assert response.status_code == 204
@ -2217,7 +2483,7 @@ def test_clear_completed_executions_action_removes_history_and_log_artifacts(
completed_prefix.with_suffix(suffix).write_text("history", encoding="utf-8")
running_log_path.write_text("running", encoding="utf-8")
response = await client.post("/actions/completed-executions/clear")
response = await client.post("/admin/actions/completed-executions/clear")
assert response.status_code == 204
assert (
@ -2297,18 +2563,18 @@ def test_move_queued_execution_action_reorders_queue(
)
response = await client.post(
f"/actions/queued-executions/{int(second_execution.get_id())}/move-up"
f"/admin/actions/queued-executions/{int(second_execution.get_id())}/move-up"
)
assert response.status_code == 204
body = str(await render_runs(app))
assert body.index("second-queued-source") < body.index("first-queued-source")
assert (
f"/actions/queued-executions/{int(second_execution.get_id())}/move-down"
f"/admin/actions/queued-executions/{int(second_execution.get_id())}/move-down"
in body
)
assert (
f"/actions/queued-executions/{int(first_execution.get_id())}/move-up"
f"/admin/actions/queued-executions/{int(first_execution.get_id())}/move-up"
in body
)
@ -2349,7 +2615,7 @@ def test_toggle_job_enabled_action_removes_queued_execution(
)
)
response = await client.post(f"/actions/jobs/{job.id}/toggle-enabled")
response = await client.post(f"/admin/actions/jobs/{job.id}/toggle-enabled")
assert response.status_code == 204
assert _db_reader(lambda: Job.get_by_id(job.id).enabled) is False
@ -2361,7 +2627,7 @@ def test_toggle_job_enabled_action_removes_queued_execution(
)
body = str(await render_runs(app))
assert (
f"/actions/queued-executions/{int(queued_execution.get_id())}/cancel"
f"/admin/actions/queued-executions/{int(queued_execution.get_id())}/cancel"
not in body
)
assert "Disabled" in body
@ -2439,7 +2705,7 @@ def test_render_execution_logs_uses_app_route(monkeypatch, tmp_path: Path) -> No
)
assert f"Job {job.id} / execution {execution.get_id()}" in body
assert f"/job/{job.id}/execution/{execution.get_id()}/logs" in body
assert f"/admin/job/{job.id}/execution/{execution.get_id()}/logs" in body
assert "waiting for more log lines" in body
asyncio.run(run())