edit sources
This commit is contained in:
parent
847aeae772
commit
328a70ff9b
7 changed files with 512 additions and 38 deletions
|
|
@ -112,3 +112,5 @@ uv run repub crawl -c repub.toml
|
||||||
- Runtime ffmpeg availability is provided by the flake package and devshell.
|
- Runtime ffmpeg availability is provided by the flake package and devshell.
|
||||||
- Tests live under `tests/`.
|
- Tests live under `tests/`.
|
||||||
- `prompts/` is git ignored intentionally
|
- `prompts/` is git ignored intentionally
|
||||||
|
- Treat the repo-root `republisher.db` as user-owned local state. Do not delete or reset it as part of routine testing or verification.
|
||||||
|
- For automated tests or isolated verification, use a separate database path via `REPUBLISHER_DB_PATH` instead of mutating or removing the repo-root database.
|
||||||
|
|
|
||||||
|
|
@ -273,7 +273,16 @@ def input_field(
|
||||||
placeholder: str = "",
|
placeholder: str = "",
|
||||||
help_text: str | None = None,
|
help_text: str | None = None,
|
||||||
signal_name: str | None = None,
|
signal_name: str | None = None,
|
||||||
|
disabled: bool = False,
|
||||||
) -> Renderable:
|
) -> Renderable:
|
||||||
|
class_name = (
|
||||||
|
"mt-2 block w-full rounded-2xl border-0 px-3.5 py-2.5 text-sm shadow-sm ring-1 "
|
||||||
|
+ (
|
||||||
|
"cursor-not-allowed bg-slate-100 text-slate-500 ring-slate-200"
|
||||||
|
if disabled
|
||||||
|
else "bg-white text-slate-900 ring-slate-200 placeholder:text-slate-400 focus:outline-hidden focus:ring-2 focus:ring-amber-500"
|
||||||
|
)
|
||||||
|
)
|
||||||
return h.div[
|
return h.div[
|
||||||
h.label(for_=field_id, class_="block text-sm font-medium text-slate-900")[
|
h.label(for_=field_id, class_="block text-sm font-medium text-slate-900")[
|
||||||
label
|
label
|
||||||
|
|
@ -285,7 +294,8 @@ def input_field(
|
||||||
type="text",
|
type="text",
|
||||||
value=value,
|
value=value,
|
||||||
placeholder=placeholder,
|
placeholder=placeholder,
|
||||||
class_="mt-2 block w-full rounded-2xl border-0 bg-white px-3.5 py-2.5 text-sm text-slate-900 shadow-sm ring-1 ring-slate-200 placeholder:text-slate-400 focus:outline-hidden focus:ring-2 focus:ring-amber-500",
|
disabled=disabled,
|
||||||
|
class_=class_name,
|
||||||
),
|
),
|
||||||
help_text and h.p(class_="mt-2 text-xs text-slate-500")[help_text],
|
help_text and h.p(class_="mt-2 text-xs text-slate-500")[help_text],
|
||||||
]
|
]
|
||||||
|
|
|
||||||
141
repub/model.py
141
repub/model.py
|
|
@ -89,6 +89,59 @@ def source_slug_exists(slug: str) -> bool:
|
||||||
return Source.select().where(Source.slug == slug).exists()
|
return Source.select().where(Source.slug == slug).exists()
|
||||||
|
|
||||||
|
|
||||||
|
def load_source_form(slug: str) -> dict[str, object] | None:
|
||||||
|
with database.connection_context():
|
||||||
|
source = Source.get_or_none(Source.slug == slug)
|
||||||
|
if source is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
job = Job.get(Job.source == source)
|
||||||
|
form_data: dict[str, object] = {
|
||||||
|
"name": source.name,
|
||||||
|
"slug": source.slug,
|
||||||
|
"source_type": source.source_type,
|
||||||
|
"notes": source.notes,
|
||||||
|
"spider_arguments": job.spider_arguments,
|
||||||
|
"enabled": job.enabled,
|
||||||
|
"cron_minute": job.cron_minute,
|
||||||
|
"cron_hour": job.cron_hour,
|
||||||
|
"cron_day_of_month": job.cron_day_of_month,
|
||||||
|
"cron_day_of_week": job.cron_day_of_week,
|
||||||
|
"cron_month": job.cron_month,
|
||||||
|
"feed_url": "",
|
||||||
|
"pangea_domain": "",
|
||||||
|
"pangea_category": "",
|
||||||
|
"content_format": "MOBILE_3",
|
||||||
|
"content_type": "articles",
|
||||||
|
"max_articles": "10",
|
||||||
|
"oldest_article": "3",
|
||||||
|
"only_newest": True,
|
||||||
|
"include_authors": True,
|
||||||
|
"exclude_media": False,
|
||||||
|
"include_content": True,
|
||||||
|
}
|
||||||
|
if source.source_type == "feed":
|
||||||
|
feed = SourceFeed.get(SourceFeed.source == source)
|
||||||
|
form_data["feed_url"] = feed.feed_url
|
||||||
|
else:
|
||||||
|
pangea = SourcePangea.get(SourcePangea.source == source)
|
||||||
|
form_data.update(
|
||||||
|
{
|
||||||
|
"pangea_domain": pangea.domain,
|
||||||
|
"pangea_category": pangea.category_name,
|
||||||
|
"content_format": pangea.content_format,
|
||||||
|
"content_type": pangea.content_type,
|
||||||
|
"max_articles": str(pangea.max_articles),
|
||||||
|
"oldest_article": str(pangea.oldest_article),
|
||||||
|
"only_newest": pangea.only_newest,
|
||||||
|
"include_authors": pangea.include_authors,
|
||||||
|
"exclude_media": pangea.exclude_media,
|
||||||
|
"include_content": pangea.include_content,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return form_data
|
||||||
|
|
||||||
|
|
||||||
def create_source(
|
def create_source(
|
||||||
*,
|
*,
|
||||||
name: str,
|
name: str,
|
||||||
|
|
@ -154,6 +207,94 @@ def create_source(
|
||||||
return source
|
return source
|
||||||
|
|
||||||
|
|
||||||
|
def update_source(
|
||||||
|
source_slug: str,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
slug: str,
|
||||||
|
source_type: str,
|
||||||
|
notes: str,
|
||||||
|
spider_arguments: str,
|
||||||
|
enabled: bool,
|
||||||
|
cron_minute: str,
|
||||||
|
cron_hour: str,
|
||||||
|
cron_day_of_month: str,
|
||||||
|
cron_day_of_week: str,
|
||||||
|
cron_month: str,
|
||||||
|
feed_url: str = "",
|
||||||
|
pangea_domain: str = "",
|
||||||
|
pangea_category: str = "",
|
||||||
|
content_type: str = "",
|
||||||
|
only_newest: bool = True,
|
||||||
|
max_articles: int | None = None,
|
||||||
|
oldest_article: int | None = None,
|
||||||
|
include_authors: bool = True,
|
||||||
|
exclude_media: bool = False,
|
||||||
|
include_content: bool = True,
|
||||||
|
content_format: str = "",
|
||||||
|
) -> Source | None:
|
||||||
|
with database.connection_context():
|
||||||
|
with database.atomic():
|
||||||
|
source = Source.get_or_none(Source.slug == source_slug)
|
||||||
|
if source is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
source.name = name
|
||||||
|
source.notes = notes
|
||||||
|
source.source_type = source_type
|
||||||
|
source.save()
|
||||||
|
|
||||||
|
job = Job.get(Job.source == source)
|
||||||
|
job.enabled = enabled
|
||||||
|
job.spider_arguments = spider_arguments
|
||||||
|
job.cron_minute = cron_minute
|
||||||
|
job.cron_hour = cron_hour
|
||||||
|
job.cron_day_of_month = cron_day_of_month
|
||||||
|
job.cron_day_of_week = cron_day_of_week
|
||||||
|
job.cron_month = cron_month
|
||||||
|
job.save()
|
||||||
|
|
||||||
|
if source_type == "feed":
|
||||||
|
SourcePangea.delete().where(SourcePangea.source == source).execute()
|
||||||
|
feed = SourceFeed.get_or_none(SourceFeed.source == source)
|
||||||
|
if feed is None:
|
||||||
|
SourceFeed.create(source=source, feed_url=feed_url)
|
||||||
|
else:
|
||||||
|
feed.feed_url = feed_url
|
||||||
|
feed.save()
|
||||||
|
else:
|
||||||
|
SourceFeed.delete().where(SourceFeed.source == source).execute()
|
||||||
|
pangea = SourcePangea.get_or_none(SourcePangea.source == source)
|
||||||
|
if pangea is None:
|
||||||
|
SourcePangea.create(
|
||||||
|
source=source,
|
||||||
|
domain=pangea_domain,
|
||||||
|
category_name=pangea_category,
|
||||||
|
content_type=content_type,
|
||||||
|
only_newest=only_newest,
|
||||||
|
max_articles=max_articles,
|
||||||
|
oldest_article=oldest_article,
|
||||||
|
include_authors=include_authors,
|
||||||
|
exclude_media=exclude_media,
|
||||||
|
include_content=include_content,
|
||||||
|
content_format=content_format,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
pangea.domain = pangea_domain
|
||||||
|
pangea.category_name = pangea_category
|
||||||
|
pangea.content_type = content_type
|
||||||
|
pangea.only_newest = only_newest
|
||||||
|
pangea.max_articles = max_articles
|
||||||
|
pangea.oldest_article = oldest_article
|
||||||
|
pangea.include_authors = include_authors
|
||||||
|
pangea.exclude_media = exclude_media
|
||||||
|
pangea.include_content = include_content
|
||||||
|
pangea.content_format = content_format
|
||||||
|
pangea.save()
|
||||||
|
|
||||||
|
return source
|
||||||
|
|
||||||
|
|
||||||
def load_sources() -> tuple[dict[str, object], ...]:
|
def load_sources() -> tuple[dict[str, object], ...]:
|
||||||
with database.connection_context():
|
with database.connection_context():
|
||||||
sources = tuple(Source.select().order_by(Source.created_at.desc()))
|
sources = tuple(Source.select().order_by(Source.created_at.desc()))
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,12 @@
|
||||||
from repub.pages.dashboard import dashboard_page
|
from repub.pages.dashboard import dashboard_page
|
||||||
from repub.pages.runs import execution_logs_page, runs_page
|
from repub.pages.runs import execution_logs_page, runs_page
|
||||||
from repub.pages.shim import shim_page
|
from repub.pages.shim import shim_page
|
||||||
from repub.pages.sources import create_source_page, sources_page
|
from repub.pages.sources import create_source_page, edit_source_page, sources_page
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"create_source_page",
|
"create_source_page",
|
||||||
"dashboard_page",
|
"dashboard_page",
|
||||||
|
"edit_source_page",
|
||||||
"execution_logs_page",
|
"execution_logs_page",
|
||||||
"runs_page",
|
"runs_page",
|
||||||
"shim_page",
|
"shim_page",
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,19 @@ PANGEA_CONTENT_TYPES = (
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _value(source: Mapping[str, object] | None, key: str, default: str = "") -> str:
|
||||||
|
if source is None:
|
||||||
|
return default
|
||||||
|
return str(source.get(key, default))
|
||||||
|
|
||||||
|
|
||||||
|
def _checked(source: Mapping[str, object] | None, key: str, default: bool) -> bool:
|
||||||
|
if source is None:
|
||||||
|
return default
|
||||||
|
value = source.get(key, default)
|
||||||
|
return bool(value)
|
||||||
|
|
||||||
|
|
||||||
def _source_row(source: Mapping[str, object]) -> tuple[Node, ...]:
|
def _source_row(source: Mapping[str, object]) -> tuple[Node, ...]:
|
||||||
return (
|
return (
|
||||||
h.div[
|
h.div[
|
||||||
|
|
@ -64,7 +77,9 @@ def _source_row(source: Mapping[str, object]) -> tuple[Node, ...]:
|
||||||
h.p(class_="mt-2 text-xs text-slate-500")[str(source["last_run"])],
|
h.p(class_="mt-2 text-xs text-slate-500")[str(source["last_run"])],
|
||||||
],
|
],
|
||||||
h.div(class_="flex flex-nowrap items-center gap-3")[
|
h.div(class_="flex flex-nowrap items-center gap-3")[
|
||||||
inline_link(href="/sources/create", label="Edit", tone="amber"),
|
inline_link(
|
||||||
|
href=f"/sources/{source['slug']}/edit", label="Edit", tone="amber"
|
||||||
|
),
|
||||||
inline_link(href="/runs", label="View runs"),
|
inline_link(href="/runs", label="View runs"),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
@ -97,7 +112,27 @@ def sources_page(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def create_source_form(*, action_path: str = "/actions/sources/create") -> Renderable:
|
def source_form(
|
||||||
|
*,
|
||||||
|
mode: str,
|
||||||
|
action_path: str,
|
||||||
|
source: Mapping[str, object] | None = None,
|
||||||
|
) -> Renderable:
|
||||||
|
source_type = _value(source, "source_type", "pangea")
|
||||||
|
slug = _value(source, "slug")
|
||||||
|
title = "Source and job setup" if mode == "create" else "Edit source"
|
||||||
|
eyebrow = "Create" if mode == "create" else "Edit"
|
||||||
|
description = (
|
||||||
|
"Create the source and its paired job record."
|
||||||
|
if mode == "create"
|
||||||
|
else "Update the existing source and its paired job record."
|
||||||
|
)
|
||||||
|
status_label = "New source" if mode == "create" else "Existing source"
|
||||||
|
submit_label = "Create source" if mode == "create" else "Save changes"
|
||||||
|
initial_signals = "{sourceType: 'pangea'}"
|
||||||
|
if mode == "edit":
|
||||||
|
initial_signals = f"{{sourceType: '{source_type}', sourceSlug: '{slug}'}}"
|
||||||
|
|
||||||
return section_card(
|
return section_card(
|
||||||
content=(
|
content=(
|
||||||
h.div(
|
h.div(
|
||||||
|
|
@ -106,20 +141,16 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
h.div[
|
h.div[
|
||||||
h.p(
|
h.p(
|
||||||
class_="text-xs font-semibold uppercase tracking-[0.22em] text-amber-600"
|
class_="text-xs font-semibold uppercase tracking-[0.22em] text-amber-600"
|
||||||
)["Create"],
|
)[eyebrow],
|
||||||
h.h2(class_="mt-2 text-xl font-semibold text-slate-950")[
|
h.h2(class_="mt-2 text-xl font-semibold text-slate-950")[title],
|
||||||
"Source and job setup"
|
h.p(class_="mt-2 max-w-3xl text-sm text-slate-600")[description],
|
||||||
],
|
],
|
||||||
h.p(class_="mt-2 max-w-3xl text-sm text-slate-600")[
|
status_badge(label=status_label, tone="scheduled"),
|
||||||
"The create flow lives on its own page and creates the source plus its paired job record. This pass is visual only, but the fields already reflect the intended shape."
|
|
||||||
],
|
|
||||||
],
|
|
||||||
status_badge(label="New source", tone="scheduled"),
|
|
||||||
],
|
],
|
||||||
h.form(
|
h.form(
|
||||||
{
|
{
|
||||||
"data-signals": "{_formError: '', _formSuccess: ''}",
|
"data-signals": "{_formError: '', _formSuccess: ''}",
|
||||||
"data-signals__ifmissing": "{sourceType: 'pangea'}",
|
"data-signals__ifmissing": initial_signals,
|
||||||
"data-on:submit": f"@post('{action_path}')",
|
"data-on:submit": f"@post('{action_path}')",
|
||||||
},
|
},
|
||||||
class_="mt-5 space-y-6",
|
class_="mt-5 space-y-6",
|
||||||
|
|
@ -142,13 +173,16 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
input_field(
|
input_field(
|
||||||
label="Source name",
|
label="Source name",
|
||||||
field_id="source-name",
|
field_id="source-name",
|
||||||
|
value=_value(source, "name"),
|
||||||
signal_name="sourceName",
|
signal_name="sourceName",
|
||||||
),
|
),
|
||||||
input_field(
|
input_field(
|
||||||
label="Slug",
|
label="Slug",
|
||||||
field_id="source-slug",
|
field_id="source-slug",
|
||||||
|
value=slug,
|
||||||
help_text="Immutable after creation.",
|
help_text="Immutable after creation.",
|
||||||
signal_name="sourceSlug",
|
signal_name="sourceSlug",
|
||||||
|
disabled=mode == "edit",
|
||||||
),
|
),
|
||||||
h.div[
|
h.div[
|
||||||
h.label(
|
h.label(
|
||||||
|
|
@ -161,8 +195,12 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
name="source-type",
|
name="source-type",
|
||||||
class_="mt-2 block w-full rounded-2xl border-0 bg-white px-3.5 py-2.5 text-sm text-slate-900 shadow-sm ring-1 ring-slate-200 focus:outline-hidden focus:ring-2 focus:ring-amber-500",
|
class_="mt-2 block w-full rounded-2xl border-0 bg-white px-3.5 py-2.5 text-sm text-slate-900 shadow-sm ring-1 ring-slate-200 focus:outline-hidden focus:ring-2 focus:ring-amber-500",
|
||||||
)[
|
)[
|
||||||
h.option(value="feed")["feed"],
|
h.option(value="feed", selected=source_type == "feed")[
|
||||||
h.option(value="pangea", selected=True)["pangea"],
|
"feed"
|
||||||
|
],
|
||||||
|
h.option(value="pangea", selected=source_type == "pangea")[
|
||||||
|
"pangea"
|
||||||
|
],
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
|
|
@ -185,6 +223,7 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
input_field(
|
input_field(
|
||||||
label="Feed URL",
|
label="Feed URL",
|
||||||
field_id="feed-url",
|
field_id="feed-url",
|
||||||
|
value=_value(source, "feed_url"),
|
||||||
placeholder="https://example.com/feed.xml",
|
placeholder="https://example.com/feed.xml",
|
||||||
signal_name="feedUrl",
|
signal_name="feedUrl",
|
||||||
),
|
),
|
||||||
|
|
@ -209,37 +248,39 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
input_field(
|
input_field(
|
||||||
label="Pangea domain",
|
label="Pangea domain",
|
||||||
field_id="pangea-domain",
|
field_id="pangea-domain",
|
||||||
|
value=_value(source, "pangea_domain"),
|
||||||
signal_name="pangeaDomain",
|
signal_name="pangeaDomain",
|
||||||
),
|
),
|
||||||
input_field(
|
input_field(
|
||||||
label="Category name",
|
label="Category name",
|
||||||
field_id="pangea-category",
|
field_id="pangea-category",
|
||||||
|
value=_value(source, "pangea_category"),
|
||||||
signal_name="pangeaCategory",
|
signal_name="pangeaCategory",
|
||||||
),
|
),
|
||||||
select_field(
|
select_field(
|
||||||
label="Content format",
|
label="Content format",
|
||||||
field_id="content-format",
|
field_id="content-format",
|
||||||
options=PANGEA_CONTENT_FORMATS,
|
options=PANGEA_CONTENT_FORMATS,
|
||||||
selected="MOBILE_3",
|
selected=_value(source, "content_format", "MOBILE_3"),
|
||||||
signal_name="contentFormat",
|
signal_name="contentFormat",
|
||||||
),
|
),
|
||||||
select_field(
|
select_field(
|
||||||
label="Content type",
|
label="Content type",
|
||||||
field_id="content-type",
|
field_id="content-type",
|
||||||
options=PANGEA_CONTENT_TYPES,
|
options=PANGEA_CONTENT_TYPES,
|
||||||
selected="articles",
|
selected=_value(source, "content_type", "articles"),
|
||||||
signal_name="contentType",
|
signal_name="contentType",
|
||||||
),
|
),
|
||||||
input_field(
|
input_field(
|
||||||
label="Max articles",
|
label="Max articles",
|
||||||
field_id="max-articles",
|
field_id="max-articles",
|
||||||
value="10",
|
value=_value(source, "max_articles", "10"),
|
||||||
signal_name="maxArticles",
|
signal_name="maxArticles",
|
||||||
),
|
),
|
||||||
input_field(
|
input_field(
|
||||||
label="Oldest article (days)",
|
label="Oldest article (days)",
|
||||||
field_id="oldest-article",
|
field_id="oldest-article",
|
||||||
value="3",
|
value=_value(source, "oldest_article", "3"),
|
||||||
signal_name="oldestArticle",
|
signal_name="oldestArticle",
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
|
|
@ -248,25 +289,25 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
label="Only newest",
|
label="Only newest",
|
||||||
description="Limit Pangea syncs to the newest material available in the selected category.",
|
description="Limit Pangea syncs to the newest material available in the selected category.",
|
||||||
signal_name="onlyNewest",
|
signal_name="onlyNewest",
|
||||||
checked=True,
|
checked=_checked(source, "only_newest", True),
|
||||||
),
|
),
|
||||||
toggle_field(
|
toggle_field(
|
||||||
label="Include authors",
|
label="Include authors",
|
||||||
description="Carry author bylines into mirrored output where upstream data exists.",
|
description="Carry author bylines into mirrored output where upstream data exists.",
|
||||||
signal_name="includeAuthors",
|
signal_name="includeAuthors",
|
||||||
checked=True,
|
checked=_checked(source, "include_authors", True),
|
||||||
),
|
),
|
||||||
toggle_field(
|
toggle_field(
|
||||||
label="Exclude media",
|
label="Exclude media",
|
||||||
description="Skip image and media attachment mirroring for this source.",
|
description="Skip image and media attachment mirroring for this source.",
|
||||||
signal_name="excludeMedia",
|
signal_name="excludeMedia",
|
||||||
checked=False,
|
checked=_checked(source, "exclude_media", False),
|
||||||
),
|
),
|
||||||
toggle_field(
|
toggle_field(
|
||||||
label="Include content",
|
label="Include content",
|
||||||
description="Store article body content in mirrored output when the upstream provides it.",
|
description="Store article body content in mirrored output when the upstream provides it.",
|
||||||
signal_name="includeContent",
|
signal_name="includeContent",
|
||||||
checked=True,
|
checked=_checked(source, "include_content", True),
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
|
|
@ -274,13 +315,17 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
textarea_field(
|
textarea_field(
|
||||||
label="Notes",
|
label="Notes",
|
||||||
field_id="source-notes",
|
field_id="source-notes",
|
||||||
value="",
|
value=_value(source, "notes"),
|
||||||
signal_name="sourceNotes",
|
signal_name="sourceNotes",
|
||||||
),
|
),
|
||||||
textarea_field(
|
textarea_field(
|
||||||
label="Spider arguments",
|
label="Spider arguments",
|
||||||
field_id="spider-arguments",
|
field_id="spider-arguments",
|
||||||
value="language=en\ndownload_media=true",
|
value=_value(
|
||||||
|
source,
|
||||||
|
"spider_arguments",
|
||||||
|
"language=en\ndownload_media=true",
|
||||||
|
),
|
||||||
signal_name="spiderArguments",
|
signal_name="spiderArguments",
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
|
|
@ -300,31 +345,31 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
input_field(
|
input_field(
|
||||||
label="Minute",
|
label="Minute",
|
||||||
field_id="cron-minute",
|
field_id="cron-minute",
|
||||||
value="*/30",
|
value=_value(source, "cron_minute", "*/30"),
|
||||||
signal_name="cronMinute",
|
signal_name="cronMinute",
|
||||||
),
|
),
|
||||||
input_field(
|
input_field(
|
||||||
label="Hour",
|
label="Hour",
|
||||||
field_id="cron-hour",
|
field_id="cron-hour",
|
||||||
value="*",
|
value=_value(source, "cron_hour", "*"),
|
||||||
signal_name="cronHour",
|
signal_name="cronHour",
|
||||||
),
|
),
|
||||||
input_field(
|
input_field(
|
||||||
label="Day of month",
|
label="Day of month",
|
||||||
field_id="cron-day-of-month",
|
field_id="cron-day-of-month",
|
||||||
value="*",
|
value=_value(source, "cron_day_of_month", "*"),
|
||||||
signal_name="cronDayOfMonth",
|
signal_name="cronDayOfMonth",
|
||||||
),
|
),
|
||||||
input_field(
|
input_field(
|
||||||
label="Day of week",
|
label="Day of week",
|
||||||
field_id="cron-day-of-week",
|
field_id="cron-day-of-week",
|
||||||
value="*",
|
value=_value(source, "cron_day_of_week", "*"),
|
||||||
signal_name="cronDayOfWeek",
|
signal_name="cronDayOfWeek",
|
||||||
),
|
),
|
||||||
input_field(
|
input_field(
|
||||||
label="Month",
|
label="Month",
|
||||||
field_id="cron-month",
|
field_id="cron-month",
|
||||||
value="*",
|
value=_value(source, "cron_month", "*"),
|
||||||
signal_name="cronMonth",
|
signal_name="cronMonth",
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
|
|
@ -341,7 +386,7 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
label="Job enabled",
|
label="Job enabled",
|
||||||
description="Scheduler will consider the new job immediately after creation.",
|
description="Scheduler will consider the new job immediately after creation.",
|
||||||
signal_name="jobEnabled",
|
signal_name="jobEnabled",
|
||||||
checked=True,
|
checked=_checked(source, "enabled", True),
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
|
|
@ -353,7 +398,7 @@ def create_source_form(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
h.button(
|
h.button(
|
||||||
type="submit",
|
type="submit",
|
||||||
class_="rounded-full bg-slate-950 px-4 py-2.5 text-sm font-semibold text-white transition hover:bg-slate-800",
|
class_="rounded-full bg-slate-950 px-4 py-2.5 text-sm font-semibold text-white transition hover:bg-slate-800",
|
||||||
)["Create source"],
|
)[submit_label],
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
@ -369,7 +414,27 @@ def create_source_page(*, action_path: str = "/actions/sources/create") -> Rende
|
||||||
current_path="/sources/create",
|
current_path="/sources/create",
|
||||||
eyebrow="Source creation",
|
eyebrow="Source creation",
|
||||||
title="Create source",
|
title="Create source",
|
||||||
description="Dedicated create page for the source form. The list page stays focused on scanning existing sources, while this page handles the new source and job configuration flow.",
|
description="Create a new source and its paired job configuration.",
|
||||||
actions=actions,
|
actions=actions,
|
||||||
content=create_source_form(action_path=action_path),
|
content=source_form(mode="create", action_path=action_path),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def edit_source_page(
|
||||||
|
*,
|
||||||
|
slug: str,
|
||||||
|
source: Mapping[str, object],
|
||||||
|
action_path: str,
|
||||||
|
) -> Renderable:
|
||||||
|
actions = (
|
||||||
|
muted_action_link(href="/sources", label="Back to sources"),
|
||||||
|
header_action_link(href="/runs", label="View runs"),
|
||||||
|
)
|
||||||
|
return page_shell(
|
||||||
|
current_path=f"/sources/{slug}/edit",
|
||||||
|
eyebrow="Source editing",
|
||||||
|
title="Edit source",
|
||||||
|
description="Update an existing source and its paired job configuration.",
|
||||||
|
actions=actions,
|
||||||
|
content=source_form(mode="edit", action_path=action_path, source=source),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
52
repub/web.py
52
repub/web.py
|
|
@ -18,12 +18,15 @@ from repub.datastar import RefreshBroker, render_stream
|
||||||
from repub.model import (
|
from repub.model import (
|
||||||
create_source,
|
create_source,
|
||||||
initialize_database,
|
initialize_database,
|
||||||
|
load_source_form,
|
||||||
load_sources,
|
load_sources,
|
||||||
source_slug_exists,
|
source_slug_exists,
|
||||||
|
update_source,
|
||||||
)
|
)
|
||||||
from repub.pages import (
|
from repub.pages import (
|
||||||
create_source_page,
|
create_source_page,
|
||||||
dashboard_page,
|
dashboard_page,
|
||||||
|
edit_source_page,
|
||||||
execution_logs_page,
|
execution_logs_page,
|
||||||
runs_page,
|
runs_page,
|
||||||
shim_page,
|
shim_page,
|
||||||
|
|
@ -85,12 +88,15 @@ def create_app() -> Quart:
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
@app.get("/sources")
|
@app.get("/sources")
|
||||||
@app.get("/sources/create")
|
@app.get("/sources/create")
|
||||||
|
@app.get("/sources/<string:slug>/edit")
|
||||||
@app.get("/runs")
|
@app.get("/runs")
|
||||||
@app.get("/job/<int:job_id>/execution/<int:execution_id>/logs")
|
@app.get("/job/<int:job_id>/execution/<int:execution_id>/logs")
|
||||||
async def page_shim(
|
async def page_shim(
|
||||||
job_id: int | None = None, execution_id: int | None = None
|
slug: str | None = None,
|
||||||
|
job_id: int | None = None,
|
||||||
|
execution_id: int | None = None,
|
||||||
) -> Response:
|
) -> Response:
|
||||||
del job_id, execution_id
|
del slug, job_id, execution_id
|
||||||
body, etag = _render_shim_page(
|
body, etag = _render_shim_page(
|
||||||
stylesheet_href=url_for("static", filename="app.css"),
|
stylesheet_href=url_for("static", filename="app.css"),
|
||||||
datastar_src=url_for("static", filename="datastar@1.0.0-RC.8.js"),
|
datastar_src=url_for("static", filename="datastar@1.0.0-RC.8.js"),
|
||||||
|
|
@ -116,6 +122,10 @@ def create_app() -> Quart:
|
||||||
async def create_source_patch() -> DatastarResponse:
|
async def create_source_patch() -> DatastarResponse:
|
||||||
return _page_patch_response(app, lambda: render_create_source(app))
|
return _page_patch_response(app, lambda: render_create_source(app))
|
||||||
|
|
||||||
|
@app.post("/sources/<string:slug>/edit")
|
||||||
|
async def edit_source_patch(slug: str) -> DatastarResponse:
|
||||||
|
return _page_patch_response(app, lambda: render_edit_source(slug))
|
||||||
|
|
||||||
@app.post("/actions/sources/create")
|
@app.post("/actions/sources/create")
|
||||||
async def create_source_action() -> DatastarResponse:
|
async def create_source_action() -> DatastarResponse:
|
||||||
signals = cast(dict[str, object], await read_signals())
|
signals = cast(dict[str, object], await read_signals())
|
||||||
|
|
@ -140,6 +150,30 @@ def create_app() -> Quart:
|
||||||
trigger_refresh(app)
|
trigger_refresh(app)
|
||||||
return DatastarResponse(SSE.redirect("/sources"))
|
return DatastarResponse(SSE.redirect("/sources"))
|
||||||
|
|
||||||
|
@app.post("/actions/sources/<string:slug>/edit")
|
||||||
|
async def edit_source_action(slug: str) -> DatastarResponse:
|
||||||
|
signals = cast(dict[str, object], await read_signals())
|
||||||
|
source, error = validate_source_form(
|
||||||
|
signals,
|
||||||
|
slug_exists=lambda candidate: candidate != slug
|
||||||
|
and source_slug_exists(candidate),
|
||||||
|
immutable_slug=slug,
|
||||||
|
)
|
||||||
|
if error is not None:
|
||||||
|
return DatastarResponse(
|
||||||
|
SSE.patch_signals({"_formError": error, "_formSuccess": ""})
|
||||||
|
)
|
||||||
|
|
||||||
|
assert source is not None
|
||||||
|
if update_source(slug, **source) is None:
|
||||||
|
return DatastarResponse(
|
||||||
|
SSE.patch_signals(
|
||||||
|
{"_formError": "Source does not exist.", "_formSuccess": ""}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
trigger_refresh(app)
|
||||||
|
return DatastarResponse(SSE.redirect("/sources"))
|
||||||
|
|
||||||
@app.post("/runs")
|
@app.post("/runs")
|
||||||
async def runs_patch() -> DatastarResponse:
|
async def runs_patch() -> DatastarResponse:
|
||||||
return _page_patch_response(app, render_runs)
|
return _page_patch_response(app, render_runs)
|
||||||
|
|
@ -176,6 +210,17 @@ async def render_create_source(app: Quart | None = None) -> Renderable:
|
||||||
return create_source_page()
|
return create_source_page()
|
||||||
|
|
||||||
|
|
||||||
|
async def render_edit_source(slug: str) -> Renderable:
|
||||||
|
source = load_source_form(slug)
|
||||||
|
if source is None:
|
||||||
|
return sources_page(sources=())
|
||||||
|
return edit_source_page(
|
||||||
|
slug=slug,
|
||||||
|
source=source,
|
||||||
|
action_path=f"/actions/sources/{slug}/edit",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def render_runs() -> Renderable:
|
async def render_runs() -> Renderable:
|
||||||
return runs_page()
|
return runs_page()
|
||||||
|
|
||||||
|
|
@ -208,6 +253,7 @@ def validate_source_form(
|
||||||
signals: dict[str, object] | None,
|
signals: dict[str, object] | None,
|
||||||
*,
|
*,
|
||||||
slug_exists: Callable[[str], bool],
|
slug_exists: Callable[[str], bool],
|
||||||
|
immutable_slug: str | None = None,
|
||||||
) -> tuple[SourceFormData | None, str | None]:
|
) -> tuple[SourceFormData | None, str | None]:
|
||||||
if signals is None:
|
if signals is None:
|
||||||
return None, "Missing form data."
|
return None, "Missing form data."
|
||||||
|
|
@ -235,6 +281,8 @@ def validate_source_form(
|
||||||
errors.append("Source name is required.")
|
errors.append("Source name is required.")
|
||||||
if source_slug == "":
|
if source_slug == "":
|
||||||
errors.append("Slug is required.")
|
errors.append("Slug is required.")
|
||||||
|
elif immutable_slug is not None and source_slug != immutable_slug:
|
||||||
|
errors.append("Slug is immutable.")
|
||||||
elif slug_exists(source_slug):
|
elif slug_exists(source_slug):
|
||||||
errors.append("Slug must be unique.")
|
errors.append("Slug must be unique.")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,12 +5,13 @@ from pathlib import Path
|
||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
|
|
||||||
from repub.datastar import RefreshBroker, render_sse_event, render_stream
|
from repub.datastar import RefreshBroker, render_sse_event, render_stream
|
||||||
from repub.model import Job, Source, SourceFeed, SourcePangea
|
from repub.model import Job, Source, SourceFeed, SourcePangea, create_source
|
||||||
from repub.web import (
|
from repub.web import (
|
||||||
create_app,
|
create_app,
|
||||||
get_refresh_broker,
|
get_refresh_broker,
|
||||||
render_create_source,
|
render_create_source,
|
||||||
render_dashboard,
|
render_dashboard,
|
||||||
|
render_edit_source,
|
||||||
render_execution_logs,
|
render_execution_logs,
|
||||||
render_runs,
|
render_runs,
|
||||||
render_sources,
|
render_sources,
|
||||||
|
|
@ -171,7 +172,7 @@ def test_render_create_source_shows_dedicated_form_page() -> None:
|
||||||
async def run() -> None:
|
async def run() -> None:
|
||||||
body = str(await render_create_source())
|
body = str(await render_create_source())
|
||||||
|
|
||||||
assert "Dedicated create page for the source form" in body
|
assert "Create a new source and its paired job configuration." in body
|
||||||
assert "Source and job setup" in body
|
assert "Source and job setup" in body
|
||||||
assert "data-signals__ifmissing" in body
|
assert "data-signals__ifmissing" in body
|
||||||
assert "/actions/sources/create" in body
|
assert "/actions/sources/create" in body
|
||||||
|
|
@ -206,6 +207,55 @@ def test_render_create_source_shows_dedicated_form_page() -> None:
|
||||||
asyncio.run(run())
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_edit_source_shows_existing_values(monkeypatch, tmp_path: Path) -> None:
|
||||||
|
db_path = tmp_path / "edit-page.db"
|
||||||
|
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
|
||||||
|
create_app()
|
||||||
|
create_source(
|
||||||
|
name="Kenya health desk",
|
||||||
|
slug="kenya-health",
|
||||||
|
source_type="pangea",
|
||||||
|
notes="Regional health alerts.",
|
||||||
|
spider_arguments="language=en\ndownload_media=true",
|
||||||
|
enabled=True,
|
||||||
|
cron_minute="0",
|
||||||
|
cron_hour="*/6",
|
||||||
|
cron_day_of_month="*",
|
||||||
|
cron_day_of_week="*",
|
||||||
|
cron_month="*",
|
||||||
|
pangea_domain="example.org",
|
||||||
|
pangea_category="Health",
|
||||||
|
content_type="breakingnews",
|
||||||
|
only_newest=True,
|
||||||
|
max_articles=12,
|
||||||
|
oldest_article=5,
|
||||||
|
include_authors=True,
|
||||||
|
exclude_media=False,
|
||||||
|
include_content=True,
|
||||||
|
content_format="MOBILE_3",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def run() -> None:
|
||||||
|
body = str(await render_edit_source("kenya-health"))
|
||||||
|
|
||||||
|
assert "Edit source" in body
|
||||||
|
assert "/actions/sources/kenya-health/edit" in body
|
||||||
|
assert "Kenya health desk" in body
|
||||||
|
assert "kenya-health" in body
|
||||||
|
assert 'id="source-slug"' in body
|
||||||
|
assert (
|
||||||
|
'id="source-slug" name="source-slug" type="text" value="kenya-health"'
|
||||||
|
in body
|
||||||
|
)
|
||||||
|
assert " disabled " in body
|
||||||
|
assert "cursor-not-allowed bg-slate-100 text-slate-500" in body
|
||||||
|
assert "example.org" in body
|
||||||
|
assert "Health" in body
|
||||||
|
assert "language=en\ndownload_media=true" in body
|
||||||
|
|
||||||
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
def test_create_source_action_creates_pangea_source_and_job_in_database(
|
def test_create_source_action_creates_pangea_source_and_job_in_database(
|
||||||
monkeypatch, tmp_path: Path
|
monkeypatch, tmp_path: Path
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
@ -314,6 +364,163 @@ def test_create_source_action_creates_feed_source_and_job_in_database(
|
||||||
asyncio.run(run())
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
|
def test_edit_source_action_updates_existing_source_and_job_in_database(
|
||||||
|
monkeypatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
db_path = tmp_path / "edit-source.db"
|
||||||
|
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
|
||||||
|
create_app()
|
||||||
|
create_source(
|
||||||
|
name="Kenya health desk",
|
||||||
|
slug="kenya-health",
|
||||||
|
source_type="pangea",
|
||||||
|
notes="Regional health alerts.",
|
||||||
|
spider_arguments="language=en\ndownload_media=true",
|
||||||
|
enabled=True,
|
||||||
|
cron_minute="0",
|
||||||
|
cron_hour="*/6",
|
||||||
|
cron_day_of_month="*",
|
||||||
|
cron_day_of_week="*",
|
||||||
|
cron_month="*",
|
||||||
|
pangea_domain="example.org",
|
||||||
|
pangea_category="Health",
|
||||||
|
content_type="breakingnews",
|
||||||
|
only_newest=True,
|
||||||
|
max_articles=12,
|
||||||
|
oldest_article=5,
|
||||||
|
include_authors=True,
|
||||||
|
exclude_media=False,
|
||||||
|
include_content=True,
|
||||||
|
content_format="MOBILE_3",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def run() -> None:
|
||||||
|
app = create_app()
|
||||||
|
client = app.test_client()
|
||||||
|
|
||||||
|
response = await client.post(
|
||||||
|
"/actions/sources/kenya-health/edit",
|
||||||
|
headers={"Datastar-Request": "true"},
|
||||||
|
json={
|
||||||
|
"sourceName": "Kenya health desk nightly",
|
||||||
|
"sourceSlug": "kenya-health",
|
||||||
|
"sourceType": "pangea",
|
||||||
|
"pangeaDomain": "example.org",
|
||||||
|
"pangeaCategory": "Nightly",
|
||||||
|
"contentFormat": "TEXT_ONLY",
|
||||||
|
"contentType": "articles",
|
||||||
|
"maxArticles": "25",
|
||||||
|
"oldestArticle": "7",
|
||||||
|
"sourceNotes": "Updated nightly run.",
|
||||||
|
"spiderArguments": "language=sw\ninclude_audio=false",
|
||||||
|
"cronMinute": "15",
|
||||||
|
"cronHour": "2",
|
||||||
|
"cronDayOfMonth": "*",
|
||||||
|
"cronDayOfWeek": "*",
|
||||||
|
"cronMonth": "*",
|
||||||
|
"jobEnabled": False,
|
||||||
|
"onlyNewest": False,
|
||||||
|
"includeAuthors": False,
|
||||||
|
"excludeMedia": True,
|
||||||
|
"includeContent": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
body = await response.get_data(as_text=True)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert "window.location = '/sources'" in body
|
||||||
|
|
||||||
|
source = Source.get(Source.slug == "kenya-health")
|
||||||
|
pangea = SourcePangea.get(SourcePangea.source == source)
|
||||||
|
job = Job.get(Job.source == source)
|
||||||
|
rendered_sources = str(await render_sources(app))
|
||||||
|
|
||||||
|
assert source.name == "Kenya health desk nightly"
|
||||||
|
assert source.notes == "Updated nightly run."
|
||||||
|
assert pangea.category_name == "Nightly"
|
||||||
|
assert pangea.content_format == "TEXT_ONLY"
|
||||||
|
assert pangea.max_articles == 25
|
||||||
|
assert pangea.include_authors is False
|
||||||
|
assert pangea.exclude_media is True
|
||||||
|
assert job.enabled is False
|
||||||
|
assert job.spider_arguments == "language=sw\ninclude_audio=false"
|
||||||
|
assert job.cron_hour == "2"
|
||||||
|
assert "Kenya health desk nightly" in rendered_sources
|
||||||
|
assert "example.org / Nightly" in rendered_sources
|
||||||
|
assert "Disabled" in rendered_sources
|
||||||
|
|
||||||
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
|
def test_edit_source_action_rejects_slug_changes(monkeypatch, tmp_path: Path) -> None:
|
||||||
|
db_path = tmp_path / "edit-invalid.db"
|
||||||
|
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
|
||||||
|
create_app()
|
||||||
|
create_source(
|
||||||
|
name="Kenya health desk",
|
||||||
|
slug="kenya-health",
|
||||||
|
source_type="pangea",
|
||||||
|
notes="Regional health alerts.",
|
||||||
|
spider_arguments="language=en\ndownload_media=true",
|
||||||
|
enabled=True,
|
||||||
|
cron_minute="0",
|
||||||
|
cron_hour="*/6",
|
||||||
|
cron_day_of_month="*",
|
||||||
|
cron_day_of_week="*",
|
||||||
|
cron_month="*",
|
||||||
|
pangea_domain="example.org",
|
||||||
|
pangea_category="Health",
|
||||||
|
content_type="breakingnews",
|
||||||
|
only_newest=True,
|
||||||
|
max_articles=12,
|
||||||
|
oldest_article=5,
|
||||||
|
include_authors=True,
|
||||||
|
exclude_media=False,
|
||||||
|
include_content=True,
|
||||||
|
content_format="MOBILE_3",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def run() -> None:
|
||||||
|
app = create_app()
|
||||||
|
client = app.test_client()
|
||||||
|
|
||||||
|
response = await client.post(
|
||||||
|
"/actions/sources/kenya-health/edit",
|
||||||
|
headers={"Datastar-Request": "true"},
|
||||||
|
json={
|
||||||
|
"sourceName": "Kenya health desk",
|
||||||
|
"sourceSlug": "kenya-health-renamed",
|
||||||
|
"sourceType": "pangea",
|
||||||
|
"pangeaDomain": "example.org",
|
||||||
|
"pangeaCategory": "Health",
|
||||||
|
"contentFormat": "MOBILE_3",
|
||||||
|
"contentType": "breakingnews",
|
||||||
|
"maxArticles": "12",
|
||||||
|
"oldestArticle": "5",
|
||||||
|
"sourceNotes": "Regional health alerts.",
|
||||||
|
"spiderArguments": "language=en\ndownload_media=true",
|
||||||
|
"cronMinute": "0",
|
||||||
|
"cronHour": "*/6",
|
||||||
|
"cronDayOfMonth": "*",
|
||||||
|
"cronDayOfWeek": "*",
|
||||||
|
"cronMonth": "*",
|
||||||
|
"jobEnabled": True,
|
||||||
|
"onlyNewest": True,
|
||||||
|
"includeAuthors": True,
|
||||||
|
"excludeMedia": False,
|
||||||
|
"includeContent": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
body = await response.get_data(as_text=True)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert "Slug is immutable." in body
|
||||||
|
assert Source.get(Source.slug == "kenya-health").name == "Kenya health desk"
|
||||||
|
assert Source.select().where(Source.slug == "kenya-health-renamed").count() == 0
|
||||||
|
|
||||||
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
def test_create_source_action_validates_duplicate_slug_and_pangea_type(
|
def test_create_source_action_validates_duplicate_slug_and_pangea_type(
|
||||||
monkeypatch, tmp_path: Path
|
monkeypatch, tmp_path: Path
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue