anynews/republisher
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add settings and live sidebar counts

Browse source
This commit is contained in:
Abel Luck 2026-03-30 18:26:02 +02:00
parent 2a99edeec3
commit a809bde16c
16 changed files with 696 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

32
tests/test_config.py
View file

@ -195,3 +195,35 @@ def test_build_feed_settings_uses_runtime_media_dir_overrides(tmp_path: Path) ->
assert feed_settings["AUDIO_STORE"] == str(
out_dir / "feeds" / "gp-pod" / "audio-custom"
)
def test_build_feed_settings_can_disable_image_and_video_conversion(
tmp_path: Path,
) -> None:
out_dir = (tmp_path / "mirror").resolve()
config = RepublisherConfig(
config_path=tmp_path / "repub.toml",
out_dir=out_dir,
feeds=(
FeedConfig(
name="Guardian Project Podcast",
slug="gp-pod",
url="https://guardianproject.info/podcast/podcast.xml",
),
),
scrapy_settings={},
)
base_settings = build_base_settings(config)
feed_settings = build_feed_settings(
base_settings,
out_dir=out_dir,
feed_slug="gp-pod",
convert_images=False,
convert_video=False,
)
assert "repub.pipelines.ImagePipeline" not in feed_settings["ITEM_PIPELINES"]
assert "repub.pipelines.VideoPipeline" not in feed_settings["ITEM_PIPELINES"]
assert feed_settings["ITEM_PIPELINES"]["repub.pipelines.AudioPipeline"] == 2
assert feed_settings["ITEM_PIPELINES"]["repub.pipelines.FilePipeline"] == 4

34
tests/test_model.py
View file

@ -7,11 +7,14 @@ import pytest
from peewee import IntegrityError
from repub.model import (
AppSetting,
Job,
Source,
database,
initialize_database,
load_max_concurrent_jobs,
resolve_database_path,
save_setting,
)
@ -51,6 +54,7 @@ def test_initialize_database_bootstraps_schema_from_sql_files(tmp_path: Path) ->
)
}
assert table_names == {
"app_setting",
"job",
"job_execution",
"source",
@ -59,17 +63,10 @@ def test_initialize_database_bootstraps_schema_from_sql_files(tmp_path: Path) ->
}
defaults = {
row[1]: row[4]
for row in connection.execute("PRAGMA table_info('source_pangea')")
row[1]: row[4] for row in connection.execute("PRAGMA table_info('job')")
}
assert defaults["content_type"] is None
assert defaults["only_newest"] is None
assert defaults["max_articles"] is None
assert defaults["oldest_article"] is None
assert defaults["include_authors"] is None
assert defaults["exclude_media"] is None
assert defaults["include_content"] is None
assert defaults["content_format"] is None
assert defaults["convert_images"] == "1"
assert defaults["convert_video"] == "1"
finally:
connection.close()
@ -168,3 +165,20 @@ def test_job_table_allows_exactly_one_job_per_source(tmp_path: Path) -> None:
cron_day_of_week="*",
cron_month="*",
)
def test_load_max_concurrent_jobs_defaults_to_one(tmp_path: Path) -> None:
initialize_database(tmp_path / "settings-defaults.db")
assert load_max_concurrent_jobs() == 1
def test_save_setting_persists_json_value(tmp_path: Path) -> None:
initialize_database(tmp_path / "settings-roundtrip.db")
save_setting("max_concurrent_jobs", 4)
row = AppSetting.get(AppSetting.key == "max_concurrent_jobs")
assert row.value == "4"
assert load_max_concurrent_jobs() == 4

63
tests/test_scheduler_runtime.py
View file

@ -20,6 +20,7 @@ from repub.model import (
Source,
create_source,
initialize_database,
save_setting,
)
from repub.web import create_app, get_job_runtime, render_execution_logs, render_runs
@ -137,6 +138,68 @@ def test_job_runtime_run_now_writes_log_and_stats_and_marks_success(
runtime.shutdown()
def test_job_runtime_respects_max_concurrent_jobs_setting(tmp_path: Path) -> None:
db_path = tmp_path / "max-concurrency.db"
log_dir = tmp_path / "out" / "logs"
initialize_database(db_path)
save_setting("max_concurrent_jobs", 1)
with _slow_feed_server() as feed_url:
first_source = create_source(
name="First source",
slug="first-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=False,
cron_minute="*/5",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url=feed_url,
)
second_source = create_source(
name="Second source",
slug="second-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=False,
cron_minute="*/5",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url=feed_url,
)
first_job = Job.get(Job.source == first_source)
second_job = Job.get(Job.source == second_source)
runtime = JobRuntime(log_dir=log_dir)
try:
runtime.start()
first_execution_id = runtime.run_job_now(first_job.id, reason="manual")
assert first_execution_id is not None
_wait_for_running_execution(first_execution_id)
second_execution_id = runtime.run_job_now(second_job.id, reason="manual")
assert second_execution_id is None
assert (
JobExecution.select()
.where(JobExecution.running_status == JobExecutionStatus.RUNNING)
.count()
== 1
)
runtime.request_execution_cancel(first_execution_id)
finished_execution = _wait_for_terminal_execution(first_execution_id)
assert finished_execution.running_status == JobExecutionStatus.CANCELED
finally:
runtime.shutdown()
def test_job_runtime_cancel_marks_execution_canceled(tmp_path: Path) -> None:
initialize_database(tmp_path / "cancel.db")
with _slow_feed_server() as feed_url:

174
tests/test_web.py
View file

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import os
import re
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, cast
@ -17,6 +18,8 @@ from repub.model import (
SourceFeed,
SourcePangea,
create_source,
load_max_concurrent_jobs,
save_setting,
)
from repub.pages.runs import runs_page
from repub.web import (
@ -27,6 +30,7 @@ from repub.web import (
render_edit_source,
render_execution_logs,
render_runs,
render_settings,
render_sources,
)
@ -109,6 +113,7 @@ def test_root_get_serves_datastar_shim() -> None:
assert '<main id="morph"' in body
assert 'href="/sources"' in body
assert 'href="/runs"' in body
assert 'href="/settings"' in body
assert "Connecting" in body
asyncio.run(run())
@ -430,6 +435,94 @@ def test_render_sources_shows_table_and_create_link() -> None:
asyncio.run(run())
def test_render_sources_shows_live_sidebar_badges(monkeypatch, tmp_path: Path) -> None:
db_path = tmp_path / "sources-sidebar.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
app = create_app()
create_source(
name="First source",
slug="first-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=True,
cron_minute="0",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url="https://example.com/first.xml",
)
create_source(
name="Second source",
slug="second-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=True,
cron_minute="0",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url="https://example.com/second.xml",
)
async def run() -> None:
body = str(await render_sources(app))
assert re.search(
r'href="/sources"[^>]*>.*?<span>Sources</span>\s*<span[^>]*>2</span>',
body,
re.S,
)
assert re.search(
r'href="/runs"[^>]*>.*?<span>Runs</span>\s*<span[^>]*>0</span>',
body,
re.S,
)
asyncio.run(run())
def test_render_dashboard_shows_live_sidebar_badges(
monkeypatch, tmp_path: Path
) -> None:
db_path = tmp_path / "dashboard-sidebar.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
app = create_app()
create_source(
name="Dashboard source",
slug="dashboard-source",
source_type="feed",
notes="",
spider_arguments="",
enabled=True,
cron_minute="0",
cron_hour="*",
cron_day_of_month="*",
cron_day_of_week="*",
cron_month="*",
feed_url="https://example.com/dashboard.xml",
)
async def run() -> None:
body = str(await render_dashboard(app))
assert re.search(
r'href="/sources"[^>]*>.*?<span>Sources</span>\s*<span[^>]*>1</span>',
body,
re.S,
)
assert re.search(
r'href="/runs"[^>]*>.*?<span>Runs</span>\s*<span[^>]*>0</span>',
body,
re.S,
)
asyncio.run(run())
def test_render_sources_shows_delete_action_for_each_source(
monkeypatch, tmp_path: Path
) -> None:
@ -476,6 +569,8 @@ def test_render_create_source_shows_dedicated_form_page() -> None:
assert "includeAuthors" in body
assert "excludeMedia" in body
assert "includeContent" in body
assert "convertImages" in body
assert "convertVideo" in body
assert "TEXT_ONLY" in body
assert "breakingnews" in body
assert "Pangea domain" in body
@ -512,6 +607,8 @@ def test_render_edit_source_shows_existing_values(monkeypatch, tmp_path: Path) -
notes="Regional health alerts.",
spider_arguments="language=en\ndownload_media=true",
enabled=True,
convert_images=False,
convert_video=False,
cron_minute="0",
cron_hour="*/6",
cron_day_of_month="*",
@ -546,6 +643,28 @@ def test_render_edit_source_shows_existing_values(monkeypatch, tmp_path: Path) -
assert "example.org" in body
assert "Health" in body
assert "language=en\ndownload_media=true" in body
assert "convertImages: false" in body
assert "convertVideo: false" in body
asyncio.run(run())
def test_render_settings_shows_current_max_concurrent_jobs(
monkeypatch, tmp_path: Path
) -> None:
db_path = tmp_path / "settings-page.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
create_app()
save_setting("max_concurrent_jobs", 3)
async def run() -> None:
app = create_app()
body = str(await render_settings(app))
assert ">Settings<" in body
assert "/actions/settings" in body
assert 'value="3"' in body
assert "Max concurrent jobs" in body
asyncio.run(run())
@ -602,6 +721,8 @@ def test_create_source_action_creates_pangea_source_and_job_in_database(
assert pangea.content_type == "breakingnews"
assert pangea.include_content is True
assert job.enabled is True
assert job.convert_images is True
assert job.convert_video is True
assert job.spider_arguments == "language=en\ndownload_media=true"
assert job.cron_hour == "*/6"
assert "kenya-health" in rendered_sources
@ -713,6 +834,8 @@ def test_edit_source_action_updates_existing_source_and_job_in_database(
"cronDayOfWeek": "*",
"cronMonth": "*",
"jobEnabled": False,
"convertImages": False,
"convertVideo": False,
"onlyNewest": False,
"includeAuthors": False,
"excludeMedia": True,
@ -737,6 +860,8 @@ def test_edit_source_action_updates_existing_source_and_job_in_database(
assert pangea.include_authors is False
assert pangea.exclude_media is True
assert job.enabled is False
assert job.convert_images is False
assert job.convert_video is False
assert job.spider_arguments == "language=sw\ninclude_audio=false"
assert job.cron_hour == "2"
assert "Kenya health desk nightly" in rendered_sources
@ -863,6 +988,55 @@ def test_create_source_action_validates_duplicate_slug_and_pangea_type(
asyncio.run(run())
def test_settings_action_updates_max_concurrent_jobs(
monkeypatch, tmp_path: Path
) -> None:
db_path = tmp_path / "settings-action.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
async def run() -> None:
app = create_app()
client = app.test_client()
response = await client.post(
"/actions/settings",
headers={"Datastar-Request": "true"},
json={"maxConcurrentJobs": "3"},
)
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert "window.location = '/settings'" in body
assert load_max_concurrent_jobs() == 3
assert 'value="3"' in str(await render_settings(app))
asyncio.run(run())
def test_settings_action_rejects_non_positive_max_concurrent_jobs(
monkeypatch, tmp_path: Path
) -> None:
db_path = tmp_path / "settings-invalid.db"
monkeypatch.setenv("REPUBLISHER_DB_PATH", str(db_path))
async def run() -> None:
app = create_app()
client = app.test_client()
response = await client.post(
"/actions/settings",
headers={"Datastar-Request": "true"},
json={"maxConcurrentJobs": "0"},
)
body = await response.get_data(as_text=True)
assert response.status_code == 200
assert "Max concurrent jobs must be at least 1." in body
assert load_max_concurrent_jobs() == 1
asyncio.run(run())
def test_render_runs_shows_running_upcoming_and_completed_tables(
monkeypatch, tmp_path: Path
) -> None: