anynews/republisher
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add settings and live sidebar counts

Browse source
This commit is contained in:
Abel Luck 2026-03-30 18:26:02 +02:00
parent 2a99edeec3
commit a809bde16c
16 changed files with 696 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

25
repub/job_runner.py
View file

@ -186,6 +186,8 @@ class JobSourceConfig:
source_slug: str
source_type: str
spider_arguments: dict[str, str]
convert_images: bool = True
convert_video: bool = True
feed_url: str | None = None
pangea_domain: str | None = None
pangea_category: str | None = None
@ -267,6 +269,8 @@ def main(argv: list[str] | None = None) -> int:
out_dir=out_dir,
feed=feed,
stats_path=stats_path,
convert_images=source_config.convert_images,
convert_video=source_config.convert_video,
)
)
print(
@ -326,6 +330,8 @@ def _load_job_source_config(*, db_path: str, job_id: int) -> JobSourceConfig:
source_slug=source.slug,
source_type=source.source_type,
spider_arguments=spider_arguments,
convert_images=bool(job.convert_images),
convert_video=bool(job.convert_video),
feed_url=feed.feed_url,
)
@ -339,6 +345,8 @@ def _load_job_source_config(*, db_path: str, job_id: int) -> JobSourceConfig:
source_slug=source.slug,
source_type=source.source_type,
spider_arguments=spider_arguments,
convert_images=bool(job.convert_images),
convert_video=bool(job.convert_video),
pangea_domain=pangea.domain,
pangea_category=pangea.category_name,
content_type=pangea.content_type,
@ -409,7 +417,14 @@ def _resolve_feed(
)
def _build_crawl_settings(*, out_dir: Path, feed: FeedConfig, stats_path: Path):
def _build_crawl_settings(
*,
out_dir: Path,
feed: FeedConfig,
stats_path: Path,
convert_images: bool = True,
convert_video: bool = True,
):
base_settings = build_base_settings(
RepublisherConfig(
config_path=out_dir / "job-runner.toml",
@ -419,7 +434,13 @@ def _build_crawl_settings(*, out_dir: Path, feed: FeedConfig, stats_path: Path):
)
)
prepare_output_dirs(out_dir, feed.slug)
settings = build_feed_settings(base_settings, out_dir=out_dir, feed_slug=feed.slug)
settings = build_feed_settings(
base_settings,
out_dir=out_dir,
feed_slug=feed.slug,
convert_images=convert_images,
convert_video=convert_video,
)
settings.set("LOG_FILE", None, priority="cmdline")
settings.set(
"STATS_CLASS",