add dev-mode

repub/entrypoint.py

@@ -42,6 +42,11 @@ def parse_args(argv: list[str] | None = None) -> tuple[str, argparse.Namespace]:
         default=os.environ.get("REPUBLISHER_PORT", "8080"),
         help="Port for the web UI",
     )
+    serve_parser.add_argument(
+        "--dev-mode",
+        action="store_true",
+        help="Serve published feeds from /feeds for local development",
+    )
 
     crawl_parser = subparsers.add_parser("crawl", help="Run the feed crawler once")
     crawl_parser.add_argument(
@@ -51,11 +56,11 @@ def parse_args(argv: list[str] | None = None) -> tuple[str, argparse.Namespace]:
         help="Path to runtime config TOML file",
     )
     if not raw_args:
-        raw_args = ["serve"]
+        raw_args = ["serve", "--dev-mode"]
     elif raw_args[0] in {"-c", "--config"}:
         raw_args = ["crawl", *raw_args]
     elif raw_args[0] not in {"serve", "crawl"}:
-        raw_args = ["serve", *raw_args]
+        raw_args = ["serve", "--dev-mode", *raw_args]
 
     args = parser.parse_args(raw_args)
     command = args.command or "serve"
@@ -75,7 +80,7 @@ def entrypoint(argv: list[str] | None = None) -> int:
         logger.error("Invalid REPUBLISHER_PORT/--port value: %s", args.port)
         return 2
 
-    app = create_app()
+    app = create_app(dev_mode=bool(args.dev_mode))
     app.run(host=args.host, port=port)
     return 0
 

repub/jobs.py

@@ -61,16 +61,10 @@ class JobRuntime:
         self,
         *,
         log_dir: str | Path,
-        worker_duration_seconds: float = 20.0,
-        worker_stats_interval_seconds: float = 1.0,
-        worker_failure_probability: float = 0.3,
         refresh_callback: Callable[[], None] | None = None,
         graceful_stop_seconds: float = 15.0,
     ) -> None:
         self.log_dir = Path(log_dir)
-        self.worker_duration_seconds = worker_duration_seconds
-        self.worker_stats_interval_seconds = worker_stats_interval_seconds
-        self.worker_failure_probability = worker_failure_probability
         self.refresh_callback = refresh_callback
         self.graceful_stop_seconds = graceful_stop_seconds
         self.scheduler = BackgroundScheduler(timezone=UTC)

repub/web.py

@@ -13,7 +13,7 @@ from datastar_py.quart import DatastarResponse, read_signals
 from datastar_py.sse import DatastarEvent
 from htpy import Renderable
 from peewee import IntegrityError
-from quart import Quart, Response, request, url_for
+from quart import Quart, Response, request, send_from_directory, url_for
 
 from repub.datastar import RefreshBroker, render_stream
 from repub.jobs import (
@@ -46,6 +46,7 @@ from repub.pages.sources import PANGEA_CONTENT_FORMATS, PANGEA_CONTENT_TYPES
 REFRESH_BROKER_KEY = "repub.refresh_broker"
 JOB_RUNTIME_KEY = "repub.job_runtime"
 DEFAULT_LOG_DIR = Path("out/logs")
+DEFAULT_FEEDS_DIR = Path("out/feeds")
 
 RenderFunction = Callable[[], Awaitable[Renderable]]
 
@@ -95,16 +96,27 @@ def _render_shim_page(
     return body, etag
 
 
-def create_app() -> Quart:
+def create_app(*, dev_mode: bool = False) -> Quart:
     app = Quart(__name__)
     app.config["REPUB_DB_PATH"] = str(initialize_database())
     app.config.setdefault("REPUB_LOG_DIR", DEFAULT_LOG_DIR)
-    app.config.setdefault("REPUB_JOB_WORKER_DURATION_SECONDS", 20.0)
-    app.config.setdefault("REPUB_JOB_WORKER_STATS_INTERVAL_SECONDS", 1.0)
-    app.config.setdefault("REPUB_JOB_WORKER_FAILURE_PROBABILITY", 0.3)
+    app.config.setdefault("REPUB_FEEDS_DIR", DEFAULT_FEEDS_DIR)
+    app.config["REPUB_DEV_MODE"] = dev_mode
     app.extensions[REFRESH_BROKER_KEY] = RefreshBroker()
     app.extensions[JOB_RUNTIME_KEY] = None
 
+    @app.get("/feeds/<path:feed_path>")
+    async def published_feed(feed_path: str) -> Response:
+        if not bool(app.config["REPUB_DEV_MODE"]):
+            return Response(status=404)
+        response = await send_from_directory(
+            str(Path(app.config["REPUB_FEEDS_DIR"])),
+            feed_path,
+        )
+        if Path(feed_path).suffix == ".rss":
+            response.mimetype = "application/rss+xml"
+        return response
+
     @app.get("/")
     @app.get("/sources")
     @app.get("/sources/create")
@@ -257,15 +269,6 @@ def get_job_runtime(app: Quart) -> JobRuntime:
     if runtime is None:
         runtime = JobRuntime(
             log_dir=app.config["REPUB_LOG_DIR"],
-            worker_duration_seconds=float(
-                app.config["REPUB_JOB_WORKER_DURATION_SECONDS"]
-            ),
-            worker_stats_interval_seconds=float(
-                app.config["REPUB_JOB_WORKER_STATS_INTERVAL_SECONDS"]
-            ),
-            worker_failure_probability=float(
-                app.config["REPUB_JOB_WORKER_FAILURE_PROBABILITY"]
-            ),
             refresh_callback=lambda: trigger_refresh(app),
         )
         app.extensions[JOB_RUNTIME_KEY] = runtime