Refactor database access through managed connections

This commit is contained in:
Abel Luck 2026-03-31 17:30:07 +02:00
parent f19bab6fa2
commit 3f28e46ff6
10 changed files with 1327 additions and 716 deletions

View file

@ -19,6 +19,7 @@ from repub.model import (
JobExecutionStatus,
Source,
create_source,
database,
initialize_database,
save_setting,
)
@ -29,6 +30,16 @@ FIXTURE_FEED_PATH = (
).resolve()
def _db_reader(callable_):
with database.reader():
return callable_()
def _db_writer(callable_):
with database.writer():
return callable_()
def initialize_runtime_database(db_path: Path) -> None:
initialize_database(db_path)
save_setting("feed_url", "http://localhost:8080")
@ -64,8 +75,9 @@ def test_job_runtime_syncs_enabled_jobs_into_apscheduler(tmp_path: Path) -> None
cron_month="*",
feed_url="https://example.com/disabled.xml",
)
enabled_job = Job.get(Job.source == enabled_source)
disabled_job = Job.get(Job.source == disabled_source)
with database.reader():
enabled_job = Job.get(Job.source == enabled_source)
disabled_job = Job.get(Job.source == disabled_source)
runtime = JobRuntime(log_dir=tmp_path / "out" / "logs")
try:
@ -77,8 +89,10 @@ def test_job_runtime_syncs_enabled_jobs_into_apscheduler(tmp_path: Path) -> None
assert f"job-{enabled_job.id}" in scheduled_ids
assert f"job-{disabled_job.id}" not in scheduled_ids
enabled_job.enabled = False
enabled_job.save()
with database.writer():
enabled_job = Job.get_by_id(enabled_job.id)
enabled_job.enabled = False
enabled_job.save()
runtime.sync_jobs()
scheduled_ids = {job.id for job in runtime.scheduler.get_jobs()}
@ -105,7 +119,8 @@ def test_job_runtime_run_now_writes_log_and_stats_and_marks_success(
cron_month="*",
feed_url=FIXTURE_FEED_PATH.as_uri(),
)
job = Job.get(Job.source == source)
with database.reader():
job = Job.get(Job.source == source)
runtime = JobRuntime(log_dir=tmp_path / "out" / "logs")
try:
@ -178,8 +193,9 @@ def test_job_runtime_respects_max_concurrent_jobs_setting(tmp_path: Path) -> Non
cron_month="*",
feed_url=feed_url,
)
first_job = Job.get(Job.source == first_source)
second_job = Job.get(Job.source == second_source)
with database.reader():
first_job = Job.get(Job.source == first_source)
second_job = Job.get(Job.source == second_source)
runtime = JobRuntime(log_dir=log_dir)
try:
@ -197,16 +213,20 @@ def test_job_runtime_respects_max_concurrent_jobs_setting(tmp_path: Path) -> Non
JobExecutionStatus.PENDING,
)
assert (
JobExecution.select()
.where(JobExecution.running_status == JobExecutionStatus.RUNNING)
.count()
_db_reader(
lambda: JobExecution.select()
.where(JobExecution.running_status == JobExecutionStatus.RUNNING)
.count()
)
== 1
)
assert second_execution.started_at is None
assert (
JobExecution.select()
.where(JobExecution.running_status == JobExecutionStatus.PENDING)
.count()
_db_reader(
lambda: JobExecution.select()
.where(JobExecution.running_status == JobExecutionStatus.PENDING)
.count()
)
== 1
)
runtime.request_execution_cancel(first_execution_id)
@ -253,8 +273,9 @@ def test_job_runtime_starts_queued_execution_after_capacity_opens(
cron_month="*",
feed_url=FIXTURE_FEED_PATH.as_uri(),
)
first_job = Job.get(Job.source == first_source)
second_job = Job.get(Job.source == second_source)
with database.reader():
first_job = Job.get(Job.source == first_source)
second_job = Job.get(Job.source == second_source)
runtime = JobRuntime(log_dir=log_dir)
try:
@ -314,8 +335,9 @@ def test_job_runtime_deduplicates_manual_queue_requests(tmp_path: Path) -> None:
cron_month="*",
feed_url="https://example.com/queued.xml",
)
blocking_job = Job.get(Job.source == blocking_source)
queued_job = Job.get(Job.source == queued_source)
with database.reader():
blocking_job = Job.get(Job.source == blocking_source)
queued_job = Job.get(Job.source == queued_source)
runtime = JobRuntime(log_dir=log_dir)
try:
@ -332,12 +354,14 @@ def test_job_runtime_deduplicates_manual_queue_requests(tmp_path: Path) -> None:
assert first_pending_id is not None
assert second_pending_id == first_pending_id
assert (
JobExecution.select()
.where(
(JobExecution.job == queued_job)
& (JobExecution.running_status == JobExecutionStatus.PENDING)
_db_reader(
lambda: JobExecution.select()
.where(
(JobExecution.job == queued_job)
& (JobExecution.running_status == JobExecutionStatus.PENDING)
)
.count()
)
.count()
== 1
)
finally:
@ -367,7 +391,8 @@ def test_job_runtime_allows_one_running_and_one_pending_per_job(
cron_month="*",
feed_url=feed_url,
)
job = Job.get(Job.source == source)
with database.reader():
job = Job.get(Job.source == source)
runtime = JobRuntime(log_dir=log_dir)
try:
@ -383,17 +408,21 @@ def test_job_runtime_allows_one_running_and_one_pending_per_job(
assert pending_execution_id is not None
assert duplicate_pending_id == pending_execution_id
assert (
JobExecution.select()
.where(JobExecution.job == job)
.where(JobExecution.running_status == JobExecutionStatus.RUNNING)
.count()
_db_reader(
lambda: JobExecution.select()
.where(JobExecution.job == job)
.where(JobExecution.running_status == JobExecutionStatus.RUNNING)
.count()
)
== 1
)
assert (
JobExecution.select()
.where(JobExecution.job == job)
.where(JobExecution.running_status == JobExecutionStatus.PENDING)
.count()
_db_reader(
lambda: JobExecution.select()
.where(JobExecution.job == job)
.where(JobExecution.running_status == JobExecutionStatus.PENDING)
.count()
)
== 1
)
finally:
@ -420,11 +449,12 @@ def test_job_runtime_start_drains_pending_rows_created_before_start(
cron_month="*",
feed_url=FIXTURE_FEED_PATH.as_uri(),
)
job = Job.get(Job.source == source)
pending_execution = JobExecution.create(
job=job,
running_status=JobExecutionStatus.PENDING,
)
with database.writer():
job = Job.get(Job.source == source)
pending_execution = JobExecution.create(
job=job,
running_status=JobExecutionStatus.PENDING,
)
runtime = JobRuntime(log_dir=log_dir)
try:
@ -477,18 +507,23 @@ def test_job_runtime_scheduled_runs_use_the_persistent_queue(
cron_month="*",
feed_url="https://example.com/second-scheduled.xml",
)
first_job = Job.get(Job.source == first_source)
second_job = Job.get(Job.source == second_source)
with database.reader():
first_job = Job.get(Job.source == first_source)
second_job = Job.get(Job.source == second_source)
runtime = JobRuntime(log_dir=log_dir)
try:
runtime.start()
runtime.run_scheduled_job(first_job.id)
first_execution = JobExecution.get(JobExecution.job == first_job)
first_execution = _db_reader(
lambda: JobExecution.get(JobExecution.job == first_job)
)
_wait_for_running_execution(int(first_execution.get_id()))
runtime.run_scheduled_job(second_job.id)
second_execution = JobExecution.get(JobExecution.job == second_job)
second_execution = _db_reader(
lambda: JobExecution.get(JobExecution.job == second_job)
)
assert second_execution.running_status == JobExecutionStatus.PENDING
assert second_execution.started_at is None
@ -519,7 +554,8 @@ def test_job_runtime_cancel_pending_follow_up_keeps_running_worker_alive(
cron_month="*",
feed_url=feed_url,
)
job = Job.get(Job.source == source)
with database.reader():
job = Job.get(Job.source == source)
runtime = JobRuntime(log_dir=log_dir)
try:
@ -533,9 +569,14 @@ def test_job_runtime_cancel_pending_follow_up_keeps_running_worker_alive(
_wait_for_execution_status(pending_execution_id, JobExecutionStatus.PENDING)
assert runtime.cancel_queued_execution(pending_execution_id) is True
assert JobExecution.get_or_none(id=pending_execution_id) is None
assert (
JobExecution.get_by_id(running_execution_id).running_status
_db_reader(lambda: JobExecution.get_or_none(id=pending_execution_id))
is None
)
assert (
_db_reader(
lambda: JobExecution.get_by_id(running_execution_id).running_status
)
== JobExecutionStatus.RUNNING
)
finally:
@ -559,7 +600,8 @@ def test_job_runtime_cancel_marks_execution_canceled(tmp_path: Path) -> None:
cron_month="*",
feed_url=feed_url,
)
job = Job.get(Job.source == source)
with database.reader():
job = Job.get(Job.source == source)
runtime = JobRuntime(log_dir=tmp_path / "out" / "logs")
try:
@ -602,12 +644,13 @@ def test_job_runtime_start_reconciles_stale_running_execution(tmp_path: Path) ->
cron_month="*",
feed_url="https://example.com/stale.xml",
)
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
started_at="2026-03-30 12:30:00+00:00",
running_status=JobExecutionStatus.RUNNING,
)
with database.writer():
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
started_at="2026-03-30 12:30:00+00:00",
running_status=JobExecutionStatus.RUNNING,
)
artifacts = JobArtifacts.for_execution(
log_dir=tmp_path / "out" / "logs",
job_id=job.id,
@ -622,7 +665,9 @@ def test_job_runtime_start_reconciles_stale_running_execution(tmp_path: Path) ->
runtime = JobRuntime(log_dir=tmp_path / "out" / "logs")
try:
runtime.start()
reconciled_execution = JobExecution.get_by_id(execution.get_id())
reconciled_execution = _db_reader(
lambda: JobExecution.get_by_id(execution.get_id())
)
assert reconciled_execution.running_status == JobExecutionStatus.FAILED
assert reconciled_execution.ended_at is not None
@ -649,12 +694,13 @@ def test_job_runtime_publishes_refresh_while_jobs_are_running(tmp_path: Path) ->
cron_month="*",
feed_url="https://example.com/running.xml",
)
job = Job.get(Job.source == source)
JobExecution.create(
job=job,
started_at=datetime(2026, 3, 30, 12, 0, tzinfo=UTC),
running_status=JobExecutionStatus.RUNNING,
)
with database.writer():
job = Job.get(Job.source == source)
JobExecution.create(
job=job,
started_at=datetime(2026, 3, 30, 12, 0, tzinfo=UTC),
running_status=JobExecutionStatus.RUNNING,
)
events: list[object] = []
runtime = JobRuntime(
@ -688,12 +734,13 @@ def test_job_runtime_start_reattaches_live_worker_after_app_restart(
cron_month="*",
feed_url=feed_url,
)
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
started_at=datetime.now(UTC),
running_status=JobExecutionStatus.RUNNING,
)
with database.writer():
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
started_at=datetime.now(UTC),
running_status=JobExecutionStatus.RUNNING,
)
artifacts = JobArtifacts.for_execution(
log_dir=log_dir,
job_id=job.id,
@ -728,7 +775,9 @@ def test_job_runtime_start_reattaches_live_worker_after_app_restart(
time.sleep(0.1)
runtime.start()
running_execution = JobExecution.get_by_id(execution.get_id())
running_execution = _db_reader(
lambda: JobExecution.get_by_id(execution.get_id())
)
assert running_execution.running_status == JobExecutionStatus.RUNNING
assert running_execution.ended_at is None
@ -764,13 +813,14 @@ def test_job_runtime_start_restores_live_worker_marked_failed_by_restart_bug(
cron_month="*",
feed_url=feed_url,
)
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
started_at=datetime.now(UTC),
ended_at=datetime.now(UTC),
running_status=JobExecutionStatus.FAILED,
)
with database.writer():
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
started_at=datetime.now(UTC),
ended_at=datetime.now(UTC),
running_status=JobExecutionStatus.FAILED,
)
artifacts = JobArtifacts.for_execution(
log_dir=log_dir,
job_id=job.id,
@ -805,7 +855,9 @@ def test_job_runtime_start_restores_live_worker_marked_failed_by_restart_bug(
time.sleep(0.1)
runtime.start()
restored_execution = JobExecution.get_by_id(execution.get_id())
restored_execution = _db_reader(
lambda: JobExecution.get_by_id(execution.get_id())
)
assert restored_execution.running_status == JobExecutionStatus.RUNNING
assert restored_execution.ended_at is None
@ -895,14 +947,15 @@ def test_load_runs_view_humanizes_completed_execution_end_time(
cron_month="*",
feed_url="https://example.com/completed.xml",
)
job = Job.get(Job.source == source)
reference_time = datetime(2026, 1, 15, 12, 0, tzinfo=UTC)
ended_at = reference_time - timedelta(hours=2)
JobExecution.create(
job=job,
running_status=JobExecutionStatus.SUCCEEDED,
ended_at=ended_at,
)
with database.writer():
job = Job.get(Job.source == source)
reference_time = datetime(2026, 1, 15, 12, 0, tzinfo=UTC)
ended_at = reference_time - timedelta(hours=2)
JobExecution.create(
job=job,
running_status=JobExecutionStatus.SUCCEEDED,
ended_at=ended_at,
)
view = load_runs_view(log_dir=app.config["REPUB_LOG_DIR"], now=reference_time)
completed = view["completed"][0]
@ -934,14 +987,15 @@ def test_load_runs_view_humanizes_running_execution_start_time(
cron_month="*",
feed_url="https://example.com/running.xml",
)
job = Job.get(Job.source == source)
reference_time = datetime(2026, 1, 15, 12, 0, tzinfo=UTC)
started_at = reference_time - timedelta(hours=2)
JobExecution.create(
job=job,
running_status=JobExecutionStatus.RUNNING,
started_at=started_at,
)
with database.writer():
job = Job.get(Job.source == source)
reference_time = datetime(2026, 1, 15, 12, 0, tzinfo=UTC)
started_at = reference_time - timedelta(hours=2)
JobExecution.create(
job=job,
running_status=JobExecutionStatus.RUNNING,
started_at=started_at,
)
view = load_runs_view(log_dir=app.config["REPUB_LOG_DIR"], now=reference_time)
running = view["running"][0]
@ -974,7 +1028,8 @@ def test_render_runs_uses_database_backed_jobs_and_executions(
cron_month="*",
feed_url=FIXTURE_FEED_PATH.as_uri(),
)
job = Job.get(Job.source == source)
with database.reader():
job = Job.get(Job.source == source)
runtime = get_job_runtime(app)
runtime.start()
try:
@ -1021,11 +1076,12 @@ def test_render_execution_logs_handles_missing_execution_and_missing_log_file(
cron_month="*",
feed_url="https://example.com/log-source.xml",
)
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
running_status=JobExecutionStatus.FAILED,
)
with database.writer():
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
running_status=JobExecutionStatus.FAILED,
)
async def run() -> None:
missing_execution = str(
@ -1067,18 +1123,25 @@ def test_delete_job_action_removes_source_job_and_execution_history(
cron_month="*",
feed_url="https://example.com/delete.xml",
)
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
running_status=JobExecutionStatus.SUCCEEDED,
)
with database.writer():
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
running_status=JobExecutionStatus.SUCCEEDED,
)
response = await client.post(f"/actions/jobs/{job.id}/delete")
assert response.status_code == 204
assert Source.get_or_none(Source.slug == "delete-source") is None
assert Job.get_or_none(id=job.id) is None
assert JobExecution.get_or_none(id=int(execution.get_id())) is None
assert (
_db_reader(lambda: Source.get_or_none(Source.slug == "delete-source"))
is None
)
assert _db_reader(lambda: Job.get_or_none(id=job.id)) is None
assert (
_db_reader(lambda: JobExecution.get_or_none(id=int(execution.get_id())))
is None
)
asyncio.run(run())
@ -1107,18 +1170,25 @@ def test_delete_source_action_removes_source_job_and_execution_history(
cron_month="*",
feed_url="https://example.com/delete-source-row.xml",
)
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
running_status=JobExecutionStatus.SUCCEEDED,
)
with database.writer():
job = Job.get(Job.source == source)
execution = JobExecution.create(
job=job,
running_status=JobExecutionStatus.SUCCEEDED,
)
response = await client.post("/actions/sources/delete-source-row/delete")
assert response.status_code == 204
assert Source.get_or_none(Source.slug == "delete-source-row") is None
assert Job.get_or_none(id=job.id) is None
assert JobExecution.get_or_none(id=int(execution.get_id())) is None
assert (
_db_reader(lambda: Source.get_or_none(Source.slug == "delete-source-row"))
is None
)
assert _db_reader(lambda: Job.get_or_none(id=job.id)) is None
assert (
_db_reader(lambda: JobExecution.get_or_none(id=int(execution.get_id())))
is None
)
asyncio.run(run())
@ -1128,7 +1198,7 @@ def _wait_for_running_execution(
) -> JobExecution:
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
execution = JobExecution.get_by_id(execution_id)
execution = _db_reader(lambda: JobExecution.get_by_id(execution_id))
if execution.running_status == JobExecutionStatus.RUNNING:
return execution
time.sleep(0.02)
@ -1143,7 +1213,7 @@ def _wait_for_execution_status(
) -> JobExecution:
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
execution = JobExecution.get_by_id(execution_id)
execution = _db_reader(lambda: JobExecution.get_by_id(execution_id))
if execution.running_status == status:
return execution
time.sleep(0.02)
@ -1155,7 +1225,7 @@ def _wait_for_terminal_execution(
) -> JobExecution:
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
execution = JobExecution.get_by_id(execution_id)
execution = _db_reader(lambda: JobExecution.get_by_id(execution_id))
if execution.running_status in {
JobExecutionStatus.SUCCEEDED,
JobExecutionStatus.FAILED,