from pathlib import Path from scrapy.http import TextResponse from scrapy.settings import Settings from repub import entrypoint as entrypoint_module from repub.spiders.rss_spider import RssFeedSpider from repub.utils import FileType, local_audio_path, local_image_path def test_entrypoint_supports_file_feed_urls(tmp_path: Path, monkeypatch) -> None: fixture_path = ( Path(__file__).resolve().parents[1] / "demo" / "fixtures" / "local-feed.rss" ).resolve() config_path = tmp_path / "repub.toml" config_path.write_text( f""" out_dir = "out" [[feeds]] name = "Local Demo" slug = "local-file" url = "{fixture_path.as_uri()}" [scrapy.settings] LOG_LEVEL = "ERROR" DOWNLOAD_TIMEOUT = 5 """.strip() + "\n", encoding="utf-8", ) monkeypatch.setattr(entrypoint_module, "check_runtime", lambda *_: True) exit_code = entrypoint_module.entrypoint(["--config", str(config_path)]) output_path = tmp_path / "out" / "feeds" / "local-file" / "feed.rss" assert exit_code == 0 assert output_path.exists() output = output_path.read_text(encoding="utf-8") assert "Local Demo Feed" in output assert "Local Demo Entry" in output def test_rss_spider_rewrites_public_asset_urls_as_relative_paths() -> None: spider = RssFeedSpider(feed_name="demo", url="https://example.com/feed.rss") spider.settings = Settings( values={ "REPUBLISHER_IMAGE_DIR": "images", "REPUBLISHER_FILE_DIR": "files", "REPUBLISHER_AUDIO_DIR": "audio", "REPUBLISHER_VIDEO_DIR": "video", } ) assert ( spider.rewrite_image_url("https://example.com/media/photo.jpg") == f"images/{local_image_path('https://example.com/media/photo.jpg')}" ) assert ( spider.rewrite_file_url( FileType.AUDIO, "https://example.com/media/podcast.mp3", ) == f"audio/{local_audio_path('https://example.com/media/podcast.mp3')}" ) def test_rss_spider_keeps_items_with_empty_content_encoded() -> None: feed_text = """ Empty Content Feed https://example.com Feed with empty HTML content blocks. Entry With Empty Content https://example.com/entry Summary text still exists. entry-1 Mon, 01 Jan 2024 00:00:00 +0000 """ spider = RssFeedSpider(feed_name="demo", url="https://example.com/feed.rss") spider.settings = Settings( values={ "REPUBLISHER_IMAGE_DIR": "images", "REPUBLISHER_FILE_DIR": "files", "REPUBLISHER_AUDIO_DIR": "audio", "REPUBLISHER_VIDEO_DIR": "video", } ) response = TextResponse( url="https://example.com/feed.rss", body=feed_text.encode("utf-8"), encoding="utf-8", ) parse_result = spider._parse(response) assert parse_result is not None items = list(parse_result) assert len(items) == 2 assert items[0].el.findtext("title") == "Empty Content Feed" assert items[1].el.findtext("title") == "Entry With Empty Content"