switch to uv and to nix flakes
This commit is contained in:
parent
14005f36ce
commit
b1bdef2d5d
20 changed files with 1522 additions and 1751 deletions
|
|
@ -1,6 +1,7 @@
|
|||
import copy
|
||||
|
||||
import scrapy.utils.log
|
||||
|
||||
from colorlog import ColoredFormatter
|
||||
|
||||
color_formatter = ColoredFormatter(
|
||||
|
|
|
|||
|
|
@ -25,12 +25,13 @@ class FeedNameFilter:
|
|||
|
||||
|
||||
def execute_spider(queue, name, url):
|
||||
from repub.media import check_runtime
|
||||
from repub.spiders.rss_spider import RssFeedSpider
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
from repub.media import check_runtime
|
||||
from repub.spiders.rss_spider import RssFeedSpider
|
||||
|
||||
try:
|
||||
settings: Settings = {
|
||||
**get_project_settings(),
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
from repub import rss
|
||||
from scrapy.exporters import BaseItemExporter
|
||||
|
||||
from .exceptions import *
|
||||
from repub import rss
|
||||
|
||||
from .items import ChannelElementItem
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -345,7 +345,7 @@ def transcode_video(input_file: str, output_dir: str, params: Dict[str, Any]) ->
|
|||
return output_file
|
||||
except ffmpeg.Error as e:
|
||||
print(e.stderr, file=sys.stderr)
|
||||
logger.error(f"Failed to transcode")
|
||||
logger.error("Failed to transcode")
|
||||
logger.error(e)
|
||||
raise RuntimeError(f"Failed to transcode video: {e.stderr.decode()}") from e
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@
|
|||
# See documentation in:
|
||||
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
# useful for handling different item types with a single interface
|
||||
from itemadapter import ItemAdapter, is_item
|
||||
from scrapy import signals
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,13 +4,14 @@ from io import BytesIO
|
|||
from os import PathLike
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
import repub.utils
|
||||
from repub import media
|
||||
from scrapy.pipelines.files import FilesPipeline as BaseFilesPipeline
|
||||
from scrapy.pipelines.images import ImagesPipeline as BaseImagesPipeline
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.misc import md5sum
|
||||
|
||||
import repub.utils
|
||||
from repub import media
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from typing import List, Tuple
|
||||
from datetime import datetime
|
||||
from time import mktime
|
||||
|
||||
import lxml.etree as ET
|
||||
import lxml.html
|
||||
|
|
@ -53,9 +54,6 @@ ATOM = SafeElementMaker(nsmap={None: nsmap["atom"]}, namespace=nsmap["atom"])
|
|||
E: ElementMaker = SafeElementMaker(nsmap=nsmap)
|
||||
CDATA = ET.CDATA
|
||||
|
||||
from datetime import datetime
|
||||
from time import mktime
|
||||
|
||||
|
||||
def rss():
|
||||
return E.rss({"version": "2.0"})
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@ import logging
|
|||
from typing import Dict, List, Tuple
|
||||
|
||||
import feedparser
|
||||
from repub.items import ChannelElementItem, ElementItem
|
||||
from repub.rss import CDATA, CONTENT, ITUNES, MEDIA, E, munge_cdata_html, normalize_date
|
||||
from repub.utils import FileType, determine_file_type, local_file_path, local_image_path
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
from repub.items import ChannelElementItem, ElementItem
|
||||
from repub.rss import CDATA, CONTENT, ITUNES, MEDIA, E, munge_cdata_html, normalize_date
|
||||
from repub.utils import FileType, determine_file_type, local_file_path
|
||||
|
||||
|
||||
class BaseRssFeedSpider(Spider):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
|||
import math
|
||||
|
||||
# See https://infra.spec.whatwg.org/#ascii-whitespace
|
||||
WHITESPACES = ("\u0009", "\u000A", "\u000C", "\u000D", "\u0020") # \t # " "
|
||||
WHITESPACES = ("\u0009", "\u000a", "\u000c", "\u000d", "\u0020") # \t # " "
|
||||
|
||||
STATE_IN_DESCRIPTOR = 1
|
||||
STATE_AFTER_DESCRIPTOR = 2
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import hashlib
|
|||
import mimetypes
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue