switch to uv and to nix flakes
This commit is contained in:
parent
14005f36ce
commit
b1bdef2d5d
20 changed files with 1522 additions and 1751 deletions
5
.envrc
5
.envrc
|
|
@ -1,2 +1,3 @@
|
|||
use nix
|
||||
#dotenv
|
||||
use flake
|
||||
dotenv_if_exists
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
# republisher-redux
|
||||
|
||||
``` shell
|
||||
mkdir logs out
|
||||
poetry install
|
||||
poetry run repub
|
||||
mkdir -p logs out
|
||||
nix develop
|
||||
uv sync --all-groups
|
||||
uv run repub
|
||||
```
|
||||
|
||||
## TODO
|
||||
|
|
|
|||
115
flake.lock
generated
Normal file
115
flake.lock
generated
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
{
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1774386573,
|
||||
"narHash": "sha256-4hAV26quOxdC6iyG7kYaZcM3VOskcPUrdCQd/nx8obc=",
|
||||
"rev": "46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9",
|
||||
"revCount": 969196,
|
||||
"type": "tarball",
|
||||
"url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.969196%2Brev-46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9/019d279e-af65-79ce-92be-5dee7b1e36d4/source.tar.gz"
|
||||
},
|
||||
"original": {
|
||||
"type": "tarball",
|
||||
"url": "https://flakehub.com/f/NixOS/nixpkgs/0.1"
|
||||
}
|
||||
},
|
||||
"pyproject-build-systems": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"pyproject-nix": [
|
||||
"pyproject-nix"
|
||||
],
|
||||
"uv2nix": [
|
||||
"uv2nix"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1773870109,
|
||||
"narHash": "sha256-ZoTdqZP03DcdoyxvpFHCAek4bkPUTUPUF3oCCgc3dP4=",
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "build-system-pkgs",
|
||||
"rev": "b6e74f433b02fa4b8a7965ee24680f4867e2926f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "build-system-pkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"pyproject-nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1774498001,
|
||||
"narHash": "sha256-wTfdyzzrmpuqt4TQQNqilF91v0m5Mh1stNy9h7a/WK4=",
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "pyproject.nix",
|
||||
"rev": "794afa6eb588b498344f2eaa36ab1ceb7e6b0b09",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "pyproject.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs",
|
||||
"pyproject-build-systems": "pyproject-build-systems",
|
||||
"pyproject-nix": "pyproject-nix",
|
||||
"treefmt-nix": "treefmt-nix",
|
||||
"uv2nix": "uv2nix"
|
||||
}
|
||||
},
|
||||
"treefmt-nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1774781724,
|
||||
"narHash": "sha256-81GlxqpDZroeH6f+GZEM7V3VEqlLA4U/jU5e5L2yM0Y=",
|
||||
"path": "/home/abel/src/github.com/numtide/treefmt-nix",
|
||||
"type": "path"
|
||||
},
|
||||
"original": {
|
||||
"path": "/home/abel/src/github.com/numtide/treefmt-nix",
|
||||
"type": "path"
|
||||
}
|
||||
},
|
||||
"uv2nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"pyproject-nix": [
|
||||
"pyproject-nix"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1774705889,
|
||||
"narHash": "sha256-TRTIM18gP3ccBj3m8bV1zx82xeYweNYp8/lgcdR4Zz0=",
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "uv2nix",
|
||||
"rev": "28355ed75b466a15ff324e1baa151b550619fe67",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "uv2nix",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
246
flake.nix
Normal file
246
flake.nix
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
{
|
||||
description = "republisher-redux - offline RSS and Atom feed mirroring";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.1";
|
||||
treefmt-nix = {
|
||||
url = "path:/home/abel/src/github.com/numtide/treefmt-nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
pyproject-nix = {
|
||||
url = "github:pyproject-nix/pyproject.nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
uv2nix = {
|
||||
url = "github:pyproject-nix/uv2nix";
|
||||
inputs.pyproject-nix.follows = "pyproject-nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
pyproject-build-systems = {
|
||||
url = "github:pyproject-nix/build-system-pkgs";
|
||||
inputs.pyproject-nix.follows = "pyproject-nix";
|
||||
inputs.uv2nix.follows = "uv2nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
};
|
||||
|
||||
outputs =
|
||||
{
|
||||
self,
|
||||
nixpkgs,
|
||||
treefmt-nix,
|
||||
pyproject-nix,
|
||||
uv2nix,
|
||||
pyproject-build-systems,
|
||||
...
|
||||
}:
|
||||
let
|
||||
systems = [ "x86_64-linux" ];
|
||||
forAllSystems =
|
||||
fn:
|
||||
nixpkgs.lib.genAttrs systems (
|
||||
system:
|
||||
fn (
|
||||
import nixpkgs {
|
||||
inherit system;
|
||||
config.allowUnfree = true;
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
mkTreefmtConfig = pkgs: (treefmt-nix.lib.evalModule pkgs ./treefmt.nix).config;
|
||||
|
||||
workspace = uv2nix.lib.workspace.loadWorkspace { workspaceRoot = ./.; };
|
||||
overlay = workspace.mkPyprojectOverlay { sourcePreference = "wheel"; };
|
||||
pyprojectOverrides = final: prev: {
|
||||
sgmllib3k = prev.sgmllib3k.overrideAttrs (old: {
|
||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ final.setuptools ];
|
||||
});
|
||||
};
|
||||
|
||||
mkPackage =
|
||||
pkgs:
|
||||
let
|
||||
ffmpegPackage = pkgs.ffmpeg-full;
|
||||
|
||||
pythonSet =
|
||||
(pkgs.callPackage pyproject-nix.build.packages {
|
||||
python = pkgs.python313;
|
||||
}).overrideScope
|
||||
(
|
||||
pkgs.lib.composeManyExtensions [
|
||||
pyproject-build-systems.overlays.default
|
||||
overlay
|
||||
pyprojectOverrides
|
||||
]
|
||||
);
|
||||
|
||||
baseVenv = pythonSet.mkVirtualEnv "republisher-redux-env" workspace.deps.default;
|
||||
testVenv = pythonSet.mkVirtualEnv "republisher-redux-test-env" {
|
||||
"republisher-redux" = [ "dev" ];
|
||||
};
|
||||
|
||||
tests = pkgs.stdenv.mkDerivation {
|
||||
name = "republisher-redux-tests";
|
||||
src = ./.;
|
||||
dontConfigure = true;
|
||||
dontBuild = true;
|
||||
nativeBuildInputs = [ testVenv ];
|
||||
checkPhase = ''
|
||||
runHook preCheck
|
||||
export HOME="$(mktemp -d)"
|
||||
pytest tests/ -v
|
||||
runHook postCheck
|
||||
'';
|
||||
doCheck = true;
|
||||
installPhase = ''
|
||||
mkdir -p "$out"
|
||||
touch "$out/passed"
|
||||
'';
|
||||
};
|
||||
|
||||
runtimePackage = pkgs.symlinkJoin {
|
||||
name = "republisher-redux";
|
||||
paths = [ baseVenv ];
|
||||
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||
postBuild = ''
|
||||
rm -f "$out/bin/repub"
|
||||
makeWrapper "${baseVenv}/bin/repub" "$out/bin/repub" \
|
||||
--prefix PATH : "${pkgs.lib.makeBinPath [ ffmpegPackage ]}"
|
||||
'';
|
||||
meta.mainProgram = "repub";
|
||||
};
|
||||
in
|
||||
pkgs.runCommand "republisher-redux"
|
||||
{
|
||||
inherit (runtimePackage) meta;
|
||||
passthru = {
|
||||
inherit tests testVenv runtimePackage;
|
||||
};
|
||||
}
|
||||
''
|
||||
test -f "${tests}/passed"
|
||||
ln -s "${runtimePackage}" "$out"
|
||||
'';
|
||||
in
|
||||
{
|
||||
formatter = forAllSystems (pkgs: (mkTreefmtConfig pkgs).build.wrapper);
|
||||
|
||||
packages = forAllSystems (
|
||||
pkgs:
|
||||
let
|
||||
pkg = mkPackage pkgs;
|
||||
in
|
||||
{
|
||||
"republisher-redux" = pkg;
|
||||
default = pkg;
|
||||
}
|
||||
);
|
||||
|
||||
apps = forAllSystems (
|
||||
pkgs:
|
||||
let
|
||||
package = self.packages.${pkgs.stdenv.hostPlatform.system}.default;
|
||||
in
|
||||
{
|
||||
repub = {
|
||||
type = "app";
|
||||
program = "${package}/bin/repub";
|
||||
meta.description = "republisher-redux runtime";
|
||||
};
|
||||
default = {
|
||||
type = "app";
|
||||
program = "${package}/bin/repub";
|
||||
meta.description = "republisher-redux runtime";
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
checks = forAllSystems (
|
||||
pkgs:
|
||||
let
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
exportedPackage = self.packages.${system}.default;
|
||||
testVenv = exportedPackage.testVenv;
|
||||
treefmtConfig = mkTreefmtConfig pkgs;
|
||||
src = ./.;
|
||||
blackCheck = pkgs.stdenv.mkDerivation {
|
||||
name = "republisher-redux-black";
|
||||
inherit src;
|
||||
dontConfigure = true;
|
||||
dontBuild = true;
|
||||
nativeBuildInputs = [ testVenv ];
|
||||
checkPhase = ''
|
||||
runHook preCheck
|
||||
black --check repub/ tests/
|
||||
runHook postCheck
|
||||
'';
|
||||
doCheck = true;
|
||||
installPhase = ''
|
||||
mkdir -p "$out"
|
||||
touch "$out/passed"
|
||||
'';
|
||||
};
|
||||
flake8Check = pkgs.stdenv.mkDerivation {
|
||||
name = "republisher-redux-flake8";
|
||||
inherit src;
|
||||
dontConfigure = true;
|
||||
dontBuild = true;
|
||||
nativeBuildInputs = [ testVenv ];
|
||||
checkPhase = ''
|
||||
runHook preCheck
|
||||
flake8 repub/ tests/
|
||||
runHook postCheck
|
||||
'';
|
||||
doCheck = true;
|
||||
installPhase = ''
|
||||
mkdir -p "$out"
|
||||
touch "$out/passed"
|
||||
'';
|
||||
};
|
||||
isortCheck = pkgs.stdenv.mkDerivation {
|
||||
name = "republisher-redux-isort";
|
||||
inherit src;
|
||||
dontConfigure = true;
|
||||
dontBuild = true;
|
||||
nativeBuildInputs = [ testVenv ];
|
||||
checkPhase = ''
|
||||
runHook preCheck
|
||||
isort --check-only repub/ tests/
|
||||
runHook postCheck
|
||||
'';
|
||||
doCheck = true;
|
||||
installPhase = ''
|
||||
mkdir -p "$out"
|
||||
touch "$out/passed"
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
devshell-default = self.devShells.${system}.default;
|
||||
formatter = treefmtConfig.build.wrapper;
|
||||
package-default = exportedPackage;
|
||||
tests = exportedPackage.tests;
|
||||
treefmt = treefmtConfig.build.check ./.;
|
||||
black = blackCheck;
|
||||
flake8 = flake8Check;
|
||||
isort = isortCheck;
|
||||
}
|
||||
);
|
||||
|
||||
devShells = forAllSystems (pkgs: {
|
||||
default = pkgs.mkShell {
|
||||
packages = [
|
||||
pkgs.python313
|
||||
pkgs.uv
|
||||
pkgs.ffmpeg-full
|
||||
];
|
||||
env.LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
|
||||
pkgs.stdenv.cc.cc
|
||||
];
|
||||
env.UV_PROJECT_ENVIRONMENT = ".venv";
|
||||
env.UV_PYTHON_DOWNLOADS = "never";
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
1631
poetry.lock
generated
1631
poetry.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,71 +1,70 @@
|
|||
[tool.poetry]
|
||||
name = "repub"
|
||||
[project]
|
||||
name = "republisher-redux"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Abel Luck <abel@guardianproject.info>"]
|
||||
description = "Mirror RSS and Atom feeds completely offline"
|
||||
readme = "README.md"
|
||||
#packages = [{include = "repub", from = "repub"}]
|
||||
[tool.poetry.scripts]
|
||||
authors = [{ name = "Abel Luck", email = "abel@guardianproject.info" }]
|
||||
requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"scrapy>=2.11.1,<3.0.0",
|
||||
"prometheus-client>=0.20.0,<0.21.0",
|
||||
"python-dateutil>=2.9.0.post0,<3.0.0",
|
||||
"colorlog>=6.8.2,<7.0.0",
|
||||
"feedparser>=6.0.11,<7.0.0",
|
||||
"lxml>=5.2.1,<6.0.0",
|
||||
"pillow>=10.3.0,<11.0.0",
|
||||
"ffmpeg-python>=0.2.0,<0.3.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
repub = "repub.entrypoint:entrypoint"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.11"
|
||||
scrapy = "^2.11.1"
|
||||
prometheus-client = "^0.20.0"
|
||||
python-dateutil = "^2.9.0.post0"
|
||||
colorlog = "^6.8.2"
|
||||
feedparser = "^6.0.11"
|
||||
lxml = "^5.2.1"
|
||||
pillow = "^10.3.0"
|
||||
ffmpeg-python = "^0.2.0"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.1.1,<9.0.0",
|
||||
"black>=24.4.0,<25.0.0",
|
||||
"flake8>=7.0.0,<8.0.0",
|
||||
"mypy>=1.9.0,<2.0.0",
|
||||
"bandit>=1.7.8,<2.0.0",
|
||||
"types-PyYAML>=6.0.12.20240311,<7.0.0",
|
||||
"isort>=5.13.2,<6.0.0",
|
||||
"flake8-black>=0.3.6,<0.4.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
requires = ["setuptools>=68", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools]
|
||||
include-package-data = true
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
pytest = "^8.1.1"
|
||||
black = "^24.4.0"
|
||||
flake8 = "^7.0.0"
|
||||
mypy = "^1.9.0"
|
||||
bandit = "^1.7.8"
|
||||
types-PyYAML = "^6.0.12.20240311"
|
||||
isort = "^5.13.2"
|
||||
flake8-black = "^0.3.6"
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["repub*"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
|
||||
[tool.isort]
|
||||
py_version = 310
|
||||
py_version = 311
|
||||
profile = "black"
|
||||
src_paths = ["src", "tests"]
|
||||
src_paths = ["repub", "tests"]
|
||||
|
||||
[tool.black]
|
||||
line-length = 88
|
||||
target-version = ['py310']
|
||||
target-version = ['py313']
|
||||
|
||||
[tool.mypy]
|
||||
files = "gm,tests"
|
||||
files = "repub,tests"
|
||||
ignore_missing_imports = true
|
||||
follow_imports = "normal"
|
||||
# Ensure full coverage
|
||||
disallow_untyped_calls = true
|
||||
#disallow_untyped_defs = true
|
||||
#disallow_incomplete_defs = true
|
||||
#disallow_untyped_decorators = true
|
||||
#check_untyped_defs = true
|
||||
|
||||
# Restrict dynamic typing
|
||||
disallow_any_generics = true
|
||||
disallow_subclassing_any = true
|
||||
warn_return_any = true
|
||||
|
||||
# Know exactly what you're doing
|
||||
warn_redundant_casts = true
|
||||
warn_unused_ignores = true
|
||||
warn_unused_configs = true
|
||||
warn_unreachable = true
|
||||
show_error_codes = true
|
||||
|
||||
# Explicit is better than implici
|
||||
no_implicit_optional = true
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import copy
|
||||
|
||||
import scrapy.utils.log
|
||||
|
||||
from colorlog import ColoredFormatter
|
||||
|
||||
color_formatter = ColoredFormatter(
|
||||
|
|
|
|||
|
|
@ -25,12 +25,13 @@ class FeedNameFilter:
|
|||
|
||||
|
||||
def execute_spider(queue, name, url):
|
||||
from repub.media import check_runtime
|
||||
from repub.spiders.rss_spider import RssFeedSpider
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
from repub.media import check_runtime
|
||||
from repub.spiders.rss_spider import RssFeedSpider
|
||||
|
||||
try:
|
||||
settings: Settings = {
|
||||
**get_project_settings(),
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
from repub import rss
|
||||
from scrapy.exporters import BaseItemExporter
|
||||
|
||||
from .exceptions import *
|
||||
from repub import rss
|
||||
|
||||
from .items import ChannelElementItem
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -345,7 +345,7 @@ def transcode_video(input_file: str, output_dir: str, params: Dict[str, Any]) ->
|
|||
return output_file
|
||||
except ffmpeg.Error as e:
|
||||
print(e.stderr, file=sys.stderr)
|
||||
logger.error(f"Failed to transcode")
|
||||
logger.error("Failed to transcode")
|
||||
logger.error(e)
|
||||
raise RuntimeError(f"Failed to transcode video: {e.stderr.decode()}") from e
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@
|
|||
# See documentation in:
|
||||
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
# useful for handling different item types with a single interface
|
||||
from itemadapter import ItemAdapter, is_item
|
||||
from scrapy import signals
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,13 +4,14 @@ from io import BytesIO
|
|||
from os import PathLike
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
import repub.utils
|
||||
from repub import media
|
||||
from scrapy.pipelines.files import FilesPipeline as BaseFilesPipeline
|
||||
from scrapy.pipelines.images import ImagesPipeline as BaseImagesPipeline
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.misc import md5sum
|
||||
|
||||
import repub.utils
|
||||
from repub import media
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from typing import List, Tuple
|
||||
from datetime import datetime
|
||||
from time import mktime
|
||||
|
||||
import lxml.etree as ET
|
||||
import lxml.html
|
||||
|
|
@ -53,9 +54,6 @@ ATOM = SafeElementMaker(nsmap={None: nsmap["atom"]}, namespace=nsmap["atom"])
|
|||
E: ElementMaker = SafeElementMaker(nsmap=nsmap)
|
||||
CDATA = ET.CDATA
|
||||
|
||||
from datetime import datetime
|
||||
from time import mktime
|
||||
|
||||
|
||||
def rss():
|
||||
return E.rss({"version": "2.0"})
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@ import logging
|
|||
from typing import Dict, List, Tuple
|
||||
|
||||
import feedparser
|
||||
from repub.items import ChannelElementItem, ElementItem
|
||||
from repub.rss import CDATA, CONTENT, ITUNES, MEDIA, E, munge_cdata_html, normalize_date
|
||||
from repub.utils import FileType, determine_file_type, local_file_path, local_image_path
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
from repub.items import ChannelElementItem, ElementItem
|
||||
from repub.rss import CDATA, CONTENT, ITUNES, MEDIA, E, munge_cdata_html, normalize_date
|
||||
from repub.utils import FileType, determine_file_type, local_file_path
|
||||
|
||||
|
||||
class BaseRssFeedSpider(Spider):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
|||
import math
|
||||
|
||||
# See https://infra.spec.whatwg.org/#ascii-whitespace
|
||||
WHITESPACES = ("\u0009", "\u000A", "\u000C", "\u000D", "\u0020") # \t # " "
|
||||
WHITESPACES = ("\u0009", "\u000a", "\u000c", "\u000d", "\u0020") # \t # " "
|
||||
|
||||
STATE_IN_DESCRIPTOR = 1
|
||||
STATE_AFTER_DESCRIPTOR = 2
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import hashlib
|
|||
import mimetypes
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
|
|
|
|||
54
shell.nix
54
shell.nix
|
|
@ -1,54 +0,0 @@
|
|||
{
|
||||
system ? "x86_64-linux",
|
||||
pkgs ? import <nixpkgs> { inherit system; },
|
||||
dev ? true,
|
||||
}:
|
||||
|
||||
let
|
||||
pyCurrent = pkgs.python311;
|
||||
poetryExtras = if dev then [ "dev" ] else [ ];
|
||||
poetryInstallExtras = (
|
||||
if poetryExtras == [ ] then
|
||||
""
|
||||
else
|
||||
pkgs.lib.concatStrings [
|
||||
" --with="
|
||||
(pkgs.lib.concatStringsSep "," poetryExtras)
|
||||
]
|
||||
);
|
||||
packages = [
|
||||
(pkgs.ffmpeg_5-full.override {
|
||||
withUnfree = true;
|
||||
withFdkAac = true;
|
||||
})
|
||||
#(pyCurrent (ps: with ps; [ ffmpeg-python ]))
|
||||
pkgs.zsh
|
||||
(pkgs.poetry.withPlugins (ps: with ps; [ poetry-plugin-up ]))
|
||||
];
|
||||
|
||||
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
|
||||
pkgs.stdenv.cc.cc
|
||||
# Add any missing library needed
|
||||
# You can use the nix-index package to locate them, e.g. nix-locate -w --top-level --at-root /lib/libudev.so.1
|
||||
];
|
||||
|
||||
# Put the venv on the repo, so direnv can access it
|
||||
POETRY_VIRTUALENVS_IN_PROJECT = "true";
|
||||
POETRY_VIRTUALENVS_PATH = "{project-dir}/.venv";
|
||||
|
||||
# Use python from path, so you can use a different version to the one bundled with poetry
|
||||
POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON = "true";
|
||||
in
|
||||
pkgs.mkShell {
|
||||
buildInputs = packages;
|
||||
shellHook = ''
|
||||
export SHELL=${pkgs.zsh}
|
||||
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}"
|
||||
export POETRY_VIRTUALENVS_IN_PROJECT="${POETRY_VIRTUALENVS_IN_PROJECT}"
|
||||
export POETRY_VIRTUALENVS_PATH="${POETRY_VIRTUALENVS_PATH}"
|
||||
export POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON="${POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON}"
|
||||
export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
|
||||
poetry env use "${pyCurrent}/bin/python"
|
||||
poetry install -vv --sync${poetryInstallExtras}
|
||||
'';
|
||||
}
|
||||
17
tests/test_entrypoint.py
Normal file
17
tests/test_entrypoint.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
from repub.entrypoint import FeedNameFilter
|
||||
|
||||
|
||||
def test_feed_name_filter_accepts_matching_item() -> None:
|
||||
item = SimpleNamespace(feed_name="nasa")
|
||||
feed_filter = FeedNameFilter({"feed_name": "nasa"})
|
||||
|
||||
assert feed_filter.accepts(item) is True
|
||||
|
||||
|
||||
def test_feed_name_filter_rejects_non_matching_item() -> None:
|
||||
item = SimpleNamespace(feed_name="gp-pod")
|
||||
feed_filter = FeedNameFilter({"feed_name": "nasa"})
|
||||
|
||||
assert feed_filter.accepts(item) is False
|
||||
13
treefmt.nix
Normal file
13
treefmt.nix
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{ ... }:
|
||||
{
|
||||
projectRootFile = "flake.nix";
|
||||
|
||||
programs.nixfmt.enable = true;
|
||||
|
||||
programs.black.enable = true;
|
||||
|
||||
programs.isort = {
|
||||
enable = true;
|
||||
profile = "black";
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue