switch to uv and to nix flakes
This commit is contained in:
parent
14005f36ce
commit
b1bdef2d5d
20 changed files with 1522 additions and 1751 deletions
5
.envrc
5
.envrc
|
|
@ -1,2 +1,3 @@
|
||||||
use nix
|
use flake
|
||||||
#dotenv
|
dotenv_if_exists
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
# republisher-redux
|
# republisher-redux
|
||||||
|
|
||||||
``` shell
|
``` shell
|
||||||
mkdir logs out
|
mkdir -p logs out
|
||||||
poetry install
|
nix develop
|
||||||
poetry run repub
|
uv sync --all-groups
|
||||||
|
uv run repub
|
||||||
```
|
```
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
|
||||||
115
flake.lock
generated
Normal file
115
flake.lock
generated
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1774386573,
|
||||||
|
"narHash": "sha256-4hAV26quOxdC6iyG7kYaZcM3VOskcPUrdCQd/nx8obc=",
|
||||||
|
"rev": "46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9",
|
||||||
|
"revCount": 969196,
|
||||||
|
"type": "tarball",
|
||||||
|
"url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.969196%2Brev-46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9/019d279e-af65-79ce-92be-5dee7b1e36d4/source.tar.gz"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"type": "tarball",
|
||||||
|
"url": "https://flakehub.com/f/NixOS/nixpkgs/0.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pyproject-build-systems": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs"
|
||||||
|
],
|
||||||
|
"pyproject-nix": [
|
||||||
|
"pyproject-nix"
|
||||||
|
],
|
||||||
|
"uv2nix": [
|
||||||
|
"uv2nix"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1773870109,
|
||||||
|
"narHash": "sha256-ZoTdqZP03DcdoyxvpFHCAek4bkPUTUPUF3oCCgc3dP4=",
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "build-system-pkgs",
|
||||||
|
"rev": "b6e74f433b02fa4b8a7965ee24680f4867e2926f",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "build-system-pkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pyproject-nix": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1774498001,
|
||||||
|
"narHash": "sha256-wTfdyzzrmpuqt4TQQNqilF91v0m5Mh1stNy9h7a/WK4=",
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "pyproject.nix",
|
||||||
|
"rev": "794afa6eb588b498344f2eaa36ab1ceb7e6b0b09",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "pyproject.nix",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": "nixpkgs",
|
||||||
|
"pyproject-build-systems": "pyproject-build-systems",
|
||||||
|
"pyproject-nix": "pyproject-nix",
|
||||||
|
"treefmt-nix": "treefmt-nix",
|
||||||
|
"uv2nix": "uv2nix"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"treefmt-nix": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1774781724,
|
||||||
|
"narHash": "sha256-81GlxqpDZroeH6f+GZEM7V3VEqlLA4U/jU5e5L2yM0Y=",
|
||||||
|
"path": "/home/abel/src/github.com/numtide/treefmt-nix",
|
||||||
|
"type": "path"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"path": "/home/abel/src/github.com/numtide/treefmt-nix",
|
||||||
|
"type": "path"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"uv2nix": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs"
|
||||||
|
],
|
||||||
|
"pyproject-nix": [
|
||||||
|
"pyproject-nix"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1774705889,
|
||||||
|
"narHash": "sha256-TRTIM18gP3ccBj3m8bV1zx82xeYweNYp8/lgcdR4Zz0=",
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "uv2nix",
|
||||||
|
"rev": "28355ed75b466a15ff324e1baa151b550619fe67",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "pyproject-nix",
|
||||||
|
"repo": "uv2nix",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
246
flake.nix
Normal file
246
flake.nix
Normal file
|
|
@ -0,0 +1,246 @@
|
||||||
|
{
|
||||||
|
description = "republisher-redux - offline RSS and Atom feed mirroring";
|
||||||
|
|
||||||
|
inputs = {
|
||||||
|
nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.1";
|
||||||
|
treefmt-nix = {
|
||||||
|
url = "path:/home/abel/src/github.com/numtide/treefmt-nix";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
|
pyproject-nix = {
|
||||||
|
url = "github:pyproject-nix/pyproject.nix";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
|
uv2nix = {
|
||||||
|
url = "github:pyproject-nix/uv2nix";
|
||||||
|
inputs.pyproject-nix.follows = "pyproject-nix";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
|
pyproject-build-systems = {
|
||||||
|
url = "github:pyproject-nix/build-system-pkgs";
|
||||||
|
inputs.pyproject-nix.follows = "pyproject-nix";
|
||||||
|
inputs.uv2nix.follows = "uv2nix";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs =
|
||||||
|
{
|
||||||
|
self,
|
||||||
|
nixpkgs,
|
||||||
|
treefmt-nix,
|
||||||
|
pyproject-nix,
|
||||||
|
uv2nix,
|
||||||
|
pyproject-build-systems,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
systems = [ "x86_64-linux" ];
|
||||||
|
forAllSystems =
|
||||||
|
fn:
|
||||||
|
nixpkgs.lib.genAttrs systems (
|
||||||
|
system:
|
||||||
|
fn (
|
||||||
|
import nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
config.allowUnfree = true;
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
mkTreefmtConfig = pkgs: (treefmt-nix.lib.evalModule pkgs ./treefmt.nix).config;
|
||||||
|
|
||||||
|
workspace = uv2nix.lib.workspace.loadWorkspace { workspaceRoot = ./.; };
|
||||||
|
overlay = workspace.mkPyprojectOverlay { sourcePreference = "wheel"; };
|
||||||
|
pyprojectOverrides = final: prev: {
|
||||||
|
sgmllib3k = prev.sgmllib3k.overrideAttrs (old: {
|
||||||
|
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ final.setuptools ];
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
mkPackage =
|
||||||
|
pkgs:
|
||||||
|
let
|
||||||
|
ffmpegPackage = pkgs.ffmpeg-full;
|
||||||
|
|
||||||
|
pythonSet =
|
||||||
|
(pkgs.callPackage pyproject-nix.build.packages {
|
||||||
|
python = pkgs.python313;
|
||||||
|
}).overrideScope
|
||||||
|
(
|
||||||
|
pkgs.lib.composeManyExtensions [
|
||||||
|
pyproject-build-systems.overlays.default
|
||||||
|
overlay
|
||||||
|
pyprojectOverrides
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
baseVenv = pythonSet.mkVirtualEnv "republisher-redux-env" workspace.deps.default;
|
||||||
|
testVenv = pythonSet.mkVirtualEnv "republisher-redux-test-env" {
|
||||||
|
"republisher-redux" = [ "dev" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
tests = pkgs.stdenv.mkDerivation {
|
||||||
|
name = "republisher-redux-tests";
|
||||||
|
src = ./.;
|
||||||
|
dontConfigure = true;
|
||||||
|
dontBuild = true;
|
||||||
|
nativeBuildInputs = [ testVenv ];
|
||||||
|
checkPhase = ''
|
||||||
|
runHook preCheck
|
||||||
|
export HOME="$(mktemp -d)"
|
||||||
|
pytest tests/ -v
|
||||||
|
runHook postCheck
|
||||||
|
'';
|
||||||
|
doCheck = true;
|
||||||
|
installPhase = ''
|
||||||
|
mkdir -p "$out"
|
||||||
|
touch "$out/passed"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
runtimePackage = pkgs.symlinkJoin {
|
||||||
|
name = "republisher-redux";
|
||||||
|
paths = [ baseVenv ];
|
||||||
|
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||||
|
postBuild = ''
|
||||||
|
rm -f "$out/bin/repub"
|
||||||
|
makeWrapper "${baseVenv}/bin/repub" "$out/bin/repub" \
|
||||||
|
--prefix PATH : "${pkgs.lib.makeBinPath [ ffmpegPackage ]}"
|
||||||
|
'';
|
||||||
|
meta.mainProgram = "repub";
|
||||||
|
};
|
||||||
|
in
|
||||||
|
pkgs.runCommand "republisher-redux"
|
||||||
|
{
|
||||||
|
inherit (runtimePackage) meta;
|
||||||
|
passthru = {
|
||||||
|
inherit tests testVenv runtimePackage;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
''
|
||||||
|
test -f "${tests}/passed"
|
||||||
|
ln -s "${runtimePackage}" "$out"
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
{
|
||||||
|
formatter = forAllSystems (pkgs: (mkTreefmtConfig pkgs).build.wrapper);
|
||||||
|
|
||||||
|
packages = forAllSystems (
|
||||||
|
pkgs:
|
||||||
|
let
|
||||||
|
pkg = mkPackage pkgs;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
"republisher-redux" = pkg;
|
||||||
|
default = pkg;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
apps = forAllSystems (
|
||||||
|
pkgs:
|
||||||
|
let
|
||||||
|
package = self.packages.${pkgs.stdenv.hostPlatform.system}.default;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
repub = {
|
||||||
|
type = "app";
|
||||||
|
program = "${package}/bin/repub";
|
||||||
|
meta.description = "republisher-redux runtime";
|
||||||
|
};
|
||||||
|
default = {
|
||||||
|
type = "app";
|
||||||
|
program = "${package}/bin/repub";
|
||||||
|
meta.description = "republisher-redux runtime";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
checks = forAllSystems (
|
||||||
|
pkgs:
|
||||||
|
let
|
||||||
|
system = pkgs.stdenv.hostPlatform.system;
|
||||||
|
exportedPackage = self.packages.${system}.default;
|
||||||
|
testVenv = exportedPackage.testVenv;
|
||||||
|
treefmtConfig = mkTreefmtConfig pkgs;
|
||||||
|
src = ./.;
|
||||||
|
blackCheck = pkgs.stdenv.mkDerivation {
|
||||||
|
name = "republisher-redux-black";
|
||||||
|
inherit src;
|
||||||
|
dontConfigure = true;
|
||||||
|
dontBuild = true;
|
||||||
|
nativeBuildInputs = [ testVenv ];
|
||||||
|
checkPhase = ''
|
||||||
|
runHook preCheck
|
||||||
|
black --check repub/ tests/
|
||||||
|
runHook postCheck
|
||||||
|
'';
|
||||||
|
doCheck = true;
|
||||||
|
installPhase = ''
|
||||||
|
mkdir -p "$out"
|
||||||
|
touch "$out/passed"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
flake8Check = pkgs.stdenv.mkDerivation {
|
||||||
|
name = "republisher-redux-flake8";
|
||||||
|
inherit src;
|
||||||
|
dontConfigure = true;
|
||||||
|
dontBuild = true;
|
||||||
|
nativeBuildInputs = [ testVenv ];
|
||||||
|
checkPhase = ''
|
||||||
|
runHook preCheck
|
||||||
|
flake8 repub/ tests/
|
||||||
|
runHook postCheck
|
||||||
|
'';
|
||||||
|
doCheck = true;
|
||||||
|
installPhase = ''
|
||||||
|
mkdir -p "$out"
|
||||||
|
touch "$out/passed"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
isortCheck = pkgs.stdenv.mkDerivation {
|
||||||
|
name = "republisher-redux-isort";
|
||||||
|
inherit src;
|
||||||
|
dontConfigure = true;
|
||||||
|
dontBuild = true;
|
||||||
|
nativeBuildInputs = [ testVenv ];
|
||||||
|
checkPhase = ''
|
||||||
|
runHook preCheck
|
||||||
|
isort --check-only repub/ tests/
|
||||||
|
runHook postCheck
|
||||||
|
'';
|
||||||
|
doCheck = true;
|
||||||
|
installPhase = ''
|
||||||
|
mkdir -p "$out"
|
||||||
|
touch "$out/passed"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
devshell-default = self.devShells.${system}.default;
|
||||||
|
formatter = treefmtConfig.build.wrapper;
|
||||||
|
package-default = exportedPackage;
|
||||||
|
tests = exportedPackage.tests;
|
||||||
|
treefmt = treefmtConfig.build.check ./.;
|
||||||
|
black = blackCheck;
|
||||||
|
flake8 = flake8Check;
|
||||||
|
isort = isortCheck;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
devShells = forAllSystems (pkgs: {
|
||||||
|
default = pkgs.mkShell {
|
||||||
|
packages = [
|
||||||
|
pkgs.python313
|
||||||
|
pkgs.uv
|
||||||
|
pkgs.ffmpeg-full
|
||||||
|
];
|
||||||
|
env.LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
|
||||||
|
pkgs.stdenv.cc.cc
|
||||||
|
];
|
||||||
|
env.UV_PROJECT_ENVIRONMENT = ".venv";
|
||||||
|
env.UV_PYTHON_DOWNLOADS = "never";
|
||||||
|
};
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
1631
poetry.lock
generated
1631
poetry.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,71 +1,70 @@
|
||||||
[tool.poetry]
|
[project]
|
||||||
name = "repub"
|
name = "republisher-redux"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = ""
|
description = "Mirror RSS and Atom feeds completely offline"
|
||||||
authors = ["Abel Luck <abel@guardianproject.info>"]
|
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
#packages = [{include = "repub", from = "repub"}]
|
authors = [{ name = "Abel Luck", email = "abel@guardianproject.info" }]
|
||||||
[tool.poetry.scripts]
|
requires-python = ">=3.13"
|
||||||
|
dependencies = [
|
||||||
|
"scrapy>=2.11.1,<3.0.0",
|
||||||
|
"prometheus-client>=0.20.0,<0.21.0",
|
||||||
|
"python-dateutil>=2.9.0.post0,<3.0.0",
|
||||||
|
"colorlog>=6.8.2,<7.0.0",
|
||||||
|
"feedparser>=6.0.11,<7.0.0",
|
||||||
|
"lxml>=5.2.1,<6.0.0",
|
||||||
|
"pillow>=10.3.0,<11.0.0",
|
||||||
|
"ffmpeg-python>=0.2.0,<0.3.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
repub = "repub.entrypoint:entrypoint"
|
repub = "repub.entrypoint:entrypoint"
|
||||||
|
|
||||||
[tool.poetry.dependencies]
|
[dependency-groups]
|
||||||
python = "^3.11"
|
dev = [
|
||||||
scrapy = "^2.11.1"
|
"pytest>=8.1.1,<9.0.0",
|
||||||
prometheus-client = "^0.20.0"
|
"black>=24.4.0,<25.0.0",
|
||||||
python-dateutil = "^2.9.0.post0"
|
"flake8>=7.0.0,<8.0.0",
|
||||||
colorlog = "^6.8.2"
|
"mypy>=1.9.0,<2.0.0",
|
||||||
feedparser = "^6.0.11"
|
"bandit>=1.7.8,<2.0.0",
|
||||||
lxml = "^5.2.1"
|
"types-PyYAML>=6.0.12.20240311,<7.0.0",
|
||||||
pillow = "^10.3.0"
|
"isort>=5.13.2,<6.0.0",
|
||||||
ffmpeg-python = "^0.2.0"
|
"flake8-black>=0.3.6,<0.4.0",
|
||||||
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["poetry-core"]
|
requires = ["setuptools>=68", "wheel"]
|
||||||
build-backend = "poetry.core.masonry.api"
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[tool.setuptools]
|
||||||
|
include-package-data = true
|
||||||
|
|
||||||
[tool.poetry.dev-dependencies]
|
[tool.setuptools.packages.find]
|
||||||
pytest = "^8.1.1"
|
where = ["."]
|
||||||
black = "^24.4.0"
|
include = ["repub*"]
|
||||||
flake8 = "^7.0.0"
|
|
||||||
mypy = "^1.9.0"
|
[tool.pytest.ini_options]
|
||||||
bandit = "^1.7.8"
|
testpaths = ["tests"]
|
||||||
types-PyYAML = "^6.0.12.20240311"
|
|
||||||
isort = "^5.13.2"
|
|
||||||
flake8-black = "^0.3.6"
|
|
||||||
|
|
||||||
[tool.isort]
|
[tool.isort]
|
||||||
py_version = 310
|
py_version = 311
|
||||||
profile = "black"
|
profile = "black"
|
||||||
src_paths = ["src", "tests"]
|
src_paths = ["repub", "tests"]
|
||||||
|
|
||||||
[tool.black]
|
[tool.black]
|
||||||
line-length = 88
|
line-length = 88
|
||||||
target-version = ['py310']
|
target-version = ['py313']
|
||||||
|
|
||||||
[tool.mypy]
|
[tool.mypy]
|
||||||
files = "gm,tests"
|
files = "repub,tests"
|
||||||
ignore_missing_imports = true
|
ignore_missing_imports = true
|
||||||
follow_imports = "normal"
|
follow_imports = "normal"
|
||||||
# Ensure full coverage
|
|
||||||
disallow_untyped_calls = true
|
disallow_untyped_calls = true
|
||||||
#disallow_untyped_defs = true
|
|
||||||
#disallow_incomplete_defs = true
|
|
||||||
#disallow_untyped_decorators = true
|
|
||||||
#check_untyped_defs = true
|
|
||||||
|
|
||||||
# Restrict dynamic typing
|
|
||||||
disallow_any_generics = true
|
disallow_any_generics = true
|
||||||
disallow_subclassing_any = true
|
disallow_subclassing_any = true
|
||||||
warn_return_any = true
|
warn_return_any = true
|
||||||
|
|
||||||
# Know exactly what you're doing
|
|
||||||
warn_redundant_casts = true
|
warn_redundant_casts = true
|
||||||
warn_unused_ignores = true
|
warn_unused_ignores = true
|
||||||
warn_unused_configs = true
|
warn_unused_configs = true
|
||||||
warn_unreachable = true
|
warn_unreachable = true
|
||||||
show_error_codes = true
|
show_error_codes = true
|
||||||
|
|
||||||
# Explicit is better than implici
|
|
||||||
no_implicit_optional = true
|
no_implicit_optional = true
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import copy
|
import copy
|
||||||
|
|
||||||
import scrapy.utils.log
|
import scrapy.utils.log
|
||||||
|
|
||||||
from colorlog import ColoredFormatter
|
from colorlog import ColoredFormatter
|
||||||
|
|
||||||
color_formatter = ColoredFormatter(
|
color_formatter = ColoredFormatter(
|
||||||
|
|
|
||||||
|
|
@ -25,12 +25,13 @@ class FeedNameFilter:
|
||||||
|
|
||||||
|
|
||||||
def execute_spider(queue, name, url):
|
def execute_spider(queue, name, url):
|
||||||
from repub.media import check_runtime
|
|
||||||
from repub.spiders.rss_spider import RssFeedSpider
|
|
||||||
from scrapy.crawler import CrawlerProcess
|
from scrapy.crawler import CrawlerProcess
|
||||||
from scrapy.settings import Settings
|
from scrapy.settings import Settings
|
||||||
from scrapy.utils.project import get_project_settings
|
from scrapy.utils.project import get_project_settings
|
||||||
|
|
||||||
|
from repub.media import check_runtime
|
||||||
|
from repub.spiders.rss_spider import RssFeedSpider
|
||||||
|
|
||||||
try:
|
try:
|
||||||
settings: Settings = {
|
settings: Settings = {
|
||||||
**get_project_settings(),
|
**get_project_settings(),
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from repub import rss
|
|
||||||
from scrapy.exporters import BaseItemExporter
|
from scrapy.exporters import BaseItemExporter
|
||||||
|
|
||||||
from .exceptions import *
|
from repub import rss
|
||||||
|
|
||||||
from .items import ChannelElementItem
|
from .items import ChannelElementItem
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -345,7 +345,7 @@ def transcode_video(input_file: str, output_dir: str, params: Dict[str, Any]) ->
|
||||||
return output_file
|
return output_file
|
||||||
except ffmpeg.Error as e:
|
except ffmpeg.Error as e:
|
||||||
print(e.stderr, file=sys.stderr)
|
print(e.stderr, file=sys.stderr)
|
||||||
logger.error(f"Failed to transcode")
|
logger.error("Failed to transcode")
|
||||||
logger.error(e)
|
logger.error(e)
|
||||||
raise RuntimeError(f"Failed to transcode video: {e.stderr.decode()}") from e
|
raise RuntimeError(f"Failed to transcode video: {e.stderr.decode()}") from e
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,6 @@
|
||||||
# See documentation in:
|
# See documentation in:
|
||||||
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||||
|
|
||||||
# useful for handling different item types with a single interface
|
|
||||||
from itemadapter import ItemAdapter, is_item
|
|
||||||
from scrapy import signals
|
from scrapy import signals
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,13 +4,14 @@ from io import BytesIO
|
||||||
from os import PathLike
|
from os import PathLike
|
||||||
from typing import Dict, List, Optional, Union
|
from typing import Dict, List, Optional, Union
|
||||||
|
|
||||||
import repub.utils
|
|
||||||
from repub import media
|
|
||||||
from scrapy.pipelines.files import FilesPipeline as BaseFilesPipeline
|
from scrapy.pipelines.files import FilesPipeline as BaseFilesPipeline
|
||||||
from scrapy.pipelines.images import ImagesPipeline as BaseImagesPipeline
|
from scrapy.pipelines.images import ImagesPipeline as BaseImagesPipeline
|
||||||
from scrapy.settings import Settings
|
from scrapy.settings import Settings
|
||||||
from scrapy.utils.misc import md5sum
|
from scrapy.utils.misc import md5sum
|
||||||
|
|
||||||
|
import repub.utils
|
||||||
|
from repub import media
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
from typing import List, Tuple
|
from datetime import datetime
|
||||||
|
from time import mktime
|
||||||
|
|
||||||
import lxml.etree as ET
|
import lxml.etree as ET
|
||||||
import lxml.html
|
import lxml.html
|
||||||
|
|
@ -53,9 +54,6 @@ ATOM = SafeElementMaker(nsmap={None: nsmap["atom"]}, namespace=nsmap["atom"])
|
||||||
E: ElementMaker = SafeElementMaker(nsmap=nsmap)
|
E: ElementMaker = SafeElementMaker(nsmap=nsmap)
|
||||||
CDATA = ET.CDATA
|
CDATA = ET.CDATA
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
from time import mktime
|
|
||||||
|
|
||||||
|
|
||||||
def rss():
|
def rss():
|
||||||
return E.rss({"version": "2.0"})
|
return E.rss({"version": "2.0"})
|
||||||
|
|
|
||||||
|
|
@ -2,13 +2,14 @@ import logging
|
||||||
from typing import Dict, List, Tuple
|
from typing import Dict, List, Tuple
|
||||||
|
|
||||||
import feedparser
|
import feedparser
|
||||||
from repub.items import ChannelElementItem, ElementItem
|
|
||||||
from repub.rss import CDATA, CONTENT, ITUNES, MEDIA, E, munge_cdata_html, normalize_date
|
|
||||||
from repub.utils import FileType, determine_file_type, local_file_path, local_image_path
|
|
||||||
from scrapy.crawler import Crawler
|
from scrapy.crawler import Crawler
|
||||||
from scrapy.spiders import Spider
|
from scrapy.spiders import Spider
|
||||||
from scrapy.utils.spider import iterate_spider_output
|
from scrapy.utils.spider import iterate_spider_output
|
||||||
|
|
||||||
|
from repub.items import ChannelElementItem, ElementItem
|
||||||
|
from repub.rss import CDATA, CONTENT, ITUNES, MEDIA, E, munge_cdata_html, normalize_date
|
||||||
|
from repub.utils import FileType, determine_file_type, local_file_path
|
||||||
|
|
||||||
|
|
||||||
class BaseRssFeedSpider(Spider):
|
class BaseRssFeedSpider(Spider):
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
||||||
import math
|
import math
|
||||||
|
|
||||||
# See https://infra.spec.whatwg.org/#ascii-whitespace
|
# See https://infra.spec.whatwg.org/#ascii-whitespace
|
||||||
WHITESPACES = ("\u0009", "\u000A", "\u000C", "\u000D", "\u0020") # \t # " "
|
WHITESPACES = ("\u0009", "\u000a", "\u000c", "\u000d", "\u0020") # \t # " "
|
||||||
|
|
||||||
STATE_IN_DESCRIPTOR = 1
|
STATE_IN_DESCRIPTOR = 1
|
||||||
STATE_AFTER_DESCRIPTOR = 2
|
STATE_AFTER_DESCRIPTOR = 2
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ import hashlib
|
||||||
import mimetypes
|
import mimetypes
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, List, Optional
|
from typing import Optional
|
||||||
|
|
||||||
from scrapy.utils.python import to_bytes
|
from scrapy.utils.python import to_bytes
|
||||||
|
|
||||||
|
|
|
||||||
54
shell.nix
54
shell.nix
|
|
@ -1,54 +0,0 @@
|
||||||
{
|
|
||||||
system ? "x86_64-linux",
|
|
||||||
pkgs ? import <nixpkgs> { inherit system; },
|
|
||||||
dev ? true,
|
|
||||||
}:
|
|
||||||
|
|
||||||
let
|
|
||||||
pyCurrent = pkgs.python311;
|
|
||||||
poetryExtras = if dev then [ "dev" ] else [ ];
|
|
||||||
poetryInstallExtras = (
|
|
||||||
if poetryExtras == [ ] then
|
|
||||||
""
|
|
||||||
else
|
|
||||||
pkgs.lib.concatStrings [
|
|
||||||
" --with="
|
|
||||||
(pkgs.lib.concatStringsSep "," poetryExtras)
|
|
||||||
]
|
|
||||||
);
|
|
||||||
packages = [
|
|
||||||
(pkgs.ffmpeg_5-full.override {
|
|
||||||
withUnfree = true;
|
|
||||||
withFdkAac = true;
|
|
||||||
})
|
|
||||||
#(pyCurrent (ps: with ps; [ ffmpeg-python ]))
|
|
||||||
pkgs.zsh
|
|
||||||
(pkgs.poetry.withPlugins (ps: with ps; [ poetry-plugin-up ]))
|
|
||||||
];
|
|
||||||
|
|
||||||
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
|
|
||||||
pkgs.stdenv.cc.cc
|
|
||||||
# Add any missing library needed
|
|
||||||
# You can use the nix-index package to locate them, e.g. nix-locate -w --top-level --at-root /lib/libudev.so.1
|
|
||||||
];
|
|
||||||
|
|
||||||
# Put the venv on the repo, so direnv can access it
|
|
||||||
POETRY_VIRTUALENVS_IN_PROJECT = "true";
|
|
||||||
POETRY_VIRTUALENVS_PATH = "{project-dir}/.venv";
|
|
||||||
|
|
||||||
# Use python from path, so you can use a different version to the one bundled with poetry
|
|
||||||
POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON = "true";
|
|
||||||
in
|
|
||||||
pkgs.mkShell {
|
|
||||||
buildInputs = packages;
|
|
||||||
shellHook = ''
|
|
||||||
export SHELL=${pkgs.zsh}
|
|
||||||
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}"
|
|
||||||
export POETRY_VIRTUALENVS_IN_PROJECT="${POETRY_VIRTUALENVS_IN_PROJECT}"
|
|
||||||
export POETRY_VIRTUALENVS_PATH="${POETRY_VIRTUALENVS_PATH}"
|
|
||||||
export POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON="${POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON}"
|
|
||||||
export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
|
|
||||||
poetry env use "${pyCurrent}/bin/python"
|
|
||||||
poetry install -vv --sync${poetryInstallExtras}
|
|
||||||
'';
|
|
||||||
}
|
|
||||||
17
tests/test_entrypoint.py
Normal file
17
tests/test_entrypoint.py
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from repub.entrypoint import FeedNameFilter
|
||||||
|
|
||||||
|
|
||||||
|
def test_feed_name_filter_accepts_matching_item() -> None:
|
||||||
|
item = SimpleNamespace(feed_name="nasa")
|
||||||
|
feed_filter = FeedNameFilter({"feed_name": "nasa"})
|
||||||
|
|
||||||
|
assert feed_filter.accepts(item) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_feed_name_filter_rejects_non_matching_item() -> None:
|
||||||
|
item = SimpleNamespace(feed_name="gp-pod")
|
||||||
|
feed_filter = FeedNameFilter({"feed_name": "nasa"})
|
||||||
|
|
||||||
|
assert feed_filter.accepts(item) is False
|
||||||
13
treefmt.nix
Normal file
13
treefmt.nix
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
{ ... }:
|
||||||
|
{
|
||||||
|
projectRootFile = "flake.nix";
|
||||||
|
|
||||||
|
programs.nixfmt.enable = true;
|
||||||
|
|
||||||
|
programs.black.enable = true;
|
||||||
|
|
||||||
|
programs.isort = {
|
||||||
|
enable = true;
|
||||||
|
profile = "black";
|
||||||
|
};
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue