switch to uv and to nix flakes

.envrc

@@ -1,2 +1,3 @@
-use nix
+use flake
-#dotenv
+dotenv_if_exists
+

.gitignore

@@ -4,3 +4,10 @@ __pycache__
 .vscode
 .mypy_cache
 .direnv
+.scrapy
+out
+tmp/
+/test*py
+data
+logs
+archive

LICENSE.md

@@ -0,0 +1,660 @@
+# GNU AFFERO GENERAL PUBLIC LICENSE
+
+Version 3, 19 November 2007
+
+Copyright (C) 2007 Free Software Foundation, Inc.
+<https://fsf.org/>
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+## Preamble
+
+The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains
+free software for all its users.
+
+When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing
+under this license.
+
+The precise terms and conditions for copying, distribution and
+modification follow.
+
+## TERMS AND CONDITIONS
+
+### 0. Definitions.
+
+"This License" refers to version 3 of the GNU Affero General Public
+License.
+
+"Copyright" also means copyright-like laws that apply to other kinds
+of works, such as semiconductor masks.
+
+"The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of
+an exact copy. The resulting work is called a "modified version" of
+the earlier work or a work "based on" the earlier work.
+
+A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user
+through a computer network, with no transfer of a copy, is not
+conveying.
+
+An interactive user interface displays "Appropriate Legal Notices" to
+the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+### 1. Source Code.
+
+The "source code" for a work means the preferred form of the work for
+making modifications to it. "Object code" means any non-source form of
+a work.
+
+A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+The Corresponding Source need not include anything that users can
+regenerate automatically from other parts of the Corresponding Source.
+
+The Corresponding Source for a work in source code form is that same
+work.
+
+### 2. Basic Permissions.
+
+All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+You may make, run and propagate covered works that you do not convey,
+without conditions so long as your license otherwise remains in force.
+You may convey covered works to others for the sole purpose of having
+them make modifications exclusively for you, or provide you with
+facilities for running those works, provided that you comply with the
+terms of this License in conveying all material for which you do not
+control copyright. Those thus making or running the covered works for
+you must do so exclusively on your behalf, under your direction and
+control, on terms that prohibit them from making any copies of your
+copyrighted material outside their relationship with you.
+
+Conveying under any other circumstances is permitted solely under the
+conditions stated below. Sublicensing is not allowed; section 10 makes
+it unnecessary.
+
+### 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such
+circumvention is effected by exercising rights under this License with
+respect to the covered work, and you disclaim any intention to limit
+operation or modification of the work as a means of enforcing, against
+the work's users, your or third parties' legal rights to forbid
+circumvention of technological measures.
+
+### 4. Conveying Verbatim Copies.
+
+You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+### 5. Conveying Modified Source Versions.
+
+You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these
+conditions:
+
+-   a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+-   b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under
+    section 7. This requirement modifies the requirement in section 4
+    to "keep intact all notices".
+-   c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy. This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged. This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+-   d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+### 6. Conveying Non-Source Forms.
+
+You may convey a covered work in object code form under the terms of
+sections 4 and 5, provided that you also convey the machine-readable
+Corresponding Source under the terms of this License, in one of these
+ways:
+
+-   a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+-   b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the Corresponding
+    Source from a network server at no charge.
+-   c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source. This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+-   d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge. You need not require recipients to copy the
+    Corresponding Source along with the object code. If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source. Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+-   e) Convey the object code using peer-to-peer transmission,
+    provided you inform other peers where the object code and
+    Corresponding Source of the work are being offered to the general
+    public at no charge under subsection 6d.
+
+A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal,
+family, or household purposes, or (2) anything designed or sold for
+incorporation into a dwelling. In determining whether a product is a
+consumer product, doubtful cases shall be resolved in favor of
+coverage. For a particular product received by a particular user,
+"normally used" refers to a typical or common use of that class of
+product, regardless of the status of the particular user or of the way
+in which the particular user actually uses, or expects or is expected
+to use, the product. A product is a consumer product regardless of
+whether the product has substantial commercial, industrial or
+non-consumer uses, unless such uses represent the only significant
+mode of use of the product.
+
+"Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to
+install and execute modified versions of a covered work in that User
+Product from a modified version of its Corresponding Source. The
+information must suffice to ensure that the continued functioning of
+the modified object code is in no case prevented or interfered with
+solely because modification has been made.
+
+If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or
+updates for a work that has been modified or installed by the
+recipient, or for the User Product in which it has been modified or
+installed. Access to a network may be denied when the modification
+itself materially and adversely affects the operation of the network
+or violates the rules and protocols for communication across the
+network.
+
+Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+### 7. Additional Terms.
+
+"Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders
+of that material) supplement the terms of this License with terms:
+
+-   a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+-   b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+-   c) Prohibiting misrepresentation of the origin of that material,
+    or requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+-   d) Limiting the use for publicity purposes of names of licensors
+    or authors of the material; or
+-   e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+-   f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions
+    of it) with contractual assumptions of liability to the recipient,
+    for any liability that these contractual assumptions directly
+    impose on those licensors and authors.
+
+All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions; the
+above requirements apply either way.
+
+### 8. Termination.
+
+You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+However, if you cease all violation of this License, then your license
+from a particular copyright holder is reinstated (a) provisionally,
+unless and until the copyright holder explicitly and finally
+terminates your license, and (b) permanently, if the copyright holder
+fails to notify you of the violation by some reasonable means prior to
+60 days after the cessation.
+
+Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+### 9. Acceptance Not Required for Having Copies.
+
+You are not required to accept this License in order to receive or run
+a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+### 10. Automatic Licensing of Downstream Recipients.
+
+Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+### 11. Patents.
+
+A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+A contributor's "essential patent claims" are all patent claims owned
+or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+A patent license is "discriminatory" if it does not include within the
+scope of its coverage, prohibits the exercise of, or is conditioned on
+the non-exercise of one or more of the rights that are specifically
+granted under this License. You may not convey a covered work if you
+are a party to an arrangement with a third party that is in the
+business of distributing software, under which you make payment to the
+third party based on the extent of your activity of conveying the
+work, and under which the third party grants, to any of the parties
+who would receive the covered work from you, a discriminatory patent
+license (a) in connection with copies of the covered work conveyed by
+you (or copies made from those copies), or (b) primarily for and in
+connection with specific products or compilations that contain the
+covered work, unless you entered into that arrangement, or that patent
+license was granted, prior to 28 March 2007.
+
+Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+### 12. No Surrender of Others' Freedom.
+
+If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under
+this License and any other pertinent obligations, then as a
+consequence you may not convey it at all. For example, if you agree to
+terms that obligate you to collect a royalty for further conveying
+from those to whom you convey the Program, the only way you could
+satisfy both those terms and this License would be to refrain entirely
+from conveying the Program.
+
+### 13. Remote Network Interaction; Use with the GNU General Public License.
+
+Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your
+version supports such interaction) an opportunity to receive the
+Corresponding Source of your version by providing access to the
+Corresponding Source from a network server at no charge, through some
+standard or customary means of facilitating copying of software. This
+Corresponding Source shall include the Corresponding Source for any
+work covered by version 3 of the GNU General Public License that is
+incorporated pursuant to the following paragraph.
+
+Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+### 14. Revised Versions of this License.
+
+The Free Software Foundation may publish revised and/or new versions
+of the GNU Affero General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever
+published by the Free Software Foundation.
+
+If the Program specifies that a proxy can decide which future versions
+of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+### 15. Disclaimer of Warranty.
+
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT
+WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
+PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE
+DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR
+CORRECTION.
+
+### 16. Limitation of Liability.
+
+IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR
+CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT
+NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR
+LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM
+TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER
+PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+### 17. Interpretation of Sections 15 and 16.
+
+If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+END OF TERMS AND CONDITIONS
+
+## How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these
+terms.
+
+To do so, attach the following notices to the program. It is safest to
+attach them to the start of each source file to most effectively state
+the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+        <one line to give the program's name and a brief idea of what it does.>
+        Copyright (C) <year>  <name of author>
+
+        This program is free software: you can redistribute it and/or modify
+        it under the terms of the GNU Affero General Public License as
+        published by the Free Software Foundation, either version 3 of the
+        License, or (at your option) any later version.
+
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU Affero General Public License for more details.
+
+        You should have received a copy of the GNU Affero General Public License
+        along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper
+mail.
+
+If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for
+the specific requirements.
+
+You should also get your employer (if you work as a programmer) or
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. For more information on this, and how to apply and follow
+the GNU AGPL, see <https://www.gnu.org/licenses/>.

README.md

@@ -0,0 +1,41 @@
+# republisher-redux
+
+``` shell
+mkdir -p logs out
+nix develop
+uv sync --all-groups
+uv run repub
+```
+
+## TODO
+
+- [x] Offlines RSS feed xml
+- [x] Downloads media and enclosures
+- [x] Rewrites media urls
+- [x] Image normalization (JPG, RGB)
+- [x] Audio transcoding
+- [x] Video transcoding
+- [ ] Image compression - Do we want this?
+- [x] Download and rewrite media embedded in content/CDATA fields
+- [ ] Config file to drive the program
+- [ ] Daemonize the program
+- [ ] Operationalize with metrics and error reporting
+
+## License
+
+republisher-redux, a tool to mirror RSS/ATOM feeds completely offline
+
+Copyright (C) 2024 Abel Luck
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

flake.lock

@@ -0,0 +1,115 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1774386573,
+        "narHash": "sha256-4hAV26quOxdC6iyG7kYaZcM3VOskcPUrdCQd/nx8obc=",
+        "rev": "46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9",
+        "revCount": 969196,
+        "type": "tarball",
+        "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.969196%2Brev-46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9/019d279e-af65-79ce-92be-5dee7b1e36d4/source.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://flakehub.com/f/NixOS/nixpkgs/0.1"
+      }
+    },
+    "pyproject-build-systems": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "pyproject-nix": [
+          "pyproject-nix"
+        ],
+        "uv2nix": [
+          "uv2nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1773870109,
+        "narHash": "sha256-ZoTdqZP03DcdoyxvpFHCAek4bkPUTUPUF3oCCgc3dP4=",
+        "owner": "pyproject-nix",
+        "repo": "build-system-pkgs",
+        "rev": "b6e74f433b02fa4b8a7965ee24680f4867e2926f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "build-system-pkgs",
+        "type": "github"
+      }
+    },
+    "pyproject-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1774498001,
+        "narHash": "sha256-wTfdyzzrmpuqt4TQQNqilF91v0m5Mh1stNy9h7a/WK4=",
+        "owner": "pyproject-nix",
+        "repo": "pyproject.nix",
+        "rev": "794afa6eb588b498344f2eaa36ab1ceb7e6b0b09",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "pyproject.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "pyproject-build-systems": "pyproject-build-systems",
+        "pyproject-nix": "pyproject-nix",
+        "treefmt-nix": "treefmt-nix",
+        "uv2nix": "uv2nix"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1774781724,
+        "narHash": "sha256-81GlxqpDZroeH6f+GZEM7V3VEqlLA4U/jU5e5L2yM0Y=",
+        "path": "/home/abel/src/github.com/numtide/treefmt-nix",
+        "type": "path"
+      },
+      "original": {
+        "path": "/home/abel/src/github.com/numtide/treefmt-nix",
+        "type": "path"
+      }
+    },
+    "uv2nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "pyproject-nix": [
+          "pyproject-nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1774705889,
+        "narHash": "sha256-TRTIM18gP3ccBj3m8bV1zx82xeYweNYp8/lgcdR4Zz0=",
+        "owner": "pyproject-nix",
+        "repo": "uv2nix",
+        "rev": "28355ed75b466a15ff324e1baa151b550619fe67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "uv2nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,246 @@
+{
+  description = "republisher-redux - offline RSS and Atom feed mirroring";
+
+  inputs = {
+    nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.1";
+    treefmt-nix = {
+      url = "path:/home/abel/src/github.com/numtide/treefmt-nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    pyproject-nix = {
+      url = "github:pyproject-nix/pyproject.nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    uv2nix = {
+      url = "github:pyproject-nix/uv2nix";
+      inputs.pyproject-nix.follows = "pyproject-nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    pyproject-build-systems = {
+      url = "github:pyproject-nix/build-system-pkgs";
+      inputs.pyproject-nix.follows = "pyproject-nix";
+      inputs.uv2nix.follows = "uv2nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+
+  outputs =
+    {
+      self,
+      nixpkgs,
+      treefmt-nix,
+      pyproject-nix,
+      uv2nix,
+      pyproject-build-systems,
+      ...
+    }:
+    let
+      systems = [ "x86_64-linux" ];
+      forAllSystems =
+        fn:
+        nixpkgs.lib.genAttrs systems (
+          system:
+          fn (
+            import nixpkgs {
+              inherit system;
+              config.allowUnfree = true;
+            }
+          )
+        );
+
+      mkTreefmtConfig = pkgs: (treefmt-nix.lib.evalModule pkgs ./treefmt.nix).config;
+
+      workspace = uv2nix.lib.workspace.loadWorkspace { workspaceRoot = ./.; };
+      overlay = workspace.mkPyprojectOverlay { sourcePreference = "wheel"; };
+      pyprojectOverrides = final: prev: {
+        sgmllib3k = prev.sgmllib3k.overrideAttrs (old: {
+          nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ final.setuptools ];
+        });
+      };
+
+      mkPackage =
+        pkgs:
+        let
+          ffmpegPackage = pkgs.ffmpeg-full;
+
+          pythonSet =
+            (pkgs.callPackage pyproject-nix.build.packages {
+              python = pkgs.python313;
+            }).overrideScope
+              (
+                pkgs.lib.composeManyExtensions [
+                  pyproject-build-systems.overlays.default
+                  overlay
+                  pyprojectOverrides
+                ]
+              );
+
+          baseVenv = pythonSet.mkVirtualEnv "republisher-redux-env" workspace.deps.default;
+          testVenv = pythonSet.mkVirtualEnv "republisher-redux-test-env" {
+            "republisher-redux" = [ "dev" ];
+          };
+
+          tests = pkgs.stdenv.mkDerivation {
+            name = "republisher-redux-tests";
+            src = ./.;
+            dontConfigure = true;
+            dontBuild = true;
+            nativeBuildInputs = [ testVenv ];
+            checkPhase = ''
+              runHook preCheck
+              export HOME="$(mktemp -d)"
+              pytest tests/ -v
+              runHook postCheck
+            '';
+            doCheck = true;
+            installPhase = ''
+              mkdir -p "$out"
+              touch "$out/passed"
+            '';
+          };
+
+          runtimePackage = pkgs.symlinkJoin {
+            name = "republisher-redux";
+            paths = [ baseVenv ];
+            nativeBuildInputs = [ pkgs.makeWrapper ];
+            postBuild = ''
+              rm -f "$out/bin/repub"
+              makeWrapper "${baseVenv}/bin/repub" "$out/bin/repub" \
+                --prefix PATH : "${pkgs.lib.makeBinPath [ ffmpegPackage ]}"
+            '';
+            meta.mainProgram = "repub";
+          };
+        in
+        pkgs.runCommand "republisher-redux"
+          {
+            inherit (runtimePackage) meta;
+            passthru = {
+              inherit tests testVenv runtimePackage;
+            };
+          }
+          ''
+            test -f "${tests}/passed"
+            ln -s "${runtimePackage}" "$out"
+          '';
+    in
+    {
+      formatter = forAllSystems (pkgs: (mkTreefmtConfig pkgs).build.wrapper);
+
+      packages = forAllSystems (
+        pkgs:
+        let
+          pkg = mkPackage pkgs;
+        in
+        {
+          "republisher-redux" = pkg;
+          default = pkg;
+        }
+      );
+
+      apps = forAllSystems (
+        pkgs:
+        let
+          package = self.packages.${pkgs.stdenv.hostPlatform.system}.default;
+        in
+        {
+          repub = {
+            type = "app";
+            program = "${package}/bin/repub";
+            meta.description = "republisher-redux runtime";
+          };
+          default = {
+            type = "app";
+            program = "${package}/bin/repub";
+            meta.description = "republisher-redux runtime";
+          };
+        }
+      );
+
+      checks = forAllSystems (
+        pkgs:
+        let
+          system = pkgs.stdenv.hostPlatform.system;
+          exportedPackage = self.packages.${system}.default;
+          testVenv = exportedPackage.testVenv;
+          treefmtConfig = mkTreefmtConfig pkgs;
+          src = ./.;
+          blackCheck = pkgs.stdenv.mkDerivation {
+            name = "republisher-redux-black";
+            inherit src;
+            dontConfigure = true;
+            dontBuild = true;
+            nativeBuildInputs = [ testVenv ];
+            checkPhase = ''
+              runHook preCheck
+              black --check repub/ tests/
+              runHook postCheck
+            '';
+            doCheck = true;
+            installPhase = ''
+              mkdir -p "$out"
+              touch "$out/passed"
+            '';
+          };
+          flake8Check = pkgs.stdenv.mkDerivation {
+            name = "republisher-redux-flake8";
+            inherit src;
+            dontConfigure = true;
+            dontBuild = true;
+            nativeBuildInputs = [ testVenv ];
+            checkPhase = ''
+              runHook preCheck
+              flake8 repub/ tests/
+              runHook postCheck
+            '';
+            doCheck = true;
+            installPhase = ''
+              mkdir -p "$out"
+              touch "$out/passed"
+            '';
+          };
+          isortCheck = pkgs.stdenv.mkDerivation {
+            name = "republisher-redux-isort";
+            inherit src;
+            dontConfigure = true;
+            dontBuild = true;
+            nativeBuildInputs = [ testVenv ];
+            checkPhase = ''
+              runHook preCheck
+              isort --check-only repub/ tests/
+              runHook postCheck
+            '';
+            doCheck = true;
+            installPhase = ''
+              mkdir -p "$out"
+              touch "$out/passed"
+            '';
+          };
+        in
+        {
+          devshell-default = self.devShells.${system}.default;
+          formatter = treefmtConfig.build.wrapper;
+          package-default = exportedPackage;
+          tests = exportedPackage.tests;
+          treefmt = treefmtConfig.build.check ./.;
+          black = blackCheck;
+          flake8 = flake8Check;
+          isort = isortCheck;
+        }
+      );
+
+      devShells = forAllSystems (pkgs: {
+        default = pkgs.mkShell {
+          packages = [
+            pkgs.python313
+            pkgs.uv
+            pkgs.ffmpeg-full
+          ];
+          env.LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
+            pkgs.stdenv.cc.cc
+          ];
+          env.UV_PROJECT_ENVIRONMENT = ".venv";
+          env.UV_PYTHON_DOWNLOADS = "never";
+        };
+      });
+    };
+}

pyproject.toml

@@ -1,15 +1,70 @@
-[tool.poetry]
+[project]
-name = "republisher"
+name = "republisher-redux"
 version = "0.1.0"
-description = ""
+description = "Mirror RSS and Atom feeds completely offline"
-authors = ["Abel Luck <abel@guardianproject.info>"]
 readme = "README.md"
-packages = [{include = "republisher", from = "src"}]
+authors = [{ name = "Abel Luck", email = "abel@guardianproject.info" }]
+requires-python = ">=3.13"
+dependencies = [
+  "scrapy>=2.11.1,<3.0.0",
+  "prometheus-client>=0.20.0,<0.21.0",
+  "python-dateutil>=2.9.0.post0,<3.0.0",
+  "colorlog>=6.8.2,<7.0.0",
+  "feedparser>=6.0.11,<7.0.0",
+  "lxml>=5.2.1,<6.0.0",
+  "pillow>=10.3.0,<11.0.0",
+  "ffmpeg-python>=0.2.0,<0.3.0",
+]
 
-[tool.poetry.dependencies]
+[project.scripts]
-python = "^3.11"
+repub = "repub.entrypoint:entrypoint"
 
+[dependency-groups]
+dev = [
+  "pytest>=8.1.1,<9.0.0",
+  "black>=24.4.0,<25.0.0",
+  "flake8>=7.0.0,<8.0.0",
+  "mypy>=1.9.0,<2.0.0",
+  "bandit>=1.7.8,<2.0.0",
+  "types-PyYAML>=6.0.12.20240311,<7.0.0",
+  "isort>=5.13.2,<6.0.0",
+  "flake8-black>=0.3.6,<0.4.0",
+]
 
 [build-system]
-requires = ["poetry-core"]
+requires = ["setuptools>=68", "wheel"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["repub*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.isort]
+py_version = 311
+profile = "black"
+src_paths = ["repub", "tests"]
+
+[tool.black]
+line-length = 88
+target-version = ['py313']
+
+[tool.mypy]
+files = "repub,tests"
+ignore_missing_imports = true
+follow_imports = "normal"
+disallow_untyped_calls = true
+disallow_any_generics = true
+disallow_subclassing_any = true
+warn_return_any = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_unused_configs = true
+warn_unreachable = true
+show_error_codes = true
+no_implicit_optional = true

repub/colorlog.py

@@ -0,0 +1,34 @@
+import copy
+
+import scrapy.utils.log
+
+from colorlog import ColoredFormatter
+
+color_formatter = ColoredFormatter(
+    (
+        "%(log_color)s%(levelname)-5s%(reset)s "
+        "%(yellow)s[%(asctime)s]%(reset)s"
+        "%(white)s %(name)s %(funcName)s %(bold_purple)s:%(lineno)d%(reset)s "
+        "%(log_color)s%(message)s%(reset)s"
+    ),
+    datefmt="%y-%m-%d %H:%M:%S",
+    log_colors={
+        "DEBUG": "blue",
+        "INFO": "bold_cyan",
+        "WARNING": "red",
+        "ERROR": "bg_bold_red",
+        "CRITICAL": "red,bg_white",
+    },
+)
+
+_get_handler = copy.copy(scrapy.utils.log._get_handler)
+
+
+def _get_handler_custom(*args, **kwargs):
+    handler = _get_handler(*args, **kwargs)
+    handler.setFormatter(color_formatter)
+    return handler
+
+
+def load_colorlog():
+    scrapy.utils.log._get_handler = _get_handler_custom

repub/entrypoint.py

@@ -0,0 +1,96 @@
+import logging
+import multiprocessing as mp
+import multiprocessing.connection as mpc
+
+feeds = {
+    "gp-pod": {"url": "https://guardianproject.info/podcast/podcast.xml"},
+    "nasa": {"url": "https://www.nasa.gov/rss/dyn/breaking_news.rss"},
+}
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+
+class FeedNameFilter:
+    def __init__(self, feed_options):
+        self.feed_options = feed_options
+
+    def accepts(self, item):
+        return item.feed_name == self.feed_options["feed_name"]
+
+
+def execute_spider(queue, name, url):
+    from scrapy.crawler import CrawlerProcess
+    from scrapy.settings import Settings
+    from scrapy.utils.project import get_project_settings
+
+    from repub.media import check_runtime
+    from repub.spiders.rss_spider import RssFeedSpider
+
+    try:
+        settings: Settings = {
+            **get_project_settings(),
+            "REPUBLISHER_OUT_DIR": "out",
+            "FEEDS": {
+                f"out/{name}.rss": {
+                    "format": "rss",
+                    "postprocessing": [],
+                    # "item_filter": FeedNameFilter,
+                    "feed_name": name,
+                }
+            },
+            "ITEM_PIPELINES": {
+                "repub.pipelines.ImagePipeline": 1,
+                "repub.pipelines.AudioPipeline": 2,
+                "repub.pipelines.VideoPipeline": 3,
+                "repub.pipelines.FilePipeline": 4,
+            },
+            "LOG_FILE": f"logs/{name}.log",
+            "REPUBLISHER_IMAGE_DIR": "images",
+            "REPUBLISHER_VIDEO_DIR": "video",
+            "REPUBLISHER_AUDIO_DIR": "audio",
+            "REPUBLISHER_FILE_DIR": "files",
+            "IMAGES_STORE": f"out/{name}/images",
+            "AUDIO_STORE": f"out/{name}/audio",
+            "VIDEO_STORE": f"out/{name}/videos",
+            "FILES_STORE": f"out/{name}/files",
+        }
+        if not check_runtime(
+            settings.get("REPUBLISHER_FFMPEG_ENCODERS"),
+            settings.get("REPUBLISHER_FFMPEG_CODECS"),
+        ):
+            logger.error("Runtime depenencies not met")
+            queue.put("missing dependencies")
+            return
+        process = CrawlerProcess(settings)
+        # colorlog.load_colorlog()
+        process.crawl(RssFeedSpider, feed_name=name, urls=[url])
+        process.start()
+        queue.put(None)
+    except Exception as e:
+        queue.put(e)
+
+
+def entrypoint():
+    pool = []
+    for name, data in feeds.items():
+        logger.info(f"Starting feed {name}")
+        queue = mp.Queue()
+        process = mp.Process(target=execute_spider, args=(queue, name, data["url"]))
+        pool.append((name, process, queue))
+    for n, proc, q in pool:
+        proc.start()
+    mpc.wait(p.sentinel for n, p, q in pool)
+    for name, p, q in pool:
+        result = q.get()
+        if result is not None:
+            print()
+            logger.error(f"Feed {name} encountered error")
+            logger.critical(result, exc_info=True)
+        else:
+            logger.info(f"Feed {name} completed successfully")

repub/exporters.py

@@ -0,0 +1,47 @@
+from io import BytesIO
+from typing import Any
+
+from scrapy.exporters import BaseItemExporter
+
+from repub import rss
+
+from .items import ChannelElementItem
+
+
+class RssExporter(BaseItemExporter):
+    def __init__(self, file: BytesIO, **kwargs: Any):
+        super().__init__(**kwargs)
+        if not self.encoding:
+            self.encoding = "utf-8"
+        self.file: BytesIO = file
+        self.rss = rss.rss()
+        self.channel = None
+        self.item_buffer = []
+
+    def start_exporting(self) -> None:
+        pass
+
+    def export_item(self, item: Any):
+        if isinstance(item, ChannelElementItem):
+            self.channel = item.el
+            self.rss.append(item.el)
+            self.flush_buffer()
+            return
+
+        if self.channel is None:
+            self.item_buffer.append(item)
+        else:
+            self.export_rss_item(item)
+
+    def flush_buffer(self):
+        for item in self.item_buffer:
+            self.export_rss_item(item)
+        self.item_buffer = []
+
+    def export_rss_item(self, item: Any):
+        assert self.channel is not None
+        self.channel.append(item.el)
+
+    def finish_exporting(self) -> None:
+        xml_bytes = rss.serialize(self.rss)
+        self.file.write(xml_bytes)

repub/items.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+from typing import Any, List
+
+
+@dataclass
+class ElementItem:
+    feed_name: str
+    el: Any
+    image_urls: List[str]
+    images: List[Any]
+    file_urls: List[str]
+    files: List[Any]
+    audio_urls: List[str]
+    audios: List[Any]
+    video_urls: List[str]
+    videos: List[Any]
+
+
+@dataclass
+class ChannelElementItem:
+    feed_name: str
+    el: Any
+    image_urls: List[str]
+    images: List[Any]

repub/media.py

@@ -0,0 +1,418 @@
+import copy
+import logging
+import math
+import os
+import subprocess
+import sys
+from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union
+
+import ffmpeg
+
+logger = logging.getLogger(__name__)
+
+MediaMeta = Dict[str, Union[str, int, float]]
+
+MediaSettings = TypedDict(
+    "MediaSettings", {"name": str, "extension": str, "mimetype": str}
+)
+
+
+class AudioSettings(MediaSettings):
+    format: str
+    max_bitrate: int
+    ffmpeg_audio_params: Dict[str, str]
+
+
+class VideoSettings(MediaSettings):
+    container: str
+    vcodec: str
+    max_height: int
+    acodec: str
+    audio_max_bitrate: int
+    ffmpeg_audio_params: Dict[str, str]
+    ffmpeg_video_params: Dict[str, str]
+
+
+class AudioMeta(TypedDict):
+    format_name: str
+    format_long_name: str
+    duration: str
+    bit_rate: float
+    size: str
+
+
+class VideoMeta(TypedDict):
+    duration: str
+    size: str
+    format_name: str
+    format_long_name: str
+    width: int
+    height: int
+    codec_name: str
+    display_aspect_ratio: str
+    duration_ts: float
+    bit_rate: float
+
+
+def probe_media(file_path) -> Dict[str, Any]:
+    """Probes `file_path` using ffmpeg's ffprobe and returns the data."""
+    try:
+        return ffmpeg.probe(file_path)
+    except ffmpeg.Error as e:
+        print(e.stderr, file=sys.stderr)
+        logger.error(f"Failed to probe io {file_path}")
+        logger.error(e)
+        raise RuntimeError(f"Failed to probe io {file_path}") from e
+
+
+def bitrate(info) -> float:
+    try:
+        return int(info["format"]["bit_rate"])
+    except KeyError | ValueError:
+        logger.error("extracting bitrate from ffprobe failed")
+        return math.inf
+
+
+def format_name(info) -> Optional[str]:
+    try:
+        return info["format"]["format_name"]
+    except KeyError | ValueError:
+        logger.error("extracting format from ffprobe failed")
+        return None
+
+
+def primary_video_stream(probe):
+    video_streams = [
+        stream for stream in probe["streams"] if stream["codec_type"] == "video"
+    ]
+    video_streams = sorted(video_streams, key=lambda x: x["duration_ts"], reverse=True)
+    if not video_streams:
+        return None
+    if len(video_streams) > 1:
+        logger.warn(
+            "Encountered video file with more than 1 video stream!, choosing the one with the longest duration"
+        )
+    return video_streams[0]
+
+
+def primary_audio_stream(probe):
+    audio_streams = [
+        stream for stream in probe["streams"] if stream["codec_type"] == "audio"
+    ]
+    audio_streams = sorted(audio_streams, key=lambda x: x["duration_ts"], reverse=True)
+    if not audio_streams:
+        return None
+    if len(audio_streams) > 1:
+        logger.warn(
+            "Encountered video file with more than 1 audio stream!, choosing the one with the longest duration"
+        )
+    return audio_streams[0]
+
+
+def get_resolution(probe) -> Tuple[Optional[float], Optional[float]]:
+    try:
+        video_stream = primary_video_stream(probe)
+        if not video_stream:
+            return None, None
+        width = int(video_stream["width"])
+        height = int(video_stream["height"])
+        return width, height
+    except KeyError | ValueError:
+        logger.error("extracting resolution from ffprobe failed")
+        return None, None
+
+
+def get_vcodec_name(probe) -> Optional[str]:
+    try:
+        video_stream = primary_video_stream(probe)
+        if not video_stream:
+            return None
+        return video_stream["codec_name"]
+    except KeyError | ValueError:
+        logger.error("extracting video codec_name from ffprobe failed")
+        return None
+
+
+def get_acodec_info(probe) -> Tuple[Optional[str], Optional[int]]:
+    try:
+        audio_stream = primary_audio_stream(probe)
+        if not audio_stream:
+            return None, None
+        return audio_stream["codec_name"], int(audio_stream["bit_rate"])
+    except KeyError | ValueError:
+        logger.error("extracting audio codec_name from ffprobe failed")
+        return None, None
+
+
+def audio_meta(probe: Dict[str, Any]) -> Optional[AudioMeta]:
+    return AudioMeta(
+        duration=probe["format"].get("duration", ""),
+        size=probe["format"].get("size", ""),
+        format_name=probe["format"].get("format_name", ""),
+        format_long_name=probe["format"].get("format_long_name", ""),
+        bit_rate=float(probe["format"].get("bit_rate", 0.0)),
+    )
+
+
+def video_meta(probe: Dict[str, Any]) -> Optional[VideoMeta]:
+    stream = primary_video_stream(probe)
+    if not stream:
+        return None
+    return VideoMeta(
+        duration=probe["format"].get("duration", ""),
+        size=probe["format"].get("size", ""),
+        format_name=probe["format"].get("format_name", ""),
+        format_long_name=probe["format"].get("format_long_name", ""),
+        width=int(stream.get("width", 0)),
+        height=int(stream.get("height", 0)),
+        codec_name=stream.get("codec_name", ""),
+        display_aspect_ratio=stream.get("display_aspect_ratio", ""),
+        duration_ts=float(stream.get("duration_ts", 0.0)),
+        bit_rate=float(stream.get("bit_rate", 0.0)),
+    )
+
+
+def audio_transcode_params(
+    probe_result, settings: AudioSettings
+) -> Optional[Dict[str, str]]:
+    """
+    Given a probe result and some system settings,
+    this function returns a dict containing opaque data that could be passsed to compress_audio.
+    If this function returns None, then the audio does not need to be compressed
+    """
+    br = settings["max_bitrate"]
+    fmt = settings["format"]
+    if bitrate(probe_result) <= br:
+        is_br = True
+    else:
+        is_br = False
+    if format_name(probe_result) == fmt:
+        is_fmt = True
+    else:
+        is_fmt = False
+
+    if is_br and is_fmt:
+        return None
+
+    params = {"extension": settings["extension"]}
+    params.update(settings["ffmpeg_audio_params"])
+    return params
+
+
+def transcode_audio(input_file: str, output_dir: str, params: Dict[str, str]) -> str:
+    """
+    Uses ffmpeg, applying `settings` to `input_file`, storing output in `output_dir`, and returning the path to the compressed file
+    """
+    params = copy.deepcopy(params)
+    ext = params.pop("extension")
+    output_file = f"{output_dir}/converted.{ext}"
+    try:
+        logger.info(
+            f"Transcoding audio {input_file} to {output_file} with params={params}"
+        )
+        out, _ = (
+            ffmpeg.input(input_file)
+            .output(
+                output_file,
+                **params,
+                loglevel="quiet",
+            )
+            .run()
+        )
+        before = os.path.getsize(input_file) / 1024
+        after = os.path.getsize(output_file) / 1024
+        percent_difference = 0
+        if before != 0:
+            percent_difference = ((before - after) / before) * 100
+        logger.info(
+            f"Compressed from {before:.2f} KiB to {after:.2f} KiB, reduction: {percent_difference:.2f}%"
+        )
+        return output_file
+    except ffmpeg.Error as e:
+        print(e.stderr, file=sys.stderr)
+        print(e.stdout)
+        logger.error(e)
+        raise RuntimeError(f"Failed to compress audio {input_file}") from e
+
+
+def video_transcode_params(
+    probe_result, settings: VideoSettings
+) -> Optional[Dict[str, Any]]:
+    """
+    Given a probe result and some system settings,
+    this function returns a dict containing opaque data that could be passsed to compress_video.
+    If this function returns None, then the video does not need to be compressed
+    """
+    max_height = settings["max_height"]
+    target_container = settings["container"]
+    target_vcodec = settings["vcodec"]
+    target_acodec = settings["acodec"]
+    audio_max_bitrate = settings["audio_max_bitrate"]
+
+    width, height = get_resolution(probe_result)
+    vcodec = get_vcodec_name(probe_result)
+    acodec, audio_bit_rate = get_acodec_info(probe_result)
+
+    if not width or not height or not acodec or not audio_bit_rate:
+        logger.error("Failed to extract data from ffprobe")
+        # TODO: turn this into an exception and catch it for reporting
+        return None
+
+    current_container_many = format_name(probe_result)
+    is_container = False
+    if current_container_many is not None:
+        if target_container in current_container_many.split(","):
+            is_container = True
+
+    is_vcodec = vcodec == target_vcodec
+    is_acodec = acodec == target_acodec
+    is_audio_bitrate = audio_bit_rate <= audio_max_bitrate
+    is_good_height = height <= max_height
+
+    if is_good_height and is_container and is_vcodec and is_acodec and is_audio_bitrate:
+        return None
+
+    passes = settings.get("passes", [])
+    params = {"extension": settings["extension"]}
+    if len(passes) == 0 or is_vcodec:
+        if not is_good_height:
+            params["vf"] = f"scale={width}:{height}"
+        if not is_vcodec:
+            params.update(settings["ffmpeg_video_params"])
+        if not is_acodec or not is_audio_bitrate:
+            params.update(settings["ffmpeg_audio_params"])
+        return params
+    params["passes"] = []
+    for p in passes:
+        p = copy.deepcopy(p)
+        if not is_good_height:
+            p["vf"] = f"scale={width}:{height}"
+        params["passes"].append(p)
+    return params
+
+
+def transcode_video(input_file: str, output_dir: str, params: Dict[str, Any]) -> str:
+    """
+    Uses ffmpeg, applying `settings` to `input_file`, storing output in `output_dir`, and returning the path to the compressed file
+    """
+    params = copy.deepcopy(params)
+    ext = params.pop("extension")
+    output_file = f"{output_dir}/converted.{ext}"
+    try:
+        logger.info(
+            f"Transcoding video {input_file} to {output_file} with params={params}"
+        )
+        if "passes" not in params:
+            out, _ = (
+                ffmpeg.input(input_file)
+                .output(
+                    output_file,
+                    **params,
+                    # loglevel="quiet",
+                )
+                .run()
+            )
+        else:
+            passes = params["passes"]
+            ffinput = ffmpeg.input(input_file)
+            video = ffinput.video
+            audio = ffinput.audio
+            ffoutput = ffinput.output(video, "pipe:", **passes[0])
+            ffoutput = ffoutput.global_args(
+                # "-loglevel", "quiet",
+                "-stats"
+            )
+            logger.info("Running pass #1")
+            std_out, std_err = ffoutput.run(capture_stdout=True)
+            print(std_out)
+            print(std_err)
+            logger.info("Running pass #2")
+            ffoutput = ffinput.output(video, audio, output_file, **passes[1])
+            ffoutput = ffoutput.global_args(
+                # "-loglevel", "quiet",
+                "-stats"
+            )
+            ffoutput.run(overwrite_output=True)
+
+        before = os.path.getsize(input_file) / 1024
+        after = os.path.getsize(output_file) / 1024
+        percent_difference = 0
+        if before != 0:
+            percent_difference = ((before - after) / before) * 100
+        logger.info(
+            f"Compressed from {before:.2f} KiB to {after:.2f} KiB, reduction: {percent_difference:.2f}%"
+        )
+        return output_file
+    except ffmpeg.Error as e:
+        print(e.stderr, file=sys.stderr)
+        logger.error("Failed to transcode")
+        logger.error(e)
+        raise RuntimeError(f"Failed to transcode video: {e.stderr.decode()}") from e
+    except Exception as e:
+        logger.critical(e, exc_info=True)
+        raise e
+
+
+def check_codecs(codecs: List[str]) -> List[str]:
+    result = subprocess.run(
+        ["ffmpeg", "-v", "quiet", "-codecs"], capture_output=True, text=True
+    )
+    output = result.stdout
+
+    available_codecs = set(
+        line.split()[1]
+        for line in output.splitlines()
+        if len(line.split()) > 2 and "E" in line.split()[0]
+    )
+    missing_codecs = [codec for codec in codecs if codec not in available_codecs]
+
+    return missing_codecs
+
+
+def check_encoders(encoders: List[str]) -> List:
+    result = subprocess.run(
+        ["ffmpeg", "-v", "quiet", "-encoders"], capture_output=True, text=True
+    )
+    output = result.stdout
+    lines = output.split("\n")
+    encoder_lines = [
+        line.strip()
+        for line in lines
+        if line.startswith(" V") or line.startswith(" A") or line.startswith(" S")
+    ]
+    available_encoders = set(
+        line.split()[1] for line in encoder_lines if len(line.split()) > 1
+    )
+    missing_encoders = [
+        encoder for encoder in encoders if encoder not in available_encoders
+    ]
+    return missing_encoders
+
+
+def is_ffmpeg_available() -> bool:
+    try:
+        subprocess.run(
+            ["ffmpeg", "-version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+        )
+        return True
+    except OSError:
+        return False
+
+
+def check_runtime(encoders: List[str], codecs: List[str]) -> bool:
+    if not is_ffmpeg_available():
+        logger.error("FFMPEG is not available on the PATH")
+        return False
+    missing_encoders = check_encoders(encoders)
+    missing_codecs = check_codecs(codecs)
+    if missing_encoders:
+        m = ", ".join(missing_encoders)
+        logger.error(f"Missing ffmpeg encoders: {m}")
+
+    if missing_codecs:
+        m = ", ".join(missing_codecs)
+        logger.error(f"Missing ffmpeg codecs: {m}")
+    if missing_codecs or missing_encoders:
+        return False
+
+    return True

repub/middlewares.py

@@ -0,0 +1,100 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class RepubSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+
+class RepubDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)

repub/pipelines.py

@@ -0,0 +1,184 @@
+import logging
+import tempfile
+from io import BytesIO
+from os import PathLike
+from typing import Dict, List, Optional, Union
+
+from scrapy.pipelines.files import FilesPipeline as BaseFilesPipeline
+from scrapy.pipelines.images import ImagesPipeline as BaseImagesPipeline
+from scrapy.settings import Settings
+from scrapy.utils.misc import md5sum
+
+import repub.utils
+from repub import media
+
+logger = logging.getLogger(__name__)
+
+
+class ImagePipeline(BaseImagesPipeline):
+    def file_path(self, request, response=None, info=None, *, item=None):
+        return repub.utils.local_image_path(request.url)
+
+    def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
+        raise NotImplementedError()
+
+
+class FilePipeline(BaseFilesPipeline):
+    def __init__(self, store_uri, **kwargs):
+        settings = kwargs["settings"]
+        if isinstance(settings, dict) or settings is None:
+            settings = Settings(settings)
+        self.settings = settings
+        super().__init__(store_uri, **kwargs)
+
+    def file_path(self, request, response=None, info=None, *, item=None):
+        return repub.utils.local_file_path(request.url)
+
+
+def read_asset(file_path) -> BytesIO:
+    buf_converted = BytesIO()
+    with open(file_path, "rb") as f:
+        buf_converted.write(f.read())
+    buf_converted.seek(0)
+    return buf_converted
+
+
+def media_final_path(base_path, name, ext):
+    return f"{base_path}-{name}.{ext}"
+
+
+class TranscodePipeline(BaseFilesPipeline):
+    def __init__(
+        self,
+        media_type: repub.utils.FileType,
+        store_uri: Union[str, PathLike],
+        **kwargs,
+    ):
+        settings = kwargs["settings"]
+        self.media_type = media_type
+        if isinstance(settings, dict) or settings is None:
+            settings = Settings(settings)
+        self.settings = settings
+        super().__init__(store_uri, **kwargs)
+
+    def file_downloaded(self, response, request, info, *, item=None):
+        return self.media_downloaded(response, request, info, item=item)
+
+    def media_downloaded(self, response, request, info, *, item=None):
+        checksum = None
+        for path, buf, meta, mime in self.get_media(response, request, info, item=item):
+            if checksum is None:
+                buf.seek(0)
+                checksum = md5sum(buf)
+            self.store.persist_file(
+                path,
+                buf,
+                info,
+                meta=meta,
+                headers={"Content-Type": mime},
+            )
+        return checksum
+
+    def transcode(
+        self, input_file: str, settings: media.MediaSettings, tmp_dir: str
+    ) -> Optional[str]:
+        probe_result = media.probe_media(input_file)
+        params = self.get_transcode_params(probe_result, settings)
+        if params is not None:
+            converted_file = self.transcode_media(input_file, tmp_dir, params)
+            return converted_file
+        else:
+            logger.info(
+                f"Skipping audio compression for {input_file}, it meets requirements"
+            )
+            return None
+
+    def get_media_settings(self) -> List[media.MediaSettings]:
+        raise NotImplementedError()
+
+    def get_transcode_params(self, probe_result, settings) -> Optional[Dict[str, str]]:
+        raise NotImplementedError()
+
+    def transcode_media(self, input_file: str, tmp_dir: str, params) -> str:
+        raise NotImplementedError()
+
+    def get_media_meta(self, probe_result) -> media.MediaMeta:
+        raise NotImplementedError()
+
+    def get_media(self, response, request, info, *, item=None):
+        buf = BytesIO(response.body)
+        base_path = self.file_path(request, response=response, info=info, item=item)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            settings = self.get_media_settings()
+            tmp_file = f"{tmp_dir}/original"
+            with open(tmp_file, "wb") as f:
+                f.write(buf.read())
+            for setting in settings:
+                ext = setting["extension"]
+                name = setting["name"]
+                final_path = media_final_path(base_path, name, ext)
+                stat = self.store.stat_file(final_path, info)
+                if stat:
+                    logger.info(f"Skipping, transcoded media exists at {final_path}")
+                    continue
+                converted_file = self.transcode(tmp_file, setting, tmp_dir)
+                if converted_file:
+                    out_buf = read_asset(converted_file)
+                    out_file = converted_file
+                else:
+                    out_buf = buf
+                    out_file = tmp_file
+                probe_result = media.probe_media(out_file)
+                meta = self.get_media_meta(probe_result)
+                logger.info(f"{self.media_type} final {final_path} with {meta}")
+                yield final_path, out_buf, meta, setting["mimetype"]
+
+
+class AudioPipeline(TranscodePipeline):
+
+    DEFAULT_FILES_URLS_FIELD = "audio_urls"
+    DEFAULT_FILES_RESULT_FIELD = "audios"
+
+    def __init__(self, store_uri: Union[str, PathLike], **kwargs):
+        store_uri = kwargs["settings"]["AUDIO_STORE"]
+        super().__init__(repub.utils.FileType.AUDIO, store_uri, **kwargs)
+
+    def file_path(self, request, response=None, info=None, *, item=None):
+        return repub.utils.local_audio_path(request.url)
+
+    def get_media_settings(self) -> List[media.AudioSettings]:
+        return self.settings["REPUBLISHER_AUDIO"]
+
+    def get_transcode_params(self, probe_result, settings) -> Optional[Dict[str, str]]:
+        return media.audio_transcode_params(probe_result, settings)
+
+    def transcode_media(self, input_file: str, tmp_dir: str, params) -> str:
+        return media.transcode_audio(input_file, tmp_dir, params)
+
+    def get_media_meta(self, probe_result) -> Optional[media.AudioMeta]:
+        return media.audio_meta(probe_result)
+
+
+class VideoPipeline(TranscodePipeline):
+
+    DEFAULT_FILES_URLS_FIELD = "video_urls"
+    DEFAULT_FILES_RESULT_FIELD = "videos"
+
+    def __init__(self, store_uri: Union[str, PathLike], **kwargs):
+        store_uri = kwargs["settings"]["VIDEO_STORE"]
+        super().__init__(repub.utils.FileType.VIDEO, store_uri, **kwargs)
+
+    def file_path(self, request, response=None, info=None, *, item=None):
+        return repub.utils.local_video_path(request.url)
+
+    def get_media_settings(self) -> List[media.VideoSettings]:
+        return self.settings["REPUBLISHER_VIDEO"]
+
+    def get_transcode_params(self, probe_result, settings) -> Optional[Dict[str, str]]:
+        return media.video_transcode_params(probe_result, settings)
+
+    def transcode_media(self, input_file: str, tmp_dir: str, params) -> str:
+        return media.transcode_video(input_file, tmp_dir, params)
+
+    def get_media_meta(self, probe_result) -> Optional[media.VideoMeta]:
+        return media.video_meta(probe_result)

repub/rss.py

@@ -0,0 +1,136 @@
+from datetime import datetime
+from time import mktime
+
+import lxml.etree as ET
+import lxml.html
+from lxml import etree
+from lxml.builder import ElementMaker
+from lxml.etree import Element
+
+from .srcset import SRCSet
+
+# monkeypatch lxml.html.defs to support srcset as a link attr
+link_attrs_orig = lxml.html.defs.link_attrs
+lxml.html.defs.link_attrs = frozenset(link_attrs_orig.union({"srcset"}))
+
+
+class SafeElementMaker:
+    """
+    Wraps ElementMaker to silently drop None values
+    """
+
+    def __init__(self, **kwargs):
+        self._maker = ElementMaker(**kwargs)
+
+    def __getattr__(self, tag):
+        def safe_element(*children, **attrib):
+            valid_children = [
+                child
+                for child in children
+                if child is not None and (not isinstance(child, str) or child.strip())
+            ]
+            if valid_children or attrib:
+                if isinstance(tag, str):
+                    return self._maker.__getattr__(tag)(*valid_children, **attrib)
+                elif issubclass(tag, Element):
+                    return tag(*valid_children, **attrib)
+
+        return safe_element
+
+
+nsmap = {
+    "content": "http://purl.org/rss/1.0/modules/content/",
+    "media": "http://search.yahoo.com/mrss/",
+    "itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd",
+    "dc": "http://purl.org/dc/elements/1.1/",
+    "atom": "http://www.w3.org/2005/Atom",
+}
+
+CONTENT = SafeElementMaker(nsmap={None: nsmap["content"]}, namespace=nsmap["content"])
+MEDIA = SafeElementMaker(nsmap={None: nsmap["media"]}, namespace=nsmap["media"])
+ITUNES = SafeElementMaker(nsmap={None: nsmap["itunes"]}, namespace=nsmap["itunes"])
+DC = SafeElementMaker(nsmap={None: nsmap["dc"]}, namespace=nsmap["dc"])
+ATOM = SafeElementMaker(nsmap={None: nsmap["atom"]}, namespace=nsmap["atom"])
+E: ElementMaker = SafeElementMaker(nsmap=nsmap)
+CDATA = ET.CDATA
+
+
+def rss():
+    return E.rss({"version": "2.0"})
+
+
+def parse_pubdate(date_str):
+    try:
+        return datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %z")
+    except ValueError:
+        return datetime.min
+
+
+def sort_rss(root):
+    channel = root.find("channel")
+    items = list(channel.findall("item"))
+    for item in items:
+        channel.remove(item)
+
+    items.sort(
+        key=lambda x: parse_pubdate(
+            x.find("pubDate").text if x.find("pubDate") is not None else ""
+        ),
+        reverse=True,
+    )
+
+    for item in items:
+        channel.append(item)
+    return root
+
+
+def serialize(root):
+    # root = sort_rss(root)
+    return etree.tostring(
+        root, encoding="utf-8", xml_declaration=True, pretty_print=True
+    )
+
+
+def date_format(d):
+    if d:
+        return d.strftime("%a, %d %b %Y %H:%M:%S %z")
+
+
+def to_datetime(struct_time):
+    if struct_time:
+        return datetime.fromtimestamp(mktime(struct_time))
+
+
+def normalize_date(struct_time):
+    return date_format(to_datetime(struct_time))
+
+
+def munge_cdata_html(raw_html, replace_link_fn) -> str:
+    html = lxml.html.fromstring(raw_html)
+    for el, attr, link, pos in html.iterlinks():
+        if attr == "srcset":
+            # these are a messy special case
+            o = SRCSet(el.attrib["srcset"])
+            o.parse()
+            for c in o.candidates:
+                link = c["url"]
+                new_link = replace_link_fn(el, attr, link.strip())
+                c["url"] = new_link
+
+            el.set(attr, o.stringify())
+            continue
+
+        new_link = replace_link_fn(el, attr, link.strip())
+        if new_link == link:
+            continue
+        if attr is None:
+            new = el.text[:pos] + new_link + el.text[pos + len(link) :]
+            el.text = new
+        else:
+            cur = el.get(attr)
+            if not pos and len(cur) == len(link):
+                new = new_link  # most common case
+            else:
+                new = cur[:pos] + new_link + cur[pos + len(link) :]
+            el.set(attr, new)
+    return lxml.html.tostring(html, encoding="utf-8", pretty_print=True).decode("utf-8")

repub/settings.py

@@ -0,0 +1,184 @@
+# Scrapy settings for repub project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = "repub"
+
+SPIDER_MODULES = ["repub.spiders"]
+NEWSPIDER_MODULE = "repub.spiders"
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+USER_AGENT = "GuardianProject-Republisher-Redux (+https://guardianproject.info)"
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+# CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+# DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+# CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+# COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+# TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+# DEFAULT_REQUEST_HEADERS = {
+#    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+#    "Accept-Language": "en",
+# }
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+# SPIDER_MIDDLEWARES = {
+#    "repub.middlewares.RepubSpiderMiddleware": 543,
+# }
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+# DOWNLOADER_MIDDLEWARES = {
+#    "repub.middlewares.RepubDownloaderMiddleware": 543,
+# }
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+# EXTENSIONS = {
+#    "scrapy.extensions.telnet.TelnetConsole": None,
+# }
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+# ITEM_PIPELINES = {}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+# AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+# AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+# AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+# AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+HTTPCACHE_ENABLED = True
+HTTPCACHE_EXPIRATION_SECS = 0
+HTTPCACHE_DIR = "httpcache"
+HTTPCACHE_IGNORE_HTTP_CODES = []
+HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+FEED_EXPORT_ENCODING = "utf-8"
+FEED_EXPORTERS = {
+    "rss": "repub.exporters.RssExporter",
+}
+
+TELNETCONSOLE_ENABLED = False
+
+LOG_LEVEL = "INFO"
+# LOG_LEVEL = "ERROR"
+
+MEDIA_ALLOW_REDIRECTS = True
+
+REPUBLISHER_AUDIO = [
+    {
+        "name": "vbr7",
+        "format": "mp3",
+        "max_bitrate": 96000,
+        "mimetype": "audio/mp3",
+        "extension": "mp3",
+        "ffmpeg_audio_params": {
+            "acodec": "libmp3lame",
+            # https://trac.ffmpeg.org/wiki/Encode/MP3#VBREncoding
+            "qscale:a": "7",
+        },
+    },
+    {
+        "name": "vbr3",
+        "format": "aac",
+        "max_bitrate": 96000,
+        "mimetype": "audio/aac",
+        "extension": "aac",
+        "ffmpeg_audio_params": {
+            "acodec": "libfdk_aac",
+            # https://trac.ffmpeg.org/wiki/Encode/MP3#VBREncoding
+            "vbr": "3",
+        },
+    },
+]
+REPUBLISHER_VIDEO = [
+    {
+        "name": "720",
+        "container": "mp4",
+        "vcodec": "h264",
+        "acodec": "mp3",
+        "audio_max_bitrate": 96000,
+        "ffmpeg_audio_params": {
+            "acodec": "libmp3lame",
+            # https://trac.ffmpeg.org/wiki/Encode/MP3#VBREncoding
+            "qscale:a": "7",
+        },
+        "ffmpeg_video_params": {"vcodec": "h264", "strict": "-2"},
+        "max_height": 720,
+        "mimetype": "video/mp4",
+        "extension": "mp4",
+    },
+    # {
+    #    "passes": [
+    #        {
+    #            "c:v": "libvpx-vp9",
+    #            "b:v": "0",
+    #            "crf": "30",
+    #            "pass": "1",
+    #            "deadline": "good",
+    #            "row-mt": "1",
+    #            "f": "null",
+    #        },
+    #        {
+    #            "c:v": "libvpx-vp9",
+    #            "b:v": "0",
+    #            "crf": "30",
+    #            "pass": "2",
+    #            "deadline": "good",
+    #            "row-mt": "1",
+    #            "c:a": "libopus",
+    #            "b:a": "96k",
+    #            "ac": "2",
+    #        },
+    #    ],
+    #    "name": "720",
+    #    "container": "webm",
+    #    "vcodec": "libvpx-vp9",
+    #    "acodec": "opus",
+    #    "audio_max_bitrate": 96000,
+    #    "max_height": 720,
+    #    "mimetype": "video/webm",
+    #    "extension": "webm",
+    # },
+]
+
+REPUBLISHER_FFMPEG_ENCODERS = ["libmp3lame", "libfdk_aac", "libvpx-vp9", "libopus"]
+REPUBLISHER_FFMPEG_CODECS = ["aac", "mp3", "mpeg4", "vp9", "opus"]
+
+
+# CLOSESPIDER_ERRORCOUNT = 1

repub/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

repub/spiders/rss_spider.py

@@ -0,0 +1,283 @@
+import logging
+from typing import Dict, List, Tuple
+
+import feedparser
+from scrapy.crawler import Crawler
+from scrapy.spiders import Spider
+from scrapy.utils.spider import iterate_spider_output
+
+from repub.items import ChannelElementItem, ElementItem
+from repub.rss import CDATA, CONTENT, ITUNES, MEDIA, E, munge_cdata_html, normalize_date
+from repub.utils import FileType, determine_file_type, local_file_path
+
+
+class BaseRssFeedSpider(Spider):
+    """
+    This class intends to be the base class for spiders that scrape
+    from RSS feeds.
+    """
+
+    def __init__(self, feed_name, **kwargs):
+        super().__init__(**kwargs)
+        self.feed_name = feed_name
+
+    def _set_crawler(self, crawler: Crawler) -> None:
+        super()._set_crawler(crawler)
+        for s in [
+            "REPUBLISHER_IMAGE_DIR",
+            "REPUBLISHER_FILE_DIR",
+            "REPUBLISHER_AUDIO_DIR",
+            "REPUBLISHER_VIDEO_DIR",
+        ]:
+            if self.settings.get(s) is None:
+                raise RuntimeError(f"Missing setting: {s}")
+
+    def rewrite_file_url(self, file_type: FileType, url):
+        file_dir = self.settings["REPUBLISHER_FILE_DIR"]
+        if file_type == FileType.IMAGE:
+            file_dir = self.settings["REPUBLISHER_IMAGE_DIR"]
+        elif file_type == FileType.VIDEO:
+            file_dir = self.settings["REPUBLISHER_VIDEO_DIR"]
+        elif file_type == FileType.AUDIO:
+            file_dir = self.settings["REPUBLISHER_AUDIO_DIR"]
+        return f"/{file_dir}/{local_file_path(url)}"
+
+    def rewrite_image_url(self, url):
+        return self.rewrite_file_url(FileType.IMAGE, url)
+
+    def munge_cdata_html(self, html) -> Tuple[str, Dict[FileType, List[str]]]:
+        urls = {FileType.IMAGE: [], FileType.VIDEO: [], FileType.AUDIO: []}
+
+        def replace_link(el, attr, old_link):
+            if len(old_link) == 0 or el.tag in ["a", "iframe"]:
+                return old_link
+            file_type = None
+            if el.tag in ["img"]:
+                file_type = FileType.IMAGE
+            elif el.tag in ["source"] and el.getparent() is not None:
+                if el.getparent().tag == "video":
+                    file_type = FileType.VIDEO
+                elif el.getparent().tag == "audio":
+                    file_type = FileType.AUDIO
+                elif el.getparent().tag == "picture":
+                    file_type = FileType.IMAGE
+            if not file_type:
+                self.logger.warn(
+                    f"Could not identify file type of link, skipping. tag={el.tag} attr={attr} link={old_link}"
+                )
+                return old_link
+
+            urls[file_type].append(old_link)
+            new_link = self.rewrite_file_url(file_type, old_link)
+            if file_type != FileType.IMAGE:
+                print(f"{old_link} -> {new_link}")
+            return new_link
+
+        return munge_cdata_html(html, replace_link), urls
+
+    def parse_feed(self, feed_text):
+        parsed = feedparser.parse(feed_text, sanitize_html=False)
+        if parsed.bozo:
+            logging.error(
+                "Bozo feed data. %s: %r",
+                parsed.bozo_exception.__class__.__name__,
+                parsed.bozo_exception,
+            )
+            if hasattr(parsed.bozo_exception, "getLineNumber") and hasattr(
+                parsed.bozo_exception, "getMessage"
+            ):
+                line = parsed.bozo_exception.getLineNumber()
+                logging.error("Line %d: %s", line, parsed.bozo_exception.getMessage())
+                segment = feed_text.split("\n")[line - 1]
+                logging.info("Body segment with error: %r", segment)
+            return None
+        return parsed
+
+    def parse_channel_meta(self, response, feed):
+        f = feed.feed
+        channel = E.channel(
+            E.title(f.get("title")),
+            E.link(f.get("link")),
+            E.description(f.get("description")),
+            E.language(f.get("language")),
+            E.copyright(f.get("copyright")),
+            E.webMaster(f.get("publisher")),
+            E.generator(f.get("generator")),
+            E.pubDate(normalize_date(f.get("published_parsed"))),
+            E.lastBuildDate(normalize_date(f.get("updated_parsed"))),
+            ITUNES.explicit("yes" if f.get("itunes_explicit", False) else "no"),
+        )
+        for tag in f.get("tags", []):
+            channel.append(E.category(tag.term))
+
+        image_urls = []
+        if "image" in f:
+            if "href" in f.image:
+                image = E.image(
+                    E.title(f.get("title")),
+                    E.link(f.get("link")),
+                    E.url(self.rewrite_image_url(f.image.get("href"))),
+                    E.description(f.get("description")),
+                )
+                image_urls.append(f.image.get("href"))
+            else:
+                image = E.image(
+                    E.title(f.image.get("title")),
+                    E.link(f.image.get("link")),
+                    E.url(self.rewrite_image_url(f.image.get("url"))),
+                    E.description(f.image.get("description")),
+                    E.width(f.image.get("width")),
+                    E.height(f.image.get("height")),
+                )
+                image_urls.append(f.image.get("url"))
+            channel.append(image)
+        return ChannelElementItem(
+            feed_name=self.feed_name, el=channel, image_urls=image_urls, images=[]
+        )
+
+    def _parse(self, response, **kwargs):
+        response = self.adapt_response(response)
+        feed = self.parse_feed(response.body)
+        if feed and feed.feed:
+            return self.parse_entries(response, feed)
+
+    def parse_entry(self, response, feed, entry):
+        """This method must be overridden with your custom spider functionality"""
+        raise NotImplementedError
+
+    def parse_entries(self, response, feed):
+        channel = self.parse_channel_meta(response, feed)
+        yield channel
+        for entry in feed.entries:
+            ret = iterate_spider_output(self.parse_entry(response, feed, entry))
+            yield from self.process_results(response, feed, ret)
+
+    def process_results(self, response, feed, results):
+        """This overridable method is called for each result (item or request)
+        returned by the spider, and it's intended to perform any last time
+        processing required before returning the results to the framework core,
+        for example setting the item GUIDs. It receives a list of results and
+        the response which originated that results. It must return a list of
+        results (items or requests).
+        """
+        return results
+
+    def adapt_response(self, response):
+        """You can override this function in order to make any changes you want
+        to into the feed before parsing it. This function must return a
+        response.
+        """
+        return response
+
+
+class RssFeedSpider(BaseRssFeedSpider):
+    """A generic RSS Feed spider"""
+
+    name = "rss_spider"
+
+    def __init__(self, urls, **kwargs):
+        self.start_urls = urls
+        super().__init__(**kwargs)
+
+    def parse_entry(self, response, feed, entry):
+        image_urls = []
+        file_urls = []
+        audio_urls = []
+        video_urls = []
+
+        def add_url(file_type, url):
+            if file_type == FileType.IMAGE:
+                image_urls.append(url)
+            elif file_type == FileType.AUDIO:
+                audio_urls.append(url)
+            elif file_type == FileType.VIDEO:
+                video_urls.append(url)
+            elif file_type == FileType.FILE:
+                file_urls.append(url)
+
+        item = E.item(
+            E.title(entry.get("title")),
+            E.link(entry.get("link")),
+            E.description(entry.get("description")),
+            E.guid(
+                entry.get("id"),
+                {"isPermaLink": "true" if entry.guidislink else "false"},
+            ),
+            E.pubDate(normalize_date(entry.get("published_parsed"))),
+            E.author(entry.get("author")),
+            ITUNES.summary(entry.get("summary")),
+            ITUNES.duration(entry.get("itunes_duration")),
+            ITUNES.image(
+                None,
+                (
+                    {"href": self.rewrite_image_url(entry.get("image").href)}
+                    if "image" in entry
+                    else None
+                ),
+            ),
+        )
+        if entry.get("image"):
+            image_urls.append(entry.get("image").href)
+        for enc in entry.enclosures:
+            url = enc.get("href")
+            file_type = determine_file_type(url=url, mimetype=enc.get("type"))
+            item.append(
+                E.enclosure(
+                    E.url(self.rewrite_file_url(file_type, url)),
+                    E.length(enc.get("length")),
+                    E.type(enc.get("type")),
+                )
+            )
+            self.logger.debug(
+                f"feed {self.feed_name} encountered enclsoure {url} {file_type}"
+            )
+            add_url(file_type, url)
+
+        if "content" in entry:
+            for c in entry.content:
+                if c.type == "text/html":
+                    html, urls = self.munge_cdata_html(c.value)
+                    item.append(CONTENT.encoded(CDATA(html)))
+                    image_urls.extend(urls[FileType.IMAGE])
+                    video_urls.extend(urls[FileType.VIDEO])
+                    audio_urls.extend(urls[FileType.AUDIO])
+
+        if isinstance(entry.get("media_content"), list):
+            for media in (
+                media for media in entry["media_content"] if media.get("url")
+            ):
+                file_type = determine_file_type(
+                    url=media.get("url"),
+                    medium=media.get("medium"),
+                    mimetype=media.get("type"),
+                )
+                item.append(
+                    MEDIA.content(
+                        E.url(self.rewrite_file_url(file_type, media.get("url"))),
+                        E.type(media.get("type")),
+                        E.medium(media.get("medium")),
+                        E.isDefault(media.get("isDefault")),
+                        E.expression(media.get("expression")),
+                        E.bitrate(media.get("bitrate")),
+                        E.framerate(media.get("framerate")),
+                        E.samplingrate(media.get("samplingrate")),
+                        E.channels(media.get("channels")),
+                        E.duration(media.get("duration")),
+                        E.height(media.get("height")),
+                        E.width(media.get("width")),
+                        E.lang(media.get("lang")),
+                    )
+                )
+                add_url(file_type, media.get("url"))
+        return ElementItem(
+            feed_name=self.feed_name,
+            el=item,
+            images=[],
+            image_urls=image_urls,
+            files=[],
+            file_urls=file_urls,
+            audio_urls=audio_urls,
+            audios=[],
+            video_urls=video_urls,
+            videos=[],
+        )

repub/srcset.py

@@ -0,0 +1,214 @@
+from __future__ import unicode_literals
+
+import math
+
+# See https://infra.spec.whatwg.org/#ascii-whitespace
+WHITESPACES = ("\u0009", "\u000a", "\u000c", "\u000d", "\u0020")  # \t  # " "
+
+STATE_IN_DESCRIPTOR = 1
+STATE_AFTER_DESCRIPTOR = 2
+STATE_IN_PARENS = 3
+
+
+class SRCSet(object):
+    raw = None
+    candidates = None
+
+    def __init__(self, string):
+        self.raw = string
+
+    def parse(self):
+        """
+        Based on algorithm from https://html.spec.whatwg.org/multipage/images.html#parse-a-srcset-attribute
+        """
+        # Step 1, 2, 3
+        pos = 0
+        candidates = []
+        state = None
+
+        # Step 4
+        while True:
+            pos, _ = collect_characters_in(self.raw, pos, WHITESPACES + (",",))
+
+            # Step 5
+            if pos >= len(self.raw):
+                # The only one place where we leave the loop
+                self.candidates = candidates
+                return candidates
+
+            # Step 6
+            pos, url = collect_characters_out(self.raw, pos, WHITESPACES)
+
+            # Step 7
+            descriptors = []
+
+            # Step 8.1
+            if url[-1] == ",":
+                while len(url) and url[-1] == ",":
+                    url = url[:-1]
+                # JUMP to descriptor parser
+            else:
+                # Step 8.e.1
+                pos, _ = collect_characters_in(self.raw, pos, WHITESPACES)
+
+                # Step 8.e.2
+                current_descriptor = ""
+                state = STATE_IN_DESCRIPTOR
+
+                # Step 8.e.4
+                while True:
+                    if pos < len(self.raw):
+                        cc = self.raw[pos]
+                    else:
+                        cc = None
+                    if state == STATE_IN_DESCRIPTOR:
+                        if cc in WHITESPACES:
+                            if current_descriptor:
+                                descriptors.append(current_descriptor)
+                                current_descriptor = ""
+                                state = STATE_AFTER_DESCRIPTOR
+                        elif cc == ",":
+                            pos = pos + 1
+                            if current_descriptor:
+                                descriptors.append(current_descriptor)
+                                # JUMP to descriptor parser
+                            break
+                        elif cc == "(":
+                            current_descriptor = current_descriptor + cc
+                            state = STATE_IN_PARENS
+                        elif cc is None:
+                            if current_descriptor:
+                                descriptors.append(current_descriptor)
+                            # JUMP to descriptor parser
+                            break
+                        else:
+                            current_descriptor = current_descriptor + cc
+                    elif state == STATE_IN_PARENS:
+                        if cc == ")":
+                            current_descriptor = current_descriptor + cc
+                            state = STATE_IN_DESCRIPTOR
+                        elif cc is None:
+                            descriptors.append(current_descriptor)
+                            # JUMP to descriptor parser
+                            break
+                        else:
+                            current_descriptor = current_descriptor + cc
+                    elif state == STATE_AFTER_DESCRIPTOR:
+                        if cc in WHITESPACES:
+                            pass
+                        elif cc is None:
+                            # JUMP to descriptor parser
+                            break
+                        else:
+                            state = STATE_IN_DESCRIPTOR
+                            pos = pos - 1
+                    pos = pos + 1
+
+            # Step 9, 10, 11, 12 (descriptor parser)
+            error = False
+            width = None
+            density = None
+            h = None
+
+            # Step 13
+            # print("Descriptors", descriptors)
+            for descriptor in descriptors:
+                if len(descriptor) >= 2:
+                    last_char = descriptor[-1]
+                    value = descriptor[:-1]
+                    if last_char == "w":
+                        try:
+                            conv_value = int(value)
+                        except ValueError:
+                            error = True
+                        else:
+                            if width or density:
+                                error = True
+                            elif conv_value <= 0:
+                                error = True
+                            elif not value.isdigit():
+                                error = True
+                            else:
+                                width = value
+                    elif last_char == "x":
+                        try:
+                            conv_value = float(value)
+                        except ValueError:
+                            error = True
+                        else:
+                            if width or density or h:
+                                error = True
+                            elif conv_value < 0:
+                                error = True
+                            elif value[-1] == ".":
+                                error = True
+                            elif value[0] == "+":
+                                error = True
+                            elif math.isinf(conv_value):
+                                error = True
+                            elif math.isnan(conv_value):
+                                error = True
+                            else:
+                                density = value
+                    elif last_char == "h":
+                        try:
+                            conv_value = int(value)
+                        except ValueError:
+                            error = True
+                        else:
+                            if h or density:
+                                error = True
+                            elif conv_value <= 0:
+                                error = True
+                            elif not value.isdigit():
+                                error = True
+                            else:
+                                h = value
+                    else:
+                        error = True
+                else:
+                    error = True
+
+            if h and not width:
+                error = True
+
+            if not error:
+                candidates.append({"url": url, "w": width, "x": density, "h": h})
+
+    def stringify(self):
+        """
+        Returns string which is a valid srcset attribute
+        """
+        result = ""
+        for item in self.candidates:
+            if result:
+                result = result + ", "
+            result = result + item["url"]
+            if item["w"]:
+                result = result + " %sw" % item["w"]
+            if item["x"]:
+                result = result + " %sx" % item["x"]
+            if item["h"]:
+                result = result + " %sh" % item["h"]
+        return result
+
+
+def collect_characters_in(string, start, charset):
+    """
+    Collect all characters from `start` which are part of the `charset`
+    """
+    pos = start
+    while pos < len(string) and string[pos] in charset:
+        pos = pos + 1
+    return pos, string[start:pos]
+
+
+def collect_characters_out(string, start, charset):
+    """
+    Collect all characters from `start` until one of the characters from `charset`
+    is found
+    """
+    pos = start
+    while pos < len(string) and string[pos] not in charset:
+        pos = pos + 1
+    return pos, string[start:pos]

repub/utils.py

@@ -0,0 +1,74 @@
+import hashlib
+import mimetypes
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+from scrapy.utils.python import to_bytes
+
+
+class FileType(Enum):
+    """File types that the republisher can handle"""
+
+    VIDEO = "video"
+    IMAGE = "image"
+    AUDIO = "audio"
+    FILE = "file"
+
+
+def local_image_path(name: str) -> str:
+    image_guid = hashlib.sha1(to_bytes(name)).hexdigest()  # nosec
+    return f"full/{image_guid}.jpg"
+
+
+def local_file_path(s: str) -> str:
+    media_guid = hashlib.sha1(to_bytes(s)).hexdigest()  # nosec
+    media_ext = Path(s).suffix
+    # Handles empty and wild extensions by trying to guess the
+    # mime type then extension or default to empty string otherwise
+    if media_ext not in mimetypes.types_map:
+        media_ext = ""
+        media_type = mimetypes.guess_type(s)[0]
+        if media_type:
+            media_ext = mimetypes.guess_extension(media_type)
+    return f"{media_guid}{media_ext}"
+
+
+def local_video_path(s: str) -> str:
+    return local_file_path(s)
+
+
+def local_audio_path(s: str) -> str:
+    return local_file_path(s)
+
+
+def determine_file_type(
+    url: str, medium: Optional[str] = None, mimetype: Optional[str] = None
+):
+    """
+    Uses all available information to determine the type of a file from a path/url
+    """
+    if medium:
+        if medium == "video":
+            return FileType.VIDEO
+        if medium == "audio":
+            return FileType.AUDIO
+        if medium == "image":
+            return FileType.IMAGE
+        if medium == "document":
+            return FileType.FILE
+        if medium == "executable":
+            return FileType.FILE
+
+    if not mimetype:
+        mimetype = mimetypes.guess_type(url)[0]
+
+    if mimetype:
+        if mimetype.startswith("image"):
+            return FileType.IMAGE
+        if mimetype.startswith("audio"):
+            return FileType.AUDIO
+        if mimetype.startswith("video"):
+            return FileType.VIDEO
+
+    return FileType.FILE

scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = repub.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = repub

shell.nix

@@ -1,33 +0,0 @@
-{ system ? "x86_64-linux", pkgs ? import <nixpkgs> { inherit system; } }:
-
-let
-  packages = [
-    pkgs.python311
-    pkgs.poetry
-    pkgs.zsh
-  ];
-
-  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
-    pkgs.stdenv.cc.cc
-    # Add any missing library needed
-    # You can use the nix-index package to locate them, e.g. nix-locate -w --top-level --at-root /lib/libudev.so.1
-  ];
-
-  # Put the venv on the repo, so direnv can access it
-  POETRY_VIRTUALENVS_IN_PROJECT = "true";
-  POETRY_VIRTUALENVS_PATH = "{project-dir}/.venv";
-
-  # Use python from path, so you can use a different version to the one bundled with poetry
-  POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON = "true";
-in
-  pkgs.mkShell {
-    buildInputs = packages;
-    shellHook = ''
-      export SHELL=${pkgs.zsh}
-      export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}"
-      export POETRY_VIRTUALENVS_IN_PROJECT="${POETRY_VIRTUALENVS_IN_PROJECT}"
-      export POETRY_VIRTUALENVS_PATH="${POETRY_VIRTUALENVS_PATH}"
-      export POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON="${POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON}"
-      export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
-    '';
-  }

tests/test_entrypoint.py

@@ -0,0 +1,17 @@
+from types import SimpleNamespace
+
+from repub.entrypoint import FeedNameFilter
+
+
+def test_feed_name_filter_accepts_matching_item() -> None:
+    item = SimpleNamespace(feed_name="nasa")
+    feed_filter = FeedNameFilter({"feed_name": "nasa"})
+
+    assert feed_filter.accepts(item) is True
+
+
+def test_feed_name_filter_rejects_non_matching_item() -> None:
+    item = SimpleNamespace(feed_name="gp-pod")
+    feed_filter = FeedNameFilter({"feed_name": "nasa"})
+
+    assert feed_filter.accepts(item) is False

treefmt.nix

@@ -0,0 +1,13 @@
+{ ... }:
+{
+  projectRootFile = "flake.nix";
+
+  programs.nixfmt.enable = true;
+
+  programs.black.enable = true;
+
+  programs.isort = {
+    enable = true;
+    profile = "black";
+  };
+}