Add media retention cleanup command

repub/crawl.py

@@ -7,6 +7,7 @@ from scrapy.crawler import Crawler, CrawlerProcess
 from scrapy.settings import Settings
 from twisted.python.failure import Failure
 
+from repub.cleanup import media_retention_lock
 from repub.config import (
     FeedConfig,
     build_base_settings,
@@ -103,8 +104,9 @@ def run_feeds(
         deferred.addCallbacks(handle_success, handle_error)
         deferred.addBoth(crawl_next)
 
-    crawl_next()
-    process.start(stop_after_crawl=False)
+    with media_retention_lock(out_dir=out_dir, exclusive=False):
+        crawl_next()
+        process.start(stop_after_crawl=False)
 
     return 1 if any(failure is not None for _, failure in results) else 0