implement 2-pass vp9/webm transcoding

repub/entrypoint.py

@@ -56,7 +56,7 @@ def execute_spider(queue, name, url):
             "REPUBLISHER_FILE_DIR": "files",
             "IMAGES_STORE": f"out/{name}/images",
             "AUDIO_STORE": f"out/{name}/audio",
-            "VIDEO_STORE": f"out/{name}/images",
+            "VIDEO_STORE": f"out/{name}/videos",
             "FILES_STORE": f"out/{name}/files",
         }
         if not check_runtime(

repub/media.py

@@ -208,7 +208,7 @@ def transcode_audio(input_file: str, output_dir: str, params: Dict[str, str]) ->
     output_file = f"{output_dir}/converted.{ext}"
     try:
         logger.info(
-            f"Compressing audio {input_file} to {output_file} with params={params}"
+            f"Transcoding audio {input_file} to {output_file} with params={params}"
         )
         out, _ = (
             ffmpeg.input(input_file)
@@ -272,14 +272,22 @@ def video_transcode_params(
     if is_good_height and is_container and is_vcodec and is_acodec and is_audio_bitrate:
         return None
 
-    params = {"extension": settings["extension"], "strict": "-2"}
+    passes = settings.get("passes", [])
-    if not is_good_height:
+    params = {"extension": settings["extension"]}
-        params["vf"] = f"scale={width}:{height}"
+    if len(passes) == 0 or is_vcodec:
-
+        if not is_good_height:
-    if not is_vcodec:
+            params["vf"] = f"scale={width}:{height}"
-        params.update(settings["ffmpeg_video_params"])
+        if not is_vcodec:
-    if not is_acodec or not is_audio_bitrate:
+            params.update(settings["ffmpeg_video_params"])
-        params.update(settings["ffmpeg_audio_params"])
+        if not is_acodec or not is_audio_bitrate:
+            params.update(settings["ffmpeg_audio_params"])
+        return params
+    params["passes"] = []
+    for p in passes:
+        p = copy.deepcopy(p)
+        if not is_good_height:
+            p["vf"] = f"scale={width}:{height}"
+        params["passes"].append(p)
     return params
 
 
@@ -292,17 +300,40 @@ def transcode_video(input_file: str, output_dir: str, params: Dict[str, Any]) ->
     output_file = f"{output_dir}/converted.{ext}"
     try:
         logger.info(
-            f"Compressing video {input_file} to {output_file} with params={params}"
+            f"Transcoding video {input_file} to {output_file} with params={params}"
         )
-        out, _ = (
+        if "passes" not in params:
-            ffmpeg.input(input_file)
+            out, _ = (
-            .output(
+                ffmpeg.input(input_file)
-                output_file,
+                .output(
-                **params,
+                    output_file,
-                loglevel="quiet",
+                    **params,
+                    # loglevel="quiet",
+                )
+                .run()
             )
-            .run()
+        else:
-        )
+            passes = params["passes"]
+            ffinput = ffmpeg.input(input_file)
+            video = ffinput.video
+            audio = ffinput.audio
+            ffoutput = ffinput.output(video, "pipe:", **passes[0])
+            ffoutput = ffoutput.global_args(
+                # "-loglevel", "quiet",
+                "-stats"
+            )
+            logger.info("Running pass #1")
+            std_out, std_err = ffoutput.run(capture_stdout=True)
+            print(std_out)
+            print(std_err)
+            logger.info("Running pass #2")
+            ffoutput = ffinput.output(video, audio, output_file, **passes[1])
+            ffoutput = ffoutput.global_args(
+                # "-loglevel", "quiet",
+                "-stats"
+            )
+            ffoutput.run(overwrite_output=True)
+
         before = os.path.getsize(input_file) / 1024
         after = os.path.getsize(output_file) / 1024
         percent_difference = 0
@@ -313,7 +344,13 @@ def transcode_video(input_file: str, output_dir: str, params: Dict[str, Any]) ->
         )
         return output_file
     except ffmpeg.Error as e:
-        raise RuntimeError(f"Failed to load video: {e.stderr.decode()}") from e
+        print(e.stderr, file=sys.stderr)
+        logger.error(f"Failed to transcode")
+        logger.error(e)
+        raise RuntimeError(f"Failed to transcode video: {e.stderr.decode()}") from e
+    except Exception as e:
+        logger.critical(e, exc_info=True)
+        raise e
 
 
 def check_codecs(codecs: List[str]) -> List[str]:

repub/settings.py

@@ -142,11 +142,43 @@ REPUBLISHER_VIDEO = [
         "max_height": 720,
         "mimetype": "video/mp4",
         "extension": "mp4",
-    }
+    },
+    # {
+    #    "passes": [
+    #        {
+    #            "c:v": "libvpx-vp9",
+    #            "b:v": "0",
+    #            "crf": "30",
+    #            "pass": "1",
+    #            "deadline": "good",
+    #            "row-mt": "1",
+    #            "f": "null",
+    #        },
+    #        {
+    #            "c:v": "libvpx-vp9",
+    #            "b:v": "0",
+    #            "crf": "30",
+    #            "pass": "2",
+    #            "deadline": "good",
+    #            "row-mt": "1",
+    #            "c:a": "libopus",
+    #            "b:a": "96k",
+    #            "ac": "2",
+    #        },
+    #    ],
+    #    "name": "720",
+    #    "container": "webm",
+    #    "vcodec": "libvpx-vp9",
+    #    "acodec": "opus",
+    #    "audio_max_bitrate": 96000,
+    #    "max_height": 720,
+    #    "mimetype": "video/webm",
+    #    "extension": "webm",
+    # },
 ]
 
-REPUBLISHER_FFMPEG_ENCODERS = ["libmp3lame", "libfdk_aac"]
+REPUBLISHER_FFMPEG_ENCODERS = ["libmp3lame", "libfdk_aac", "libvpx-vp9", "libopus"]
-REPUBLISHER_FFMPEG_CODECS = ["aac", "mp3", "mpeg4", "vp9", "vorbis"]
+REPUBLISHER_FFMPEG_CODECS = ["aac", "mp3", "mpeg4", "vp9", "opus"]
 
 
 CLOSESPIDER_ERRORCOUNT = 1