Support VP8, VP9 decoding (#741)

* #594 add VP8 and VP9 codecs

* #594 forbid to set up VP8 and VP9 encoding in module

* #594 use WebM container for VP8/VP9 in video sink adapter

adapters/ds/sinks/always_on_rtsp/input_pipeline.py

@@ -54,6 +54,14 @@ def link_added_pad(
     src_pad: Gst.Pad,
     sink_pad: Gst.Pad,
 ):
+    logger.debug(
+        'Linking %s.%s to %s.%s',
+        src_pad.get_parent().get_name(),
+        src_pad.get_name(),
+        sink_pad.get_parent().get_name(),
+        sink_pad.get_name(),
+    )
+
     assert src_pad.link(sink_pad) == Gst.PadLinkReturn.OK
 
 
@@ -92,6 +100,7 @@ def on_demuxer_pad_added(
         capsfilter.sync_state_with_parent()
     else:
         decodebin = factory.create(PipelineElement('decodebin'))
+        decodebin.set_property('sink-caps', caps)
         pipeline.add(decodebin)
         decodebin_sink_pad: Gst.Pad = decodebin.get_static_pad('sink')
         decodebin.connect('pad-added', link_added_pad, sink_pad)

adapters/gst/gst_plugins/python/media_files_src_bin.py

@@ -322,24 +322,35 @@ def attach_parser(self, pad: Gst.Pad, caps: Gst.Caps):
             )
             return False
 
-        self.logger.debug(
-            'Attaching parser to pad %s.%s with caps %s',
-            pad.get_parent().get_name(),
-            pad.get_name(),
-            caps.to_string(),
-        )
-
-        parser: Gst.Element = Gst.ElementFactory.make(codec.value.parser)
-        if codec.value.parser in ['h264parse', 'h265parse']:
-            # Send VPS, SPS and PPS with every IDR frame
-            # h26xparse cannot start stream without VPS, SPS or PPS in the first frame
-            self.logger.debug('Set config-interval of %s to %s', parser.get_name(), -1)
-            parser.set_property('config-interval', -1)
-        self.add(parser)
-        self._elements.append(parser)
-        assert pad.link(parser.get_static_pad('sink')) == Gst.PadLinkReturn.OK
+        last_pad = pad
+        if codec.value.parser is not None:
+            self.logger.debug(
+                'Attaching parser to pad %s.%s with caps %s',
+                last_pad.get_parent().get_name(),
+                last_pad.get_name(),
+                caps.to_string(),
+            )
+            parser: Gst.Element = Gst.ElementFactory.make(codec.value.parser)
+            if codec.value.parser in ['h264parse', 'h265parse']:
+                # Send VPS, SPS and PPS with every IDR frame
+                # h26xparse cannot start stream without VPS, SPS or PPS in the first frame
+                self.logger.debug(
+                    'Set config-interval of %s to %s', parser.get_name(), -1
+                )
+                parser.set_property('config-interval', -1)
+            self.add(parser)
+            self._elements.append(parser)
+            assert last_pad.link(parser.get_static_pad('sink')) == Gst.PadLinkReturn.OK
+            last_pad = parser.get_static_pad('src')
+            parser.sync_state_with_parent()
 
         filtered_caps = Gst.Caps.from_string(codec.value.caps_with_params)
+        self.logger.debug(
+            'Attaching capsfilter to pad %s.%s with caps %s',
+            last_pad.get_parent().get_name(),
+            last_pad.get_name(),
+            filtered_caps.to_string(),
+        )
         self.capsfilter.set_property('caps', filtered_caps)
         modified_caps = Gst.Caps.from_string(filtered_caps[0].get_name())
         if self.file_type == FileType.IMAGE:
@@ -350,12 +361,14 @@ def attach_parser(self, pad: Gst.Pad, caps: Gst.Caps):
                 Gst.Fraction(self.framerate.numerator, self.framerate.denominator),
             )
         self.capssetter.set_property('caps', modified_caps)
-        assert parser.link(self.capsfilter)
+        assert (
+            last_pad.link(self.capsfilter.get_static_pad('sink'))
+            == Gst.PadLinkReturn.OK
+        )
         assert self.src_pad.set_active(True)
         self.send_file_location(self.source.get_property('location'))
         self.capssetter.sync_state_with_parent()
         self.capsfilter.sync_state_with_parent()
-        parser.sync_state_with_parent()
 
         return True
 

adapters/gst/sinks/video_files.py

@@ -246,25 +246,50 @@ def _open(self):
         )
         appsrc_name = 'appsrc'
         filesink_name = 'filesink'
-        self.pipeline: Gst.Pipeline = Gst.parse_launch(
-            ' ! '.join(
-                [
-                    f'appsrc name={appsrc_name} emit-signals=false format=time',
-                    'queue',
-                    'adjust_timestamps',
-                    self.frame_params.codec.value.parser,
-                    'qtmux fragment-duration=1000 fragment-mode=first-moov-then-finalise',
-                    f'filesink name={filesink_name}',
-                ]
-            ),
-        )
+        pipeline = [
+            f'appsrc name={appsrc_name} emit-signals=false format=time',
+            'queue',
+            'adjust_timestamps',
+        ]
+
+        if self.frame_params.codec.value.parser is not None:
+            pipeline.append(self.frame_params.codec.value.parser)
+
+        if self.frame_params.codec in [
+            Codec.H264,
+            Codec.HEVC,
+            Codec.JPEG,
+            Codec.PNG,
+        ]:
+            pipeline.append(
+                'qtmux fragment-duration=1000 fragment-mode=first-moov-then-finalise'
+            )
+            file_ext = 'mov'
+        elif self.frame_params.codec in [
+            Codec.VP8,
+            Codec.VP9,
+        ]:
+            pipeline.append('webmmux')
+            file_ext = 'webm'
+        else:
+            self.logger.error(
+                'Unsupported codec %s for source %s',
+                self.frame_params.codec,
+                self.source_id,
+            )
+            return
+
+        pipeline.append(f'filesink name={filesink_name}')
+        self.pipeline: Gst.Pipeline = Gst.parse_launch(' ! '.join(pipeline))
         self.pipeline.set_name(f'video_chunk_{self.source_id}_{self.chunk_idx}')
         self.appsrc: GstApp.AppSrc = self.pipeline.get_by_name(appsrc_name)
         self.appsrc.set_caps(self.caps)
 
         filesink: Gst.Element = self.pipeline.get_by_name(filesink_name)
         os.makedirs(self.base_location, exist_ok=True)
-        dst_location = os.path.join(self.base_location, f'{self.chunk_idx:04}.mov')
+        dst_location = os.path.join(
+            self.base_location, f'{self.chunk_idx:04}.{file_ext}'
+        )
         self.logger.info(
             'Writing video from source %s to file %s', self.source_id, dst_location
         )
@@ -321,7 +346,14 @@ def _write_video_frame(
         self, video_frame: VideoFrame, content: Optional[bytes]
     ) -> bool:
         frame_params = FrameParams.from_video_frame(video_frame)
-        if frame_params.codec not in [Codec.H264, Codec.HEVC, Codec.JPEG, Codec.PNG]:
+        if frame_params.codec not in [
+            Codec.H264,
+            Codec.HEVC,
+            Codec.VP8,
+            Codec.VP9,
+            Codec.JPEG,
+            Codec.PNG,
+        ]:
             self.logger.error(
                 'Frame %s/%s has unsupported codec %s',
                 video_frame.source_id,

savant/deepstream/encoding.py

@@ -24,6 +24,13 @@ def check_encoder_is_available(parameters: Dict[str, Any]) -> bool:
         return True
 
     codec = CODEC_BY_NAME[codec_name]
+    if codec.value.is_raw:
+        return True
+
+    if codec.value.sw_encoder is None and codec.value.nv_encoder is None:
+        logger.error('Encoding for %r is not supported.', codec_name)
+        return False
+
     if codec not in [Codec.H264, Codec.HEVC]:
         return True
 

savant/gstreamer/codecs.py

@@ -85,6 +85,7 @@ def encoder(self, encoder_type: Optional[str]) -> Optional[str]:
 class Codec(Enum):
     """Codec enum."""
 
+    # Video codecs
     H264 = CodecInfo(
         'h264',
         'video/x-h264',
@@ -100,8 +101,23 @@ class Codec(Enum):
         'h265parse',
         nv_encoder='nvv4l2h265enc',
     )
+    VP8 = CodecInfo(
+        'vp8',
+        'video/x-vp8',
+        [],
+    )
+    VP9 = CodecInfo(
+        'vp9',
+        'video/x-vp9',
+        [],
+        'vp9parse',
+    )
+
+    # Raw video codecs
     RAW_RGBA = CodecInfo('raw-rgba', 'video/x-raw', ['format=RGBA'], is_raw=True)
     RAW_RGB24 = CodecInfo('raw-rgb24', 'video/x-raw', ['format=RGB'], is_raw=True)
+
+    # Image codecs
     PNG = CodecInfo(
         'png',
         'image/png',