pixelflux/webcam/
decode.rs

1/*
2 * This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at https://mozilla.org/MPL/2.0/.
5 */
6
7//! Decoders for the client camera uplink, and for the checks that read a session's own output
8//! back.
9//!
10//! Clients send whatever their browser can produce: H.264 (WebCodecs or the WebRTC media track),
11//! VP8/VP9/AV1/HEVC (same sources), or MJPEG from the canvas fallback. Every codec lands in the
12//! same I420 view so the rest of the pipeline is codec-agnostic. H.264 decodes through Cisco
13//! OpenH264, VP8 and VP9 through libvpx, AV1 through dav1d, HEVC through libde265, and MJPEG
14//! through TurboJPEG, which pixelflux already uses for its own JPEG stripes. Decoding is
15//! software here; a hardware decoder only changes which `Decoder` is constructed.
16
17use std::ffi::{CStr, c_int, c_void};
18use std::ptr;
19
20use turbojpeg::{Decompressor, Image, PixelFormat, Subsamp, YuvImage};
21
22use super::convert::{I420Buffer, I420View};
23
24/// Input codecs, by wire id. The ids are part of the Selkies WebSocket framing and are exported to
25/// Python as `VirtualCamera.CODEC_*`.
26pub use crate::encoders::codec::Codec;
27use crate::encoders::codec::{
28    FRAME_KEY, annexb_nals, av1_is_key, h264_frame_type, h265_frame_type, vp8_is_key, vp9_is_key,
29};
30use crate::encoders::sps;
31
32/// Whether an encoded frame is a key frame, read from the bitstream itself: every codec here
33/// declares it in its first bytes or NAL types.
34pub fn sniff_keyframe(codec: Codec, data: &[u8]) -> Option<bool> {
35    Some(match codec {
36        Codec::Jpeg => true,
37        Codec::H264 => h264_frame_type(data) == FRAME_KEY,
38        Codec::H265 => h265_frame_type(data) == FRAME_KEY,
39        Codec::Vp8 => vp8_is_key(data),
40        Codec::Vp9 => vp9_is_key(data),
41        Codec::Av1 => av1_is_key(data),
42    })
43}
44
45#[derive(Debug)]
46pub enum DecodeError {
47    /// The packet could not be decoded; the stream needs a keyframe to resynchronize.
48    Corrupt(String),
49    /// The decoder itself failed and must be recreated.
50    Fatal(String),
51}
52
53pub trait Decoder {
54    fn codec(&self) -> Codec;
55    /// Decode one encoded frame. `Ok(true)` when `frame()` now holds a new picture.
56    fn decode(&mut self, data: &[u8]) -> Result<bool, DecodeError>;
57    fn frame(&self) -> Option<I420View<'_>>;
58}
59
60pub fn new_decoder(codec: Codec) -> Result<Box<dyn Decoder>, String> {
61    match codec {
62        Codec::Jpeg => Ok(Box::new(JpegDecoder::new()?)),
63        _ => Ok(Box::new(VideoDecoder::new(codec)?)),
64    }
65}
66
67/// The color a decoded stream declares: the matrix as ISO/IEC 23091-2 codes it (1 BT.709,
68/// 5 BT.470BG, 6 SMPTE 170M, 2 unspecified) and whether the range is full.
69#[derive(Clone, Copy, Debug, PartialEq, Eq)]
70pub struct ColorTags {
71    pub matrix: u8,
72    pub full_range: bool,
73}
74
75impl ColorTags {
76    pub const BT709_LIMITED: Self = Self {
77        matrix: 1,
78        full_range: false,
79    };
80    pub const BT709_FULL: Self = Self {
81        matrix: 1,
82        full_range: true,
83    };
84    pub const BT470BG_LIMITED: Self = Self {
85        matrix: 5,
86        full_range: false,
87    };
88    pub const BT470BG_FULL: Self = Self {
89        matrix: 5,
90        full_range: true,
91    };
92}
93
94/// The chroma sampling of a decoded picture.
95#[derive(Clone, Copy, PartialEq, Eq)]
96enum Chroma {
97    C420,
98    C422,
99    C444,
100}
101
102/// One decoded picture as a backend hands it back: borrowed planes, valid until the backend
103/// decodes again.
104struct Picture {
105    width: usize,
106    height: usize,
107    chroma: Chroma,
108    planes: [*const u8; 3],
109    strides: [usize; 3],
110    tags: Option<ColorTags>,
111}
112
113impl Picture {
114    unsafe fn plane(&self, i: usize, rows: usize, cols: usize) -> &[u8] {
115        unsafe { std::slice::from_raw_parts(self.planes[i], self.strides[i] * (rows - 1) + cols) }
116    }
117}
118
119/// A software decoder for one of the inter-coded codecs.
120pub struct VideoDecoder {
121    codec: Codec,
122    backend: Backend,
123    picture: Option<Picture>,
124    /// Planar copy of pictures that are not 4:2:0, or that a backend does not keep for us.
125    converted: I420Buffer,
126    from_converted: bool,
127}
128
129unsafe impl Send for VideoDecoder {}
130
131enum Backend {
132    H264(H264Backend),
133    Vpx(VpxBackend),
134    Av1(Box<Dav1dBackend>),
135    Hevc(De265Backend),
136}
137
138impl VideoDecoder {
139    pub fn new(codec: Codec) -> Result<Self, String> {
140        let backend = match codec {
141            Codec::H264 => Backend::H264(H264Backend::new()?),
142            Codec::Vp8 | Codec::Vp9 => Backend::Vpx(VpxBackend::new(codec)?),
143            Codec::Av1 => Backend::Av1(Box::new(Dav1dBackend::new()?)),
144            Codec::H265 => Backend::Hevc(De265Backend::new()?),
145            Codec::Jpeg => return Err("MJPEG decodes through the JPEG decoder".into()),
146        };
147        Ok(Self {
148            codec,
149            backend,
150            picture: None,
151            converted: I420Buffer::new(2, 2),
152            from_converted: false,
153        })
154    }
155
156    /// The color matrix and range the last decoded frame carries, as the bitstream declared
157    /// them; unspecified where the codec cannot declare a matrix.
158    pub fn color_tags(&self) -> Option<ColorTags> {
159        self.picture.as_ref()?.tags
160    }
161
162    /// Average a 4:2:2 or 4:4:4 picture's chroma into the I420 copy.
163    fn convert(&mut self) {
164        let Some(p) = &self.picture else { return };
165        let (w, h) = (p.width, p.height);
166        let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
167        self.converted.resize(w, h);
168        let y_len = self.converted.y_len();
169        let uv_len = self.converted.uv_len();
170        let (yp, rest) = self.converted.data.split_at_mut(y_len);
171        let (up, vp) = rest.split_at_mut(uv_len);
172        let y = unsafe { p.plane(0, h, w) };
173        for row in 0..h {
174            yp[row * w..row * w + w]
175                .copy_from_slice(&y[row * p.strides[0]..row * p.strides[0] + w]);
176        }
177        let (src_cw, src_ch) = match p.chroma {
178            Chroma::C420 => (cw, ch),
179            Chroma::C422 => (cw, h),
180            Chroma::C444 => (w, h),
181        };
182        for (i, dst) in [(1usize, &mut *up), (2usize, &mut *vp)] {
183            let src = unsafe { p.plane(i, src_ch, src_cw) };
184            let s = p.strides[i];
185            for row in 0..ch {
186                for x in 0..cw {
187                    let sum: u32 = match p.chroma {
188                        Chroma::C420 => src[row * s + x] as u32 * 4,
189                        Chroma::C422 => {
190                            let (r0, r1) =
191                                ((2 * row).min(src_ch - 1), (2 * row + 1).min(src_ch - 1));
192                            (src[r0 * s + x] as u32 + src[r1 * s + x] as u32) * 2
193                        }
194                        Chroma::C444 => {
195                            let (r0, r1) =
196                                ((2 * row).min(src_ch - 1), (2 * row + 1).min(src_ch - 1));
197                            let (x0, x1) = (2 * x, (2 * x + 1).min(src_cw - 1));
198                            src[r0 * s + x0] as u32
199                                + src[r0 * s + x1] as u32
200                                + src[r1 * s + x0] as u32
201                                + src[r1 * s + x1] as u32
202                        }
203                    };
204                    dst[row * cw + x] = ((sum + 2) / 4) as u8;
205                }
206            }
207        }
208    }
209}
210
211impl Decoder for VideoDecoder {
212    fn codec(&self) -> Codec {
213        self.codec
214    }
215
216    fn decode(&mut self, data: &[u8]) -> Result<bool, DecodeError> {
217        self.picture = None;
218        let picture = match &mut self.backend {
219            Backend::H264(b) => b.decode(data)?,
220            Backend::Vpx(b) => b.decode(data)?,
221            Backend::Av1(b) => b.decode(data)?,
222            Backend::Hevc(b) => b.decode(data)?,
223        };
224        let Some(picture) = picture else {
225            return Ok(false);
226        };
227        self.from_converted = picture.chroma != Chroma::C420;
228        self.picture = Some(picture);
229        if self.from_converted {
230            self.convert();
231        }
232        Ok(true)
233    }
234
235    fn frame(&self) -> Option<I420View<'_>> {
236        let p = self.picture.as_ref()?;
237        let full_range = p.tags.is_some_and(|t| t.full_range);
238        if self.from_converted {
239            return Some(self.converted.view(full_range));
240        }
241        let (w, h) = (p.width, p.height);
242        let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
243        Some(I420View {
244            width: w,
245            height: h,
246            y: unsafe { p.plane(0, h, w) },
247            u: unsafe { p.plane(1, ch, cw) },
248            v: unsafe { p.plane(2, ch, cw) },
249            y_stride: p.strides[0],
250            uv_stride: p.strides[1].min(p.strides[2]),
251            full_range,
252        })
253    }
254}
255
256/// Cisco OpenH264, driven through its decoder interface: it conceals the blocks a stream
257/// lost and hands back every picture it decodes, one behind a gap included, and the picture
258/// stays in its own buffer until the next call.
259struct H264Backend {
260    decoder: *mut openh264_sys2::ISVCDecoder,
261    tags: Option<ColorTags>,
262}
263
264impl Drop for H264Backend {
265    fn drop(&mut self) {
266        unsafe {
267            if let Some(uninit) = self.api().and_then(|a| a.Uninitialize) {
268                uninit(self.decoder);
269            }
270            openh264_sys2::source::APILoader::WelsDestroyDecoder(self.decoder);
271        }
272    }
273}
274
275impl H264Backend {
276    /// The decoder's method table, behind the two pointers OpenH264 hands out.
277    fn api(&self) -> Option<&openh264_sys2::ISVCDecoderVtbl> {
278        unsafe { self.decoder.as_ref().and_then(|d| d.as_ref()) }
279    }
280
281    fn new() -> Result<Self, String> {
282        use openh264_sys2::*;
283        let mut decoder: *mut ISVCDecoder = ptr::null_mut();
284        if unsafe { source::APILoader::WelsCreateDecoder(&mut decoder) } != 0 {
285            return Err("OpenH264 created no decoder".into());
286        }
287        let me = Self {
288            decoder,
289            tags: None,
290        };
291        let initialize = me
292            .api()
293            .and_then(|a| a.Initialize)
294            .ok_or("OpenH264 exposes no Initialize")?;
295        let mut param: SDecodingParam = unsafe { std::mem::zeroed() };
296        param.eEcActiveIdc = ERROR_CON_SLICE_MV_COPY_CROSS_IDR;
297        param.sVideoProperty.size = std::mem::size_of::<SVideoProperty>() as u32;
298        param.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC;
299        if unsafe { initialize(decoder, &param) } != 0 {
300            return Err("OpenH264 refused to initialize".into());
301        }
302        Ok(me)
303    }
304
305    fn decode(&mut self, data: &[u8]) -> Result<Option<Picture>, DecodeError> {
306        use openh264_sys2::*;
307        let mut sps = None;
308        let mut scalable = false;
309        for nal in annexb_nals(data) {
310            match nal[0] & 0x1f {
311                7 if sps.is_none() => sps = Some(nal),
312                14 | 15 | 20 => scalable = true,
313                _ => {}
314            }
315        }
316        if let Some(sps) = sps {
317            self.tags = Some(match sps::read_color(sps) {
318                Some(signal) => ColorTags {
319                    matrix: signal.matrix,
320                    full_range: signal.full_range,
321                },
322                None => ColorTags {
323                    matrix: 2,
324                    full_range: false,
325                },
326            });
327        }
328        // OpenH264 also decodes the scalable extension, and a prefix or extension NAL unit naming
329        // a layer above the base turns it to that layer for good, so it hands over no picture
330        // again, key frames included. An AVC decoder ignores the extension's units (prefix,
331        // subset SPS, extension slice), and so does this one.
332        let mut avc = Vec::new();
333        let data = if scalable {
334            for nal in annexb_nals(data).filter(|n| !matches!(n[0] & 0x1f, 14 | 15 | 20)) {
335                avc.extend_from_slice(&[0, 0, 0, 1]);
336                avc.extend_from_slice(nal);
337            }
338            &avc[..]
339        } else {
340            data
341        };
342        let mut dst: [*mut u8; 3] = [ptr::null_mut(); 3];
343        let mut info: SBufferInfo = unsafe { std::mem::zeroed() };
344        let decode = self
345            .api()
346            .and_then(|a| a.DecodeFrameNoDelay)
347            .ok_or_else(|| DecodeError::Fatal("OpenH264 exposes no DecodeFrameNoDelay".into()))?;
348        let mut state = unsafe {
349            decode(
350                self.decoder,
351                data.as_ptr(),
352                data.len() as c_int,
353                dst.as_mut_ptr(),
354                &mut info,
355            )
356        };
357        // A profile above baseline has the decoder hold each picture back until the next
358        // one arrives, against reordering the stream never does: the held picture is this
359        // access unit's, and flushing hands it over now. OpenH264 2.6 keeps a flushed
360        // picture's buffer when decoding single-threaded, so such a stream stalls once the
361        // pool runs dry; a Baseline stream is never held.
362        if info.iBufferStatus != 1
363            && let Some(flush) = self.api().and_then(|a| a.FlushFrame)
364        {
365            state |= unsafe { flush(self.decoder, dst.as_mut_ptr(), &mut info) };
366        }
367        if info.iBufferStatus != 1 || dst.iter().any(|p| p.is_null()) {
368            if state != dsErrorFree {
369                return Err(DecodeError::Corrupt(format!(
370                    "OpenH264 decoding state {state:#x}"
371                )));
372            }
373            return Ok(None);
374        }
375        let system = unsafe { info.UsrData.sSystemBuffer };
376        Ok(Some(Picture {
377            width: system.iWidth as usize,
378            height: system.iHeight as usize,
379            chroma: Chroma::C420,
380            planes: [dst[0], dst[1], dst[2]],
381            strides: [
382                system.iStride[0] as usize,
383                system.iStride[1] as usize,
384                system.iStride[1] as usize,
385            ],
386            tags: self.tags,
387        }))
388    }
389}
390
391/// libvpx's VP8 and VP9 decoders. VP9 declares its matrix and range in every frame; VP8's
392/// keyframe header carries one color-space bit, BT.601 or reserved, and a clamping bit.
393struct VpxBackend {
394    codec: Codec,
395    ctx: codec_sys::vpx::vpx_codec_ctx_t,
396    tags: Option<ColorTags>,
397}
398
399impl Drop for VpxBackend {
400    fn drop(&mut self) {
401        unsafe { codec_sys::vpx::vpx_codec_destroy(&mut self.ctx) };
402    }
403}
404
405/// VP8's `color_space` and `clamping_type` bits, the first two bool-coded fields of a key
406/// frame's first partition.
407fn vp8_color_bits(frame: &[u8]) -> Option<(bool, bool)> {
408    if !vp8_is_key(frame) || frame.len() < 12 {
409        return None;
410    }
411    let data = &frame[10..];
412    let mut value = ((data[0] as u32) << 8) | data[1] as u32;
413    let mut range = 255u32;
414    let mut bit_count = 0;
415    let mut at = 2;
416    let mut read = || {
417        let split = 1 + (((range - 1) * 128) >> 8);
418        let big = split << 8;
419        let bit = value >= big;
420        if bit {
421            range -= split;
422            value -= big;
423        } else {
424            range = split;
425        }
426        while range < 128 {
427            value <<= 1;
428            range <<= 1;
429            bit_count += 1;
430            if bit_count == 8 {
431                bit_count = 0;
432                value |= data.get(at).copied().unwrap_or(0) as u32;
433                at += 1;
434            }
435        }
436        bit
437    };
438    let color_space = read();
439    let clamping = read();
440    Some((color_space, clamping))
441}
442
443impl VpxBackend {
444    fn new(codec: Codec) -> Result<Self, String> {
445        use codec_sys::vpx::*;
446        let iface = unsafe {
447            if codec == Codec::Vp8 {
448                vpx_codec_vp8_dx()
449            } else {
450                vpx_codec_vp9_dx()
451            }
452        };
453        let mut ctx: vpx_codec_ctx_t = unsafe { std::mem::zeroed() };
454        if unsafe {
455            vpx_codec_dec_init_ver(
456                &mut ctx,
457                iface,
458                ptr::null(),
459                0,
460                VPX_DECODER_ABI_VERSION as c_int,
461            )
462        } != VPX_CODEC_OK
463        {
464            return Err(format!("libvpx opened no {} decoder", codec.name()));
465        }
466        Ok(Self {
467            codec,
468            ctx,
469            tags: None,
470        })
471    }
472
473    fn decode(&mut self, data: &[u8]) -> Result<Option<Picture>, DecodeError> {
474        use codec_sys::vpx::*;
475        if self.codec == Codec::Vp8
476            && let Some((reserved, clamping)) = vp8_color_bits(data)
477        {
478            self.tags = Some(ColorTags {
479                matrix: if reserved { 2 } else { 5 },
480                full_range: clamping,
481            });
482        }
483        let res = unsafe {
484            vpx_codec_decode(
485                &mut self.ctx,
486                data.as_ptr(),
487                data.len() as u32,
488                ptr::null_mut(),
489                0,
490            )
491        };
492        if res != VPX_CODEC_OK {
493            let detail = unsafe { CStr::from_ptr(vpx_codec_err_to_string(res)) }
494                .to_string_lossy()
495                .into_owned();
496            return Err(DecodeError::Corrupt(detail));
497        }
498        let mut iter: vpx_codec_iter_t = ptr::null_mut();
499        let mut image = None;
500        while let Some(next) =
501            ptr::NonNull::new(unsafe { vpx_codec_get_frame(&mut self.ctx, &mut iter) })
502        {
503            image = Some(next);
504        }
505        let Some(image) = image else { return Ok(None) };
506        let img = unsafe { image.as_ref() };
507        if img.bit_depth != 8 {
508            return Err(DecodeError::Fatal(format!(
509                "unsupported libvpx bit depth {}",
510                img.bit_depth
511            )));
512        }
513        let chroma = match (img.x_chroma_shift, img.y_chroma_shift) {
514            (1, 1) => Chroma::C420,
515            (1, 0) => Chroma::C422,
516            (0, 0) => Chroma::C444,
517            _ => {
518                return Err(DecodeError::Fatal(format!(
519                    "unsupported libvpx image format {}",
520                    img.fmt
521                )));
522            }
523        };
524        if self.codec == Codec::Vp9 {
525            let matrix = match img.cs {
526                VPX_CS_BT_601 => 5,
527                VPX_CS_BT_709 => 1,
528                VPX_CS_SMPTE_170 => 6,
529                VPX_CS_SMPTE_240 => 7,
530                VPX_CS_BT_2020 => 9,
531                VPX_CS_SRGB => 0,
532                _ => 2,
533            };
534            self.tags = Some(ColorTags {
535                matrix,
536                full_range: img.range == VPX_CR_FULL_RANGE,
537            });
538        }
539        Ok(Some(Picture {
540            width: img.d_w as usize,
541            height: img.d_h as usize,
542            chroma,
543            planes: [img.planes[0], img.planes[1], img.planes[2]],
544            strides: [
545                img.stride[0] as usize,
546                img.stride[1] as usize,
547                img.stride[2] as usize,
548            ],
549            tags: self.tags,
550        }))
551    }
552}
553
554/// dav1d, at the frame delay of one a live uplink wants.
555struct Dav1dBackend {
556    ctx: *mut codec_sys::dav1d::Dav1dContext,
557    picture: codec_sys::dav1d::Dav1dPicture,
558    held: bool,
559}
560
561impl Drop for Dav1dBackend {
562    fn drop(&mut self) {
563        unsafe {
564            if self.held {
565                codec_sys::dav1d::dav1d_picture_unref(&mut self.picture);
566            }
567            codec_sys::dav1d::dav1d_close(&mut self.ctx);
568        }
569    }
570}
571
572impl Dav1dBackend {
573    fn new() -> Result<Self, String> {
574        use codec_sys::dav1d::*;
575        let mut settings: Dav1dSettings = unsafe { std::mem::zeroed() };
576        unsafe { dav1d_default_settings(&mut settings) };
577        settings.n_threads = 2;
578        settings.max_frame_delay = 1;
579        settings.logger.callback = None;
580        let mut ctx: *mut Dav1dContext = ptr::null_mut();
581        if unsafe { dav1d_open(&mut ctx, &settings) } != 0 || ctx.is_null() {
582            return Err("dav1d opened no decoder".into());
583        }
584        Ok(Self {
585            ctx,
586            picture: unsafe { std::mem::zeroed() },
587            held: false,
588        })
589    }
590
591    fn decode(&mut self, data: &[u8]) -> Result<Option<Picture>, DecodeError> {
592        use codec_sys::dav1d::*;
593        const AGAIN: c_int = -(libc::EAGAIN);
594        let mut input: Dav1dData = unsafe { std::mem::zeroed() };
595        let buffer = unsafe { dav1d_data_create(&mut input, data.len()) };
596        if buffer.is_null() {
597            return Err(DecodeError::Fatal("dav1d allocated no input buffer".into()));
598        }
599        unsafe { ptr::copy_nonoverlapping(data.as_ptr(), buffer, data.len()) };
600        let mut got = false;
601        loop {
602            let sent = unsafe { dav1d_send_data(self.ctx, &mut input) };
603            if sent < 0 && sent != AGAIN {
604                unsafe { dav1d_data_unref(&mut input) };
605                return Err(DecodeError::Corrupt(format!(
606                    "dav1d refused the data ({sent})"
607                )));
608            }
609            loop {
610                let mut next: Dav1dPicture = unsafe { std::mem::zeroed() };
611                let res = unsafe { dav1d_get_picture(self.ctx, &mut next) };
612                if res == AGAIN {
613                    break;
614                }
615                if res < 0 {
616                    unsafe { dav1d_data_unref(&mut input) };
617                    return Err(DecodeError::Corrupt(format!(
618                        "dav1d decoded no picture ({res})"
619                    )));
620                }
621                if self.held {
622                    unsafe { dav1d_picture_unref(&mut self.picture) };
623                }
624                self.picture = next;
625                self.held = true;
626                got = true;
627            }
628            if sent == 0 {
629                break;
630            }
631        }
632        unsafe { dav1d_data_unref(&mut input) };
633        if !got {
634            return Ok(None);
635        }
636        let p = &self.picture;
637        let chroma = match p.p.layout {
638            DAV1D_PIXEL_LAYOUT_I420 => Chroma::C420,
639            DAV1D_PIXEL_LAYOUT_I422 => Chroma::C422,
640            DAV1D_PIXEL_LAYOUT_I444 => Chroma::C444,
641            other => {
642                return Err(DecodeError::Fatal(format!(
643                    "unsupported dav1d layout {other}"
644                )));
645            }
646        };
647        if p.p.bpc != 8 {
648            return Err(DecodeError::Fatal(format!(
649                "unsupported dav1d bit depth {}",
650                p.p.bpc
651            )));
652        }
653        let tags = unsafe { p.seq_hdr.as_ref() }.map(|s| ColorTags {
654            matrix: s.mtrx as u8,
655            full_range: s.color_range != 0,
656        });
657        Ok(Some(Picture {
658            width: p.p.w as usize,
659            height: p.p.h as usize,
660            chroma,
661            planes: [
662                p.data[0] as *const u8,
663                p.data[1] as *const u8,
664                p.data[2] as *const u8,
665            ],
666            strides: [
667                p.stride[0] as usize,
668                p.stride[1] as usize,
669                p.stride[1] as usize,
670            ],
671            tags,
672        }))
673    }
674}
675
676/// libde265. Its picture is valid only until the next call into the library, so the planes are
677/// copied out as they come.
678struct De265Backend {
679    ctx: *mut codec_sys::de265::de265_decoder_context,
680    planes: Vec<u8>,
681}
682
683impl Drop for De265Backend {
684    fn drop(&mut self) {
685        unsafe { codec_sys::de265::de265_free_decoder(self.ctx) };
686    }
687}
688
689impl De265Backend {
690    fn new() -> Result<Self, String> {
691        use codec_sys::de265::*;
692        let ctx = unsafe { de265_new_decoder() };
693        if ctx.is_null() {
694            return Err("libde265 opened no decoder".into());
695        }
696        unsafe {
697            de265_set_parameter_bool(ctx, DE265_DECODER_PARAM_BOOL_SEI_CHECK_HASH, 0);
698            de265_start_worker_threads(ctx, 2);
699        }
700        Ok(Self {
701            ctx,
702            planes: Vec::new(),
703        })
704    }
705
706    fn decode(&mut self, data: &[u8]) -> Result<Option<Picture>, DecodeError> {
707        use codec_sys::de265::*;
708        let text = |err: de265_error| {
709            unsafe { CStr::from_ptr(de265_get_error_text(err)) }
710                .to_string_lossy()
711                .into_owned()
712        };
713        unsafe {
714            let err = de265_push_data(
715                self.ctx,
716                data.as_ptr() as *const c_void,
717                data.len() as c_int,
718                0,
719                ptr::null_mut(),
720            );
721            if err != DE265_OK {
722                return Err(DecodeError::Corrupt(text(err)));
723            }
724            de265_push_end_of_frame(self.ctx);
725            // A decoded picture sits in the output queue until it is taken, and the decoder
726            // reports more to do for as long as one does, so each round takes what is there.
727            let mut image: *const de265_image = ptr::null();
728            let mut more: c_int = 1;
729            while more != 0 && image.is_null() {
730                let err = de265_decode(self.ctx, &mut more);
731                if err != DE265_OK && err != DE265_ERROR_WAITING_FOR_INPUT_DATA {
732                    return Err(DecodeError::Corrupt(text(err)));
733                }
734                loop {
735                    let next = de265_get_next_picture(self.ctx);
736                    if next.is_null() {
737                        break;
738                    }
739                    image = next;
740                }
741                if err == DE265_ERROR_WAITING_FOR_INPUT_DATA {
742                    break;
743                }
744            }
745            if image.is_null() {
746                return Ok(None);
747            }
748            let (w, h) = (
749                de265_get_image_width(image, 0) as usize,
750                de265_get_image_height(image, 0) as usize,
751            );
752            let format = de265_get_chroma_format(image);
753            let chroma = if format == de265_chroma_420 {
754                Chroma::C420
755            } else if format == de265_chroma_422 {
756                Chroma::C422
757            } else if format == de265_chroma_444 {
758                Chroma::C444
759            } else {
760                return Err(DecodeError::Fatal(format!(
761                    "unsupported libde265 chroma format {format}"
762                )));
763            };
764            if (0..3).any(|c| de265_get_bits_per_pixel(image, c) != 8) {
765                return Err(DecodeError::Fatal("unsupported libde265 bit depth".into()));
766            }
767            let tags = Some(ColorTags {
768                matrix: de265_get_image_matrix_coefficients(image) as u8,
769                full_range: de265_get_image_full_range_flag(image) != 0,
770            });
771            // The image lives only until the next libde265 call, so its planes are copied
772            // into this backend's own buffer, laid out as the picture was.
773            let (cw, ch) = match chroma {
774                Chroma::C420 => (w.div_ceil(2), h.div_ceil(2)),
775                Chroma::C422 => (w.div_ceil(2), h),
776                Chroma::C444 => (w, h),
777            };
778            self.planes.resize(w * h + 2 * cw * ch, 0);
779            let (yd, rest) = self.planes.split_at_mut(w * h);
780            let (ud, vd) = rest.split_at_mut(cw * ch);
781            for (c, dst, cols, rows) in [(0, yd, w, h), (1, ud, cw, ch), (2, vd, cw, ch)] {
782                let mut stride: c_int = 0;
783                let src = de265_get_image_plane(image, c, &mut stride);
784                for row in 0..rows {
785                    ptr::copy_nonoverlapping(
786                        src.add(row * stride as usize),
787                        dst.as_mut_ptr().add(row * cols),
788                        cols,
789                    );
790                }
791            }
792            Ok(Some(Picture {
793                width: w,
794                height: h,
795                chroma,
796                planes: [
797                    self.planes.as_ptr(),
798                    self.planes[w * h..].as_ptr(),
799                    self.planes[w * h + cw * ch..].as_ptr(),
800                ],
801                strides: [w, cw, cw],
802                tags,
803            }))
804        }
805    }
806}
807
808/// TurboJPEG decoder: 4:2:0 JPEGs decode straight into I420 planes; other subsamplings take the
809/// RGB route and are converted.
810pub struct JpegDecoder {
811    dec: Decompressor,
812    out: I420Buffer,
813    rgb: Vec<u8>,
814    have_frame: bool,
815}
816
817impl JpegDecoder {
818    pub fn new() -> Result<Self, String> {
819        Ok(JpegDecoder {
820            dec: Decompressor::new().map_err(|e| format!("turbojpeg: {}", e))?,
821            out: I420Buffer::new(2, 2),
822            rgb: Vec::new(),
823            have_frame: false,
824        })
825    }
826}
827
828impl Decoder for JpegDecoder {
829    fn codec(&self) -> Codec {
830        Codec::Jpeg
831    }
832
833    fn decode(&mut self, data: &[u8]) -> Result<bool, DecodeError> {
834        self.have_frame = false;
835        let hdr = self
836            .dec
837            .read_header(data)
838            .map_err(|e| DecodeError::Corrupt(format!("jpeg header: {}", e)))?;
839        if hdr.width == 0 || hdr.height == 0 {
840            return Err(DecodeError::Corrupt("empty jpeg".into()));
841        }
842        self.out.resize(hdr.width, hdr.height);
843        if hdr.subsamp == Subsamp::Sub2x2 {
844            let img = YuvImage {
845                pixels: &mut self.out.data[..],
846                width: hdr.width,
847                align: 1,
848                height: hdr.height,
849                subsamp: Subsamp::Sub2x2,
850            };
851            self.dec
852                .decompress_to_yuv(data, img)
853                .map_err(|e| DecodeError::Corrupt(format!("jpeg: {}", e)))?;
854        } else {
855            self.rgb.resize(hdr.width * hdr.height * 4, 0);
856            let img = Image {
857                pixels: &mut self.rgb[..],
858                width: hdr.width,
859                pitch: hdr.width * 4,
860                height: hdr.height,
861                format: PixelFormat::RGBA,
862            };
863            self.dec
864                .decompress(data, img)
865                .map_err(|e| DecodeError::Corrupt(format!("jpeg: {}", e)))?;
866            let y_len = self.out.y_len();
867            let uv_len = self.out.uv_len();
868            let cw = hdr.width.div_ceil(2);
869            let (yp, rest) = self.out.data.split_at_mut(y_len);
870            let (up, vp) = rest.split_at_mut(uv_len);
871            let mut planar = yuv::YuvPlanarImageMut {
872                y_plane: yuv::BufferStoreMut::Borrowed(yp),
873                y_stride: hdr.width as u32,
874                u_plane: yuv::BufferStoreMut::Borrowed(up),
875                u_stride: cw as u32,
876                v_plane: yuv::BufferStoreMut::Borrowed(vp),
877                v_stride: cw as u32,
878                width: hdr.width as u32,
879                height: hdr.height as u32,
880            };
881            yuv::rgba_to_yuv420(
882                &mut planar,
883                &self.rgb,
884                (hdr.width * 4) as u32,
885                yuv::YuvRange::Full,
886                yuv::YuvStandardMatrix::Bt601,
887                yuv::YuvConversionMode::Fast,
888            )
889            .map_err(|e| DecodeError::Corrupt(format!("rgb->i420: {:?}", e)))?;
890        }
891        self.have_frame = true;
892        Ok(true)
893    }
894
895    fn frame(&self) -> Option<I420View<'_>> {
896        if self.have_frame {
897            Some(self.out.view(true))
898        } else {
899            None
900        }
901    }
902}
903
904#[cfg(test)]
905mod tests {
906    use super::*;
907
908    #[test]
909    fn codec_ids_round_trip() {
910        for id in 0..6 {
911            let c = Codec::from_id(id).unwrap();
912            assert_eq!(c as u32, id);
913        }
914        assert!(Codec::from_id(6).is_none());
915    }
916
917    #[test]
918    fn h264_keyframe_sniff() {
919        let idr = [
920            0, 0, 0, 1, 0x67, 0x42, 0, 0, 0, 1, 0x68, 0xCE, 0, 0, 1, 0x65, 0x88,
921        ];
922        assert_eq!(sniff_keyframe(Codec::H264, &idr), Some(true));
923        let p = [0, 0, 0, 1, 0x41, 0x9A];
924        assert_eq!(sniff_keyframe(Codec::H264, &p), Some(false));
925        assert_eq!(sniff_keyframe(Codec::Vp8, &[0x10, 0, 0]), Some(true));
926        assert_eq!(sniff_keyframe(Codec::Vp8, &[0x11, 0, 0]), Some(false));
927        assert_eq!(sniff_keyframe(Codec::Vp9, &[0x82, 0x49, 0x83]), Some(true));
928        assert_eq!(sniff_keyframe(Codec::Vp9, &[0x86, 0]), Some(false));
929        assert_eq!(
930            sniff_keyframe(Codec::H265, &[0, 0, 1, 0x26, 0x01]),
931            Some(true)
932        );
933        assert_eq!(
934            sniff_keyframe(Codec::H265, &[0, 0, 1, 0x02, 0x01]),
935            Some(false)
936        );
937        assert_eq!(
938            sniff_keyframe(Codec::Av1, &[0x12, 0, 0x32, 0x01, 0x10]),
939            Some(true)
940        );
941        assert_eq!(sniff_keyframe(Codec::Av1, &[0x32, 0x01, 0x30]), Some(false));
942    }
943
944    #[test]
945    fn jpeg_round_trip() {
946        let w = 64;
947        let h = 48;
948        let mut img = I420Buffer::new(w, h);
949        let yl = img.y_len();
950        let ul = img.uv_len();
951        img.data[..yl].fill(200);
952        img.data[yl..yl + ul].fill(100);
953        img.data[yl + ul..].fill(150);
954        let src = YuvImage {
955            pixels: &img.data[..],
956            width: w,
957            align: 1,
958            height: h,
959            subsamp: Subsamp::Sub2x2,
960        };
961        let jpeg = turbojpeg::compress_yuv(src, 90).unwrap();
962        let mut dec = JpegDecoder::new().unwrap();
963        assert!(dec.decode(&jpeg).unwrap());
964        let v = dec.frame().unwrap();
965        assert_eq!((v.width, v.height), (w, h));
966        assert!(v.full_range);
967        let mid = v.y[(h / 2) * v.y_stride + w / 2];
968        assert!((mid as i32 - 200).abs() <= 3, "luma {}", mid);
969        let cu = v.u[(h / 4) * v.uv_stride + w / 4];
970        assert!((cu as i32 - 100).abs() <= 3, "cb {}", cu);
971    }
972
973    /// Every inter-coded codec has a decoder, and none holds a picture before decoding one.
974    #[test]
975    fn a_decoder_constructs_for_every_video_codec() {
976        for c in Codec::VIDEO {
977            let d = VideoDecoder::new(c).unwrap();
978            assert_eq!(d.codec(), c);
979            assert!(d.frame().is_none());
980            assert_eq!(d.color_tags(), None);
981        }
982        assert!(VideoDecoder::new(Codec::Jpeg).is_err());
983    }
984
985    #[test]
986    fn h264_round_trip_through_openh264() {
987        use openh264::encoder::Encoder;
988        use openh264::formats::YUVBuffer;
989        let (w, h) = (64usize, 48usize);
990        let mut yuv = vec![0u8; w * h * 3 / 2];
991        yuv[..w * h].fill(145);
992        yuv[w * h..w * h + w * h / 4].fill(54);
993        yuv[w * h + w * h / 4..].fill(34);
994        let source = YUVBuffer::from_vec(yuv, w, h);
995        let mut enc = Encoder::new().unwrap();
996        let mut dec = VideoDecoder::new(Codec::H264).unwrap();
997        let mut decoded = 0;
998        for _ in 0..6 {
999            let bitstream = enc.encode(&source).unwrap().to_vec();
1000            assert!(!bitstream.is_empty());
1001            if dec.decode(&bitstream).unwrap() {
1002                decoded += 1;
1003                let v = dec.frame().unwrap();
1004                assert_eq!((v.width, v.height), (w, h));
1005                assert!(!v.full_range);
1006                let y = v.y[(h / 2) * v.y_stride + w / 2];
1007                let u = v.u[(h / 4) * v.uv_stride + w / 4];
1008                let vv = v.v[(h / 4) * v.uv_stride + w / 4];
1009                assert!(
1010                    (y as i32 - 145).abs() <= 6
1011                        && (u as i32 - 54).abs() <= 6
1012                        && (vv as i32 - 34).abs() <= 6,
1013                    "yuv {} {} {}",
1014                    y,
1015                    u,
1016                    vv
1017                );
1018            }
1019        }
1020        assert!(decoded >= 5, "decoded {} of 6 frames", decoded);
1021    }
1022
1023    /// A Constrained Baseline H.264 stream of `frames` access units at `mbs` macroblocks square,
1024    /// its fields in the order of the specification's syntax tables: an SPS with picture order
1025    /// count type 0 and two reference frames, a CAVLC PPS, an IDR of I_PCM macroblocks holding
1026    /// `sample`, then P pictures of skipped macroblocks.
1027    fn h264_skipping_stream(mbs: u32, frames: u32, sample: u8) -> Vec<Vec<u8>> {
1028        struct Bits(Vec<u8>, usize);
1029        impl Bits {
1030            fn u(&mut self, value: u32, count: usize) {
1031                for shift in (0..count).rev() {
1032                    if self.1.is_multiple_of(8) {
1033                        self.0.push(0);
1034                    }
1035                    if (value >> shift) & 1 != 0 {
1036                        *self.0.last_mut().unwrap() |= 0x80 >> (self.1 % 8);
1037                    }
1038                    self.1 += 1;
1039                }
1040            }
1041            fn ue(&mut self, value: u32) {
1042                let coded = value + 1;
1043                let length = 32 - coded.leading_zeros() as usize;
1044                self.u(0, length - 1);
1045                self.u(coded, length);
1046            }
1047            fn align(&mut self) {
1048                while !self.1.is_multiple_of(8) {
1049                    self.u(0, 1);
1050                }
1051            }
1052            fn nal(mut self, header: u8) -> Vec<u8> {
1053                self.u(1, 1);
1054                self.align();
1055                let mut out = vec![0, 0, 0, 1, header];
1056                let mut zeros = 0;
1057                for byte in self.0 {
1058                    if zeros >= 2 && byte <= 3 {
1059                        out.push(3);
1060                        zeros = 0;
1061                    }
1062                    zeros = if byte == 0 { zeros + 1 } else { 0 };
1063                    out.push(byte);
1064                }
1065                out
1066            }
1067        }
1068        let mut sps = Bits(Vec::new(), 0);
1069        sps.u(66, 8);
1070        sps.u(0xc0, 8);
1071        sps.u(40, 8);
1072        for value in [0, 0, 0, 0, 2] {
1073            sps.ue(value);
1074        }
1075        sps.u(0, 1);
1076        sps.ue(mbs - 1);
1077        sps.ue(mbs - 1);
1078        sps.u(0b1100, 4);
1079        let mut pps = Bits(Vec::new(), 0);
1080        pps.ue(0);
1081        pps.ue(0);
1082        pps.u(0, 2);
1083        for value in [0, 0, 0] {
1084            pps.ue(value);
1085        }
1086        pps.u(0, 3);
1087        for value in [0, 0, 0] {
1088            pps.ue(value);
1089        }
1090        pps.u(0, 3);
1091        let mut idr = Bits(Vec::new(), 0);
1092        for value in [0, 7, 0] {
1093            idr.ue(value);
1094        }
1095        idr.u(0, 4);
1096        idr.ue(0);
1097        idr.u(0, 4);
1098        idr.u(0, 2);
1099        idr.ue(0);
1100        for _ in 0..mbs * mbs {
1101            idr.ue(25);
1102            idr.align();
1103            for _ in 0..384 {
1104                idr.u(sample as u32, 8);
1105            }
1106        }
1107        let mut units = vec![[sps.nal(0x67), pps.nal(0x68), idr.nal(0x65)].concat()];
1108        for n in 1..frames {
1109            let mut p = Bits(Vec::new(), 0);
1110            for value in [0, 5, 0] {
1111                p.ue(value);
1112            }
1113            p.u(n % 16, 4);
1114            p.u(2 * n % 16, 4);
1115            p.u(0, 3);
1116            p.ue(0);
1117            p.ue(mbs * mbs);
1118            units.push(p.nal(0x41));
1119        }
1120        units
1121    }
1122
1123    /// A prefix NAL unit naming dependency layer 1, which only a scalable stream carries, leaves
1124    /// every picture after it decoding, a key frame's included.
1125    #[test]
1126    fn h264_scalable_extension_units_never_retarget_the_decoder() {
1127        let mut units = h264_skipping_stream(2, 12, 0x60);
1128        units[4].splice(0..0, [0, 0, 0, 1, 0x4e, 0x80, 0x90, 0x07, 0x20]);
1129        let mut dec = VideoDecoder::new(Codec::H264).unwrap();
1130        for (n, unit) in units.iter().chain(&units).enumerate() {
1131            match dec.decode(unit) {
1132                Ok(true) => {}
1133                other => panic!("access unit {n}: {other:?}"),
1134            }
1135        }
1136    }
1137
1138    /// Garbage is a corrupt packet, never a decoder to throw away.
1139    #[test]
1140    fn a_decoder_rejects_garbage_without_fatal() {
1141        for c in Codec::VIDEO {
1142            let mut d = VideoDecoder::new(c).unwrap();
1143            match d.decode(&[1, 2, 3, 4, 5, 6, 7, 8]) {
1144                Ok(false) | Err(DecodeError::Corrupt(_)) => {}
1145                Ok(true) => panic!("{c:?}: garbage decoded to a frame"),
1146                Err(DecodeError::Fatal(e)) => panic!("{c:?}: fatal: {}", e),
1147            }
1148        }
1149    }
1150
1151    /// The VP8 color bits are the first two bool-coded fields of a key frame: a header whose
1152    /// first partition begins with two zero bits reads as BT.601 at limited range.
1153    /// A VP9 profile 2 key frame, 10-bit 4:2:0 at 320x240, is refused rather than read as 8-bit:
1154    /// a libvpx built with high bit depth hands it back in 16-bit samples, and one built without
1155    /// cannot decode it.
1156    #[test]
1157    fn a_10_bit_vp9_frame_never_decodes_as_8_bit() {
1158        const KEY: [u8; 77] = [
1159            0x92, 0x49, 0x83, 0x42, 0x00, 0x09, 0xf8, 0x07, 0x7b, 0x03, 0x1c, 0x12, 0x0e, 0x0c,
1160            0x2c, 0x00, 0x00, 0x48, 0x47, 0xa8, 0x3d, 0x8f, 0xc8, 0xc0, 0x0c, 0x30, 0x00, 0x6b,
1161            0x41, 0x13, 0x20, 0x95, 0xe2, 0x6a, 0x41, 0xdc, 0xb8, 0x00, 0x11, 0x50, 0x1b, 0x55,
1162            0x82, 0x76, 0x20, 0x4f, 0xd6, 0xa9, 0x82, 0x10, 0xc5, 0xe9, 0x8f, 0xe0, 0xa3, 0xf7,
1163            0xcd, 0x66, 0x00, 0xbc, 0xd3, 0xeb, 0x11, 0x3a, 0xb5, 0xed, 0x25, 0x7b, 0x25, 0x6d,
1164            0xa9, 0x8e, 0xf7, 0xf7, 0xfa, 0x2b, 0x10,
1165        ];
1166        let mut d = VideoDecoder::new(Codec::Vp9).unwrap();
1167        assert!(
1168            !matches!(d.decode(&KEY), Ok(true)),
1169            "a 10-bit picture was handed on as an 8-bit one"
1170        );
1171    }
1172
1173    #[test]
1174    fn vp8_color_bits_read_from_the_first_partition() {
1175        let mut key = vec![0x10, 0, 0, 0x9d, 0x01, 0x2a, 64, 0, 48, 0];
1176        key.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]);
1177        assert_eq!(vp8_color_bits(&key), Some((false, false)));
1178        key[10] = 0xff;
1179        key[11] = 0xff;
1180        assert_eq!(vp8_color_bits(&key), Some((true, true)));
1181        assert_eq!(
1182            vp8_color_bits(&[0x11, 0, 0]),
1183            None,
1184            "an inter frame carries no header"
1185        );
1186    }
1187}
pixelflux/webcam/decode.rs

pixelflux/webcam/
decode.rs
