vcs-controller/src/gstreamer_pipeline.rs

use gstreamer::{prelude::*, PadLinkError};
use gstreamer::{Element, ElementFactory, Pipeline};
use gstreamer_app::AppSink;
use gtk::glib::BoolError;
use snafu::prelude::*;
use std::str::FromStr;
use std::sync::{Arc, Mutex};

pub const JPEG_QUALITY: i32 = 40;

#[derive(Debug)]
pub struct WebcamPipeline {
    pub pipeline: Pipeline,

    pub sink_paintable: Element,

    pub sink_frame: Arc<Mutex<AppSink>>,
}

impl WebcamPipeline {
    pub fn new() -> Result<WebcamPipeline, PipelineError> {
        let pipeline = Pipeline::with_name("webcam_pipeline");

        // All of the following errors are unrecoverable

        let source = ElementFactory::make("mfvideosrc")
            .build()
            .context(BuildSnafu {
                element: "mfvideosrc",
            })?;
        let convert = ElementFactory::make("videoconvert")
            .build()
            .context(BuildSnafu {
                element: "videoconvert",
            })?;
        let rate = ElementFactory::make("videorate")
            .build()
            .context(BuildSnafu {
                element: "videorate",
            })?;

        let tee = ElementFactory::make("tee")
            .build()
            .context(BuildSnafu { element: "tee" })?;

        let queue_app = ElementFactory::make("queue")
            .property("max-size-time", 1u64)
            .property("max-size-buffers", 0u32)
            .property("max-size-bytes", 0u32)
            .build()
            .context(BuildSnafu {
                element: "paintable queue",
            })?;
        let sink_paintable = ElementFactory::make("gtk4paintablesink")
            .name("gtk4_output")
            .build()
            .context(BuildSnafu {
                element: "gtkpaintablesink",
            })?;

        // queue.connect_closure("overrun", false, glib::closure!(|queue: Element| {
        //     println!("The queue is full!");
        // }));

        let appsink_queue = ElementFactory::make("queue")
            .property("max-size-time", 1u64)
            .property("max-size-buffers", 0u32)
            .property("max-size-bytes", 0u32)
            .build()
            .context(BuildSnafu {
                element: "appsink queue",
            })?;

        let resize = ElementFactory::make("videoscale")
            .build()
            .context(BuildSnafu {
                element: "videoscale",
            })?;

        let jpeg_enc = ElementFactory::make("jpegenc")
            .property("quality", JPEG_QUALITY)
            .build()
            .context(BuildSnafu { element: "jpegenc" })?;

        let caps_string = "image/jpeg,width=640,height=640";
        let appsrc_caps = gstreamer::Caps::from_str(caps_string).context(BuildSnafu {
            element: "appsink caps",
        })?;

        let sink_frame = AppSink::builder()
            .name("frame_appsink")
            .sync(false)
            .max_buffers(3u32)
            .drop(true)
            .caps(&appsrc_caps)
            .build();

        sink_frame.set_property("caps", &appsrc_caps.to_value());

        pipeline
            .add_many(&[
                &source,
                &convert,
                &rate,
                &tee,
                &queue_app,
                &sink_paintable,
                &appsink_queue,
                &resize,
                &jpeg_enc,
                &sink_frame.upcast_ref(),
            ])
            .context(LinkSnafu {
                from: "all",
                to: "pipeline",
            })?;

        Element::link_many(&[&source, &convert, &rate]).context(LinkSnafu {
            from: "source et. al.",
            to: "rate",
        })?;

        // -- BEGIN PAINTABLE SINK PIPELINE
        let tee_caps =
            // gstreamer::caps::Caps::from_str("video/x-raw,framerate=15/1").context(BuildSnafu {
            gstreamer::caps::Caps::from_str("video/x-raw").context(BuildSnafu {
                element: "tee caps",
            })?;

        rate.link_filtered(&tee, &tee_caps).context(LinkSnafu {
            from: "videorate",
            to: "tee",
        })?;

        let tee_src_1 = tee
            .request_pad_simple("src_%u")
            .ok_or(PipelineError::PadRequestError {
                element: "tee pad 1".to_string(),
            })?;
        let paintable_queue_sinkpad =
            queue_app
                .static_pad("sink")
                .ok_or(PipelineError::PadRequestError {
                    element: "gtk4 sink".to_string(),
                })?;

        tee_src_1
            .link(&paintable_queue_sinkpad)
            .context(PadLinkSnafu {
                from: "tee src pad",
                to: "gtk4 paintable queue",
            })?;

        queue_app.link(&sink_paintable).context(LinkSnafu {
            from: "gtk4 paintable queue",
            to: "gtk4 paintable",
        })?;

        // -- END PAINTABLE SINK PIPELINE

        // -- BEGIN APPSINK PIPELINE
        let tee_src_2 = tee
            .request_pad_simple("src_%u")
            .ok_or(PipelineError::PadRequestError {
                element: "tee pad 2".to_string(),
            })?;
        let appsink_queue_sinkpad =
            appsink_queue
                .static_pad("sink")
                .ok_or(PipelineError::PadRequestError {
                    element: "appsink queue".to_string(),
                })?;
        tee_src_2
            .link(&appsink_queue_sinkpad)
            .context(PadLinkSnafu {
                from: "tee src pad 2",
                to: "appsink queue sinkpad",
            })?;

        appsink_queue.link(&resize).context(LinkSnafu {
            from: "appsink_queue",
            to: "resize",
        })?;

        let resize_caps =
            gstreamer::caps::Caps::from_str("video/x-raw,format=RGB,width=640,height=640")
                .context(BuildSnafu {
                    element: "resize_caps",
                })?;

        resize
            .link_filtered(&jpeg_enc, &resize_caps)
            .context(LinkSnafu {
                from: "jpeg_enc",
                to: "resize_caps",
            })?;

        Element::link_many(&[&jpeg_enc, &sink_frame.upcast_ref()]).context(LinkSnafu {
            from: "jpeg_enc",
            to: "appsink",
        })?;

        Ok(WebcamPipeline {
            pipeline,
            sink_paintable,
            sink_frame: Arc::new(Mutex::new(sink_frame)),
        })
    }
}

#[derive(Debug, Snafu)]
pub enum PipelineError {
    #[snafu(display("Error during element linking"))]
    LinkError {
        source: BoolError,
        from: String,
        to: String,
    },
    #[snafu(display("Error linking pads"))]
    PadLinkError {
        source: PadLinkError,
        from: String,
        to: String,
    },
    #[snafu(display("Error creating element"))]
    BuildError { source: BoolError, element: String },
    #[snafu(display("Error getting pad from element"))]
    PadRequestError { element: String },
}