vcs-controller/src/gstreamer_pipeline.rs

use gstreamer::{prelude::*, PadLinkError};
use gstreamer::{Element, ElementFactory, Pipeline};
use gstreamer_app::AppSink;
use gtk::glib::BoolError;
use snafu::prelude::*;
use std::str::FromStr;
use std::sync::{Arc, Mutex};

#[derive(Debug)]
pub struct WebcamPipeline {
    pub pipeline: Pipeline,

    pub src: Element,
    pub converter: Element,
    pub tee: Element,
    pub queue_app: Element,
    pub sink_paintable: Element,

    pub queue: Element,
    pub resize: Element,
    pub sink_frame: Arc<Mutex<AppSink>>,
}

impl WebcamPipeline {
    pub fn new() -> Result<WebcamPipeline, PipelineError> {
        let pipeline = Pipeline::with_name("webcam_pipeline");

        // All of the following errors are unrecoverable

        let source = ElementFactory::make("mfvideosrc")
            .build()
            .context(BuildSnafu {
                element: "mfvideosrc",
            })?;
        let convert = ElementFactory::make("videoconvert")
            .build()
            .context(BuildSnafu {
                element: "videoconvert",
            })?;
        let rate = ElementFactory::make("videorate")
            .build()
            .context(BuildSnafu {
                element: "videorate",
            })?;

        let tee = ElementFactory::make("tee")
            .build()
            .context(BuildSnafu { element: "tee" })?;

        let queue_app = ElementFactory::make("queue").build().context(BuildSnafu {
            element: "paintable queue",
        })?;
        let sink_paintable = ElementFactory::make("gtk4paintablesink")
            .name("gtk4_output")
            .build()
            .context(BuildSnafu {
                element: "gtkpaintablesink",
            })?;

        let queue = ElementFactory::make("queue").build().context(BuildSnafu {
            element: "appsink queue",
        })?;

        let resize = ElementFactory::make("videoscale")
            .build()
            .context(BuildSnafu {
                element: "videoscale",
            })?;

        let caps_string = "video/x-raw,format=RGB,width=640,height=480,max-buffers=1,drop=true";
        // let caps_string = String::from("video/x-raw,format=RGB,max-buffers=1,drop=true");
        let appsrc_caps = gstreamer::Caps::from_str(caps_string).context(BuildSnafu {
            element: "appsink caps",
        })?;

        let sink_frame = AppSink::builder()
            .name("frame_output")
            .sync(false)
            .max_buffers(1u32)
            .drop(true)
            .caps(&appsrc_caps)
            .build();

        sink_frame.set_property("caps", &appsrc_caps.to_value());

        pipeline
            .add_many(&[
                &source,
                &convert,
                &rate,
                &tee,
                &queue_app,
                &sink_paintable,
                &resize,
                &queue,
                &sink_frame.upcast_ref(),
            ])
            .context(LinkSnafu {
                from: "all",
                to: "pipeline",
            })?;

        source.link(&convert).context(LinkSnafu {
            from: "mfvideosrc",
            to: "videoconvert",
        })?;

        convert.link(&rate).context(LinkSnafu {
            from: "videoconvert",
            to: "videorate",
        })?;

        let tee_caps =
            gstreamer::caps::Caps::from_str("video/x-raw,framerate=15/1").context(BuildSnafu {
                element: "tee caps",
            })?;

        rate.link_filtered(&tee, &tee_caps).context(LinkSnafu {
            from: "videorate",
            to: "tee",
        })?;

        let tee_src_1 = tee
            .request_pad_simple("src_%u")
            .ok_or(PipelineError::PadRequestError {
                element: "tee pad 1".to_string(),
            })?;
        let paintable_queue_sinkpad =
            queue_app
                .static_pad("sink")
                .ok_or(PipelineError::PadRequestError {
                    element: "gtk4 sink".to_string(),
                })?;
        tee_src_1
            .link(&paintable_queue_sinkpad)
            .context(PadLinkSnafu {
                from: "tee src pad",
                to: "gtk4 paintable queue",
            })?;
        queue_app.link(&sink_paintable).context(LinkSnafu {
            from: "gtk4 paintable queue",
            to: "gtk4 paintable",
        })?;

        let tee_src_2 = tee
            .request_pad_simple("src_%u")
            .ok_or(PipelineError::PadRequestError {
                element: "tee pad 2".to_string(),
            })?;
        let appsink_queue_sinkpad =
            queue
                .static_pad("sink")
                .ok_or(PipelineError::PadRequestError {
                    element: "appsink queue".to_string(),
                })?;
        tee_src_2
            .link(&appsink_queue_sinkpad)
            .context(PadLinkSnafu {
                from: "tee src pad 2",
                to: "appsink queue sinkpad",
            })?;

        queue.link(&resize).context(LinkSnafu {
            from: "appsink queue",
            to: "videoscale",
        })?;
        resize.link(&sink_frame).context(LinkSnafu {
            from: "videoscale",
            to: "appsink",
        })?;

        Ok(WebcamPipeline {
            pipeline,
            src: source,
            converter: convert,
            tee,
            queue_app,
            sink_paintable,
            resize,
            queue,
            sink_frame: Arc::new(Mutex::new(sink_frame)),
        })
    }
}

#[derive(Debug, Snafu)]
pub enum PipelineError {
    #[snafu(display("Error during element linking"))]
    LinkError {
        source: BoolError,
        from: String,
        to: String,
    },
    #[snafu(display("Error linking pads"))]
    PadLinkError {
        source: PadLinkError,
        from: String,
        to: String,
    },
    #[snafu(display("Error creating element"))]
    BuildError { source: BoolError, element: String },
    #[snafu(display("Error getting pad from element"))]
    PadRequestError { element: String },
}