From 0cea5e5c117af2e5e626ade33272fcdeef297aa5 Mon Sep 17 00:00:00 2001 From: Nickiel12 <35903114+Nickiel12@users.noreply.github.com> Date: Sat, 20 Apr 2024 20:19:01 -0700 Subject: [PATCH] added webcam capture and box drawing --- Cargo.lock | 312 ++++++++++++++++++++++++- Cargo.toml | 2 + src/coordinator.rs | 15 +- src/main.rs | 4 + src/remote_sources/automated_source.rs | 55 +++-- src/remote_sources/mod.rs | 16 +- src/ui/mod.rs | 140 +++++++++-- 7 files changed, 500 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d34de53..f344f8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,21 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.81" @@ -47,6 +62,12 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + [[package]] name = "autocfg" version = "1.1.0" @@ -178,6 +199,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-targets 0.52.4", +] + [[package]] name = "concurrent-queue" version = "2.4.0" @@ -317,6 +350,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "either" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" + [[package]] name = "equivalent" version = "1.0.1" @@ -517,6 +556,31 @@ dependencies = [ "system-deps", ] +[[package]] +name = "gdk4-win32" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6181b6e5c91ee292dca0032b00d48dee8e61358253742c9752537a88486b3f" +dependencies = [ + "gdk4", + "gdk4-win32-sys", + "gio", + "glib", + "libc", +] + +[[package]] +name = "gdk4-win32-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa8530d6619cf43f007f3efd993a356e1ca4e643c4d0bd2a99832a08af2e402" +dependencies = [ + "gdk4-sys", + "glib-sys", + "libc", + "system-deps", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -719,6 +783,159 @@ dependencies = [ "system-deps", ] +[[package]] +name = "gst-plugin-gtk4" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83c12132d3619f85db54f6034b4e3872577886657b07e055c521baeb2d5f6733" +dependencies = [ + "async-channel", + "gdk4-win32", + "gst-plugin-version-helper", + "gstreamer", + "gstreamer-base", + "gstreamer-gl", + "gstreamer-video", + "gtk4", + "once_cell", + "windows-sys 0.52.0", +] + +[[package]] +name = "gst-plugin-version-helper" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5e874f1660252fd2ec81c602066df3633b3a6fcbe2b196f7f93c27cf069b2a" +dependencies = [ + "chrono", + "toml_edit 0.22.9", +] + +[[package]] +name = "gstreamer" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6f2d7dcde4b42b5297d25e9f51914cacfa148c99ba6ddabebf006fb2b18c20" +dependencies = [ + "cfg-if", + "futures-channel", + "futures-core", + "futures-util", + "glib", + "gstreamer-sys", + "itertools", + "libc", + "muldiv", + "num-integer", + "num-rational", + "once_cell", + "option-operations", + "paste", + "pin-project-lite", + "smallvec", + "thiserror", +] + +[[package]] +name = "gstreamer-base" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514c71195b53c7eced4842b66ca9149833e41cf6a1d949e45e2ca4a4fa929850" +dependencies = [ + "atomic_refcell", + "cfg-if", + "glib", + "gstreamer", + "gstreamer-base-sys", + "libc", +] + +[[package]] +name = "gstreamer-base-sys" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "286591e0f85bbda1adf9bab6f21d015acd9ca0a4d4acb61da65e3d0487e23c4e" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-gl" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d21c0c5fbf74018a0254b3ab77bca0a5b2c0f002bcfd910c09113ae90a95d98" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-base", + "gstreamer-gl-sys", + "gstreamer-video", + "libc", + "once_cell", +] + +[[package]] +name = "gstreamer-gl-sys" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61d1e3b9b02abc23835e9d770f2bd705b67a50406ea37e963b4526a77c6a7cd8" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "gstreamer-video-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sys" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5ddf526b3bf90ea627224c804f00b8bcb0452e3b447978b4d5092f8e8ff5918" +dependencies = [ + "glib-sys", + "gobject-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-video" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc0a96481ecfcdd120d5057bb7ab5a6f6cd392cc34ba1e8b86cac3ba082f788" +dependencies = [ + "cfg-if", + "futures-channel", + "glib", + "gstreamer", + "gstreamer-base", + "gstreamer-video-sys", + "libc", + "once_cell", + "thiserror", +] + +[[package]] +name = "gstreamer-video-sys" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ea7996ba44fbbf563aeeda96e24259efc9f06b407854d837ee58e260d7ba78" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + [[package]] name = "gtk4" version = "0.8.1" @@ -814,6 +1031,29 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core 0.52.0", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.5.0" @@ -873,6 +1113,15 @@ dependencies = [ "libc", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -889,6 +1138,8 @@ dependencies = [ "futures-core", "futures-util", "gilrs", + "gst-plugin-gtk4", + "gstreamer", "gtk4", "log", "serde", @@ -1002,6 +1253,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "muldiv" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "956787520e75e9bd233246045d19f42fb73242759cc57fba9611d940ae96d4b0" + [[package]] name = "nix" version = "0.28.0" @@ -1030,6 +1287,35 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -1064,6 +1350,15 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "option-operations" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c26d27bb1aeab65138e4bf7666045169d1717febcc9ff870166be8348b223d0" +dependencies = [ + "paste", +] + [[package]] name = "ordered-multimap" version = "0.6.0" @@ -1104,6 +1399,12 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + [[package]] name = "pathdiff" version = "0.2.1" @@ -1860,7 +2161,16 @@ version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" dependencies = [ - "windows-core", + "windows-core 0.54.0", + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ "windows-targets 0.52.4", ] diff --git a/Cargo.toml b/Cargo.toml index 3fa2125..725d490 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,8 @@ err-derive = "0.3.1" futures-core = "0.3.30" futures-util = { version = "0.3.30", features = ["tokio-io"] } gilrs = "0.10.6" +gstreamer = { version = "0.22.4", features = ["v1_22"] } +gst-plugin-gtk4 = { version = "0.12.2", features = ["gtk_v4_12"] } gtk = { version = "0.8.1", package = "gtk4", features = ["v4_12"] } log = "0.4.21" serde = { version = "1.0.197", features = ["derive"] } diff --git a/src/coordinator.rs b/src/coordinator.rs index 3d20ef1..6756f97 100644 --- a/src/coordinator.rs +++ b/src/coordinator.rs @@ -1,4 +1,5 @@ use std::pin::{pin, Pin}; +use std::sync::Mutex; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -16,9 +17,9 @@ use tokio::runtime::Handle; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; use crate::remote_sources; +use crate::ui::BoxCoords; use crate::{joystick_source::joystick_loop, ui::GuiUpdate}; - const PRIORITY_TIMEOUT: Duration = Duration::from_secs(2); #[derive(Clone)] @@ -43,6 +44,7 @@ pub enum ApplicationEvent { struct CoordState<'a> { pub sck_outbound: Option>, Message>>, pub sck_alive_server: Arc, + pub identity_boxes: Arc>>, pub sck_alive_recvr: Arc, pub joystick_loop_alive: Arc, @@ -61,11 +63,13 @@ impl<'a> CoordState<'a> { to_mec: Sender, to_gui: Sender, rt: Handle, + identity_boxes: Arc>>, ) -> Self { CoordState { sck_outbound: None, sck_alive_recvr: Arc::new(AtomicBool::new(false)), sck_alive_server: Arc::new(AtomicBool::new(false)), + identity_boxes, joystick_loop_alive: Arc::new(AtomicBool::new(false)), current_priority: ConnectionType::Local, @@ -138,6 +142,7 @@ impl<'a> CoordState<'a> { self.rt.clone(), self.to_mec.clone(), self.sck_alive_server.clone(), + self.identity_boxes.clone(), )); } @@ -170,12 +175,13 @@ pub async fn start_coordinator( to_mec: Sender, to_gui: Sender, runtime: Handle, + identity_boxes: Arc>>, ) { info!("Starting coordinator!"); let mec = pin!(mec); - let mut state = CoordState::new(mec, to_mec, to_gui, runtime); + let mut state = CoordState::new(mec, to_mec, to_gui, runtime, identity_boxes); state.check_states().await; @@ -197,7 +203,9 @@ pub async fn start_coordinator( } ApplicationEvent::MoveEvent(coord, priority) => { // If Automatic control, but local event happens, override the automatice events for 2 seconds - if priority <= state.current_priority || Instant::now() > state.last_update_of_priority + PRIORITY_TIMEOUT { + if priority <= state.current_priority + || Instant::now() > state.last_update_of_priority + PRIORITY_TIMEOUT + { state.last_update_of_priority = Instant::now(); state.current_priority = priority; @@ -216,7 +224,6 @@ pub async fn start_coordinator( state.socket_send(Message::Text(message)).await; } - } } } diff --git a/src/main.rs b/src/main.rs index 8dcf1d1..7f5e060 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,10 @@ fn main() -> glib::ExitCode { error!("Failed to init the simplelogger!: {e}"); } + gstreamer::init().expect("Unable to start gstreamer"); + + gstgtk4::plugin_register_static().expect("Unable to register gtk4 plugin"); + let rt = runtime::Runtime::new().expect("Could not start tokio runtime"); let handle = rt.handle().clone(); diff --git a/src/remote_sources/automated_source.rs b/src/remote_sources/automated_source.rs index 039b141..c22ab6d 100644 --- a/src/remote_sources/automated_source.rs +++ b/src/remote_sources/automated_source.rs @@ -11,7 +11,10 @@ use tokio::net::TcpStream; use tokio_tungstenite::{tungstenite::Result, WebSocketStream}; use super::TrackerState; -use crate::coordinator::{ApplicationEvent, MoveEvent}; +use crate::{ + coordinator::{ApplicationEvent, MoveEvent}, + ui::BoxCoords, +}; const HALF_FRAME_WIDTH: f64 = 320.0; @@ -19,6 +22,7 @@ pub async fn handle_connection( mut ws_stream: WebSocketStream, mec: Sender, tracker_state: Arc>, + identity_boxes: Arc>>, ) -> Result<()> { if let Ok(mut ts) = tracker_state.lock() { ts.has_active_connection = true; @@ -26,14 +30,17 @@ pub async fn handle_connection( while let Some(msg) = ws_stream.next().await { let msg = msg?; if msg.is_text() { - let (x_off, y_off) = - match process_incoming_string(msg.to_string(), tracker_state.clone()) { - Ok(val) => val, - Err(e) => { - error!("{e}"); - (0, 0) - } - }; + let (x_off, y_off) = match process_incoming_string( + msg.to_string(), + tracker_state.clone(), + &identity_boxes, + ) { + Ok(val) => val, + Err(e) => { + error!("{e}"); + (0, 0) + } + }; if let Err(e) = mec .send(ApplicationEvent::MoveEvent( @@ -53,18 +60,12 @@ pub async fn handle_connection( Ok(()) } -struct BoxInfo { - id: u32, - x1: u32, - y1: u32, - x2: u32, -} - fn process_incoming_string( message: String, tracker_state: Arc>, + identity_boxes: &Arc>>, ) -> core::result::Result<(i32, i32), String> { - let mut boxes: Vec = Vec::new(); + let mut boxes: Vec = Vec::new(); for line in message.lines() { let parts: Vec<&str> = line.split(' ').collect(); @@ -91,27 +92,37 @@ fn process_incoming_string( } let x2: u32 = coords2[0].parse().map_err(|_| "Invalid width")?; + let y2: u32 = coords2[1].parse().map_err(|_| "Invalid width")?; - boxes.push(BoxInfo { id, x1, y1, x2 }); + boxes.push(BoxCoords { id, x1, y1, x2, y2 }); } + let ret: core::result::Result<(i32, i32), String>; + if let Ok(mut ts) = tracker_state.lock() { if ts.last_detect + Duration::from_secs(2) < Instant::now() && !boxes.is_empty() { info!("Setting new target: {}", boxes[0].id); ts.tracking_id = boxes[0].id; } - if let Some(target_box) = boxes.into_iter().find(|e| e.id == ts.tracking_id) { + if let Some(target_box) = boxes.iter().find(|e| e.id == ts.tracking_id) { let x_adjust = calc_x_adjust(target_box.x1, target_box.x2); let y_adjust = calc_y_adjust(target_box.y1); ts.last_detect = Instant::now(); - Ok((x_adjust, y_adjust)) + ret = Ok((x_adjust, y_adjust)); } else { - Err("Couldn't find target in results".to_string()) + ret = Err("Couldn't find target in results".to_string()); } } else { - Err("Couldn't lock tracker state".to_string()) + ret = Err("Couldn't lock tracker state".to_string()); } + + if let Ok(mut ib) = identity_boxes.lock() { + // Replace the memory address in the mutex guard with that of the created vec above + *ib = boxes; + } + + return ret; } fn calc_x_adjust(x1: u32, x2: u32) -> i32 { diff --git a/src/remote_sources/mod.rs b/src/remote_sources/mod.rs index 1045945..851b0ce 100644 --- a/src/remote_sources/mod.rs +++ b/src/remote_sources/mod.rs @@ -23,7 +23,10 @@ use tokio_tungstenite::{ mod automated_source; mod remote_source; -use crate::coordinator::{ApplicationEvent, ConnectionType}; +use crate::{ + coordinator::{ApplicationEvent, ConnectionType}, + ui::BoxCoords, +}; pub struct TrackerState { pub has_active_connection: bool, @@ -35,6 +38,7 @@ pub async fn start_socketserver( rt: Handle, mec: Sender, stay_alive: Arc, + identity_boxes: Arc>>, ) { let addr = "127.0.0.1:9002"; let listener = TcpListener::bind(&addr).await.expect("Can't listen"); @@ -57,6 +61,7 @@ pub async fn start_socketserver( stream, mec.clone(), tracker_state.clone(), + identity_boxes.clone(), )); } @@ -68,8 +73,11 @@ async fn accept_connection( stream: TcpStream, mec: Sender, tracker_state: Arc>, + identity_boxes: Arc>>, ) { - if let Err(e) = handle_connection(peer, stream, mec.clone(), tracker_state).await { + if let Err(e) = + handle_connection(peer, stream, mec.clone(), tracker_state, identity_boxes).await + { match e { Error::ConnectionClosed | Error::Protocol(_) | Error::Utf8 => (), err => error!("Error processing connection: {}", err), @@ -82,6 +90,7 @@ async fn handle_connection( stream: TcpStream, mec: Sender, tracker_state: Arc>, + identity_boxes: Arc>>, ) -> Result<()> { let mut ws_stream = accept_async(stream).await.expect("Failed to accept"); info!("New WebSocket connection: {}", peer); @@ -132,7 +141,8 @@ async fn handle_connection( if !ws_stream.is_terminated() { match connection_type.unwrap() { ConnectionType::Automated => { - automated_source::handle_connection(ws_stream, mec, tracker_state).await?; + automated_source::handle_connection(ws_stream, mec, tracker_state, identity_boxes) + .await?; } ConnectionType::Remote => { remote_source::handle_connection().await?; diff --git a/src/ui/mod.rs b/src/ui/mod.rs index 2ee0d11..4082614 100644 --- a/src/ui/mod.rs +++ b/src/ui/mod.rs @@ -1,4 +1,9 @@ +use std::sync::{Arc, Mutex}; +use gstreamer::prelude::{ElementExt, ElementExtManual, GstBinExt}; +use gstreamer::{ElementFactory, Pipeline, State}; +use gtk::cairo::Context; +use gtk::gdk::Paintable; use gtk::{glib, prelude::*, Box, Entry, Label, ListBox}; use gtk::{Application, ApplicationWindow, Button}; use log::error; @@ -7,7 +12,6 @@ use tokio::runtime::Handle; use tokio_tungstenite::tungstenite::Message; use crate::config::{load_config, save_config}; -// use crate::{joystick_loop, JoystickThreadUpdate}; use crate::coordinator::{start_coordinator, ApplicationEvent, MoveEvent}; #[derive(Debug, Serialize, Deserialize, Clone)] @@ -30,19 +34,39 @@ pub enum GuiUpdate { MoveEvent(MoveEvent), } +pub struct BoxCoords { + pub id: u32, + pub x1: u32, + pub y1: u32, + pub x2: u32, + pub y2: u32, +} + pub fn build_ui(app: &Application, runtime: Handle) { let initial_settings = load_config(); - let main_box = ListBox::new(); + let main_box = gtk::Box::new(gtk::Orientation::Horizontal, 0); + let left_box = ListBox::builder().width_request(200).build(); + + // Create a window + let window = ApplicationWindow::builder() + .application(app) + .title("VCC Camera Controller") + .default_width(840) + .default_height(480) + .child(&main_box) + .build(); // Main Event Channel let (to_mec, mec) = async_channel::unbounded::(); let (to_gui, gui_recv) = async_channel::bounded::(10); + let identity_boxes: Arc>> = Arc::new(Mutex::new(vec![])); runtime.spawn(start_coordinator( mec, to_mec.clone(), to_gui, runtime.clone(), + identity_boxes.clone(), )); // let conn_status_label = Label::new(Some(&"No Connection".to_string())); @@ -83,10 +107,46 @@ pub fn build_ui(app: &Application, runtime: Handle) { .css_classes(vec!["JoystickCurrent"]) .build(); - main_box.append(&conn_status_label); - main_box.append(&content_box); - main_box.append(&axis_label); - main_box.append(&button2); + left_box.append(&conn_status_label); + left_box.append(&content_box); + left_box.append(&axis_label); + left_box.append(&button2); + + main_box.append(&left_box); + + let webcam_picture = gtk::Picture::builder() + .height_request(480) + .width_request(640) + .can_focus(false) + .build(); + + let overlay_box = gtk::Overlay::new(); + main_box.append(&overlay_box); + + let (pipeline, paintable) = create_pipeline(); + pipeline + .set_state(State::Playing) + .expect("Could not set pipeline state to playing"); + + webcam_picture.set_paintable(Some(&paintable)); + + window.connect_close_request(move |_| { + pipeline + .set_state(State::Null) + .expect("Could not close pipeline during window deconstruction"); + glib::Propagation::Proceed + }); + + let drawable = gtk::DrawingArea::builder() + .content_height(480) + .content_width(640) + .build(); + drawable.set_draw_func(move |_, ctx, _width, _height| { + draw_boxes(&ctx, identity_boxes.clone()); + }); + + overlay_box.set_child(Some(&webcam_picture)); + overlay_box.add_overlay(&drawable); // Connect to "clicked" signal of `button` button.connect_clicked(glib::clone!(@weak ip_entry, @weak port_entry, @strong to_mec => move |_button| { @@ -157,15 +217,67 @@ pub fn build_ui(app: &Application, runtime: Handle) { }), ); - // Create a window - let window = ApplicationWindow::builder() - .application(app) - .title("VCC Camera Controller") - .child(&main_box) - .build(); - window.connect_close_request(move |_| glib::Propagation::Proceed); // Present window window.present(); -} \ No newline at end of file +} + +fn draw_boxes(ctx: &Context, boxes: Arc>>) { + ctx.set_line_width(2.0); + ctx.set_source_rgb(0.0, 1.0, 0.0); + + if let Ok(bxs) = boxes.lock() { + for b in bxs.iter() { + ctx.rectangle( + b.x1 as f64, + b.y1 as f64, + (b.x2 - b.x1) as f64, + (b.y2 - b.y1) as f64, + ); + if let Err(e) = ctx.stroke() { + error!("Could not draw cairo stroke: {e}"); + } + ctx.stroke() + .expect("GTK Cario context did not have a stroke!"); + } + } else { + error!("Could not get lock on boxes for drawing on the draw area!"); + } +} + +// Function to create GStreamer pipeline +fn create_pipeline() -> (Pipeline, Paintable) { + // Create pipeline + let pipeline = Pipeline::with_name("webcam_pipeline"); + + // Create elements + let source = ElementFactory::make("mfvideosrc") + .build() + .expect("Could not build video source for GStreamer"); + let convert = ElementFactory::make("videoconvert") + .build() + .expect("Could not build video convert for GStreamer"); + let sink = ElementFactory::make("gtk4paintablesink") + .build() + .expect("Could not build gtk sink for GStreamer"); + + let paintable = sink.property::("paintable"); + + // Add elements to the pipeline + pipeline + .add(&source) + .expect("Could not add GStreamer webcam source to pipeline"); + pipeline + .add(&convert) + .expect("Could not add GStreamer webcam convert to pipeline"); + pipeline + .add(&sink) + .expect("Could not add GStreamer gtksink to GStreamer pipeline"); + + // Link elements + source.link(&convert).unwrap(); + convert.link(&sink).unwrap(); + + (pipeline, paintable) +}