added webcam capture and box drawing

Cargo.lock

@@ -17,6 +17,21 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.81"
@@ -47,6 +62,12 @@ dependencies = [
  "syn 2.0.53",
 ]
 
+[[package]]
+name = "atomic_refcell"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c"
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -178,6 +199,18 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
 
+[[package]]
+name = "chrono"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
+dependencies = [
+ "android-tzdata",
+ "iana-time-zone",
+ "num-traits",
+ "windows-targets 0.52.4",
+]
+
 [[package]]
 name = "concurrent-queue"
 version = "2.4.0"
@@ -317,6 +350,12 @@ dependencies = [
  "const-random",
 ]
 
+[[package]]
+name = "either"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2"
+
 [[package]]
 name = "equivalent"
 version = "1.0.1"
@@ -517,6 +556,31 @@ dependencies = [
  "system-deps",
 ]
 
+[[package]]
+name = "gdk4-win32"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab6181b6e5c91ee292dca0032b00d48dee8e61358253742c9752537a88486b3f"
+dependencies = [
+ "gdk4",
+ "gdk4-win32-sys",
+ "gio",
+ "glib",
+ "libc",
+]
+
+[[package]]
+name = "gdk4-win32-sys"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efa8530d6619cf43f007f3efd993a356e1ca4e643c4d0bd2a99832a08af2e402"
+dependencies = [
+ "gdk4-sys",
+ "glib-sys",
+ "libc",
+ "system-deps",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -719,6 +783,159 @@ dependencies = [
  "system-deps",
 ]
 
+[[package]]
+name = "gst-plugin-gtk4"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83c12132d3619f85db54f6034b4e3872577886657b07e055c521baeb2d5f6733"
+dependencies = [
+ "async-channel",
+ "gdk4-win32",
+ "gst-plugin-version-helper",
+ "gstreamer",
+ "gstreamer-base",
+ "gstreamer-gl",
+ "gstreamer-video",
+ "gtk4",
+ "once_cell",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "gst-plugin-version-helper"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e5e874f1660252fd2ec81c602066df3633b3a6fcbe2b196f7f93c27cf069b2a"
+dependencies = [
+ "chrono",
+ "toml_edit 0.22.9",
+]
+
+[[package]]
+name = "gstreamer"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc6f2d7dcde4b42b5297d25e9f51914cacfa148c99ba6ddabebf006fb2b18c20"
+dependencies = [
+ "cfg-if",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "glib",
+ "gstreamer-sys",
+ "itertools",
+ "libc",
+ "muldiv",
+ "num-integer",
+ "num-rational",
+ "once_cell",
+ "option-operations",
+ "paste",
+ "pin-project-lite",
+ "smallvec",
+ "thiserror",
+]
+
+[[package]]
+name = "gstreamer-base"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "514c71195b53c7eced4842b66ca9149833e41cf6a1d949e45e2ca4a4fa929850"
+dependencies = [
+ "atomic_refcell",
+ "cfg-if",
+ "glib",
+ "gstreamer",
+ "gstreamer-base-sys",
+ "libc",
+]
+
+[[package]]
+name = "gstreamer-base-sys"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "286591e0f85bbda1adf9bab6f21d015acd9ca0a4d4acb61da65e3d0487e23c4e"
+dependencies = [
+ "glib-sys",
+ "gobject-sys",
+ "gstreamer-sys",
+ "libc",
+ "system-deps",
+]
+
+[[package]]
+name = "gstreamer-gl"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d21c0c5fbf74018a0254b3ab77bca0a5b2c0f002bcfd910c09113ae90a95d98"
+dependencies = [
+ "glib",
+ "gstreamer",
+ "gstreamer-base",
+ "gstreamer-gl-sys",
+ "gstreamer-video",
+ "libc",
+ "once_cell",
+]
+
+[[package]]
+name = "gstreamer-gl-sys"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61d1e3b9b02abc23835e9d770f2bd705b67a50406ea37e963b4526a77c6a7cd8"
+dependencies = [
+ "glib-sys",
+ "gobject-sys",
+ "gstreamer-base-sys",
+ "gstreamer-sys",
+ "gstreamer-video-sys",
+ "libc",
+ "system-deps",
+]
+
+[[package]]
+name = "gstreamer-sys"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5ddf526b3bf90ea627224c804f00b8bcb0452e3b447978b4d5092f8e8ff5918"
+dependencies = [
+ "glib-sys",
+ "gobject-sys",
+ "libc",
+ "system-deps",
+]
+
+[[package]]
+name = "gstreamer-video"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fc0a96481ecfcdd120d5057bb7ab5a6f6cd392cc34ba1e8b86cac3ba082f788"
+dependencies = [
+ "cfg-if",
+ "futures-channel",
+ "glib",
+ "gstreamer",
+ "gstreamer-base",
+ "gstreamer-video-sys",
+ "libc",
+ "once_cell",
+ "thiserror",
+]
+
+[[package]]
+name = "gstreamer-video-sys"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1ea7996ba44fbbf563aeeda96e24259efc9f06b407854d837ee58e260d7ba78"
+dependencies = [
+ "glib-sys",
+ "gobject-sys",
+ "gstreamer-base-sys",
+ "gstreamer-sys",
+ "libc",
+ "system-deps",
+]
+
 [[package]]
 name = "gtk4"
 version = "0.8.1"
@@ -814,6 +1031,29 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "windows-core 0.52.0",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "idna"
 version = "0.5.0"
@@ -873,6 +1113,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.10"
@@ -889,6 +1138,8 @@ dependencies = [
  "futures-core",
  "futures-util",
  "gilrs",
+ "gst-plugin-gtk4",
+ "gstreamer",
  "gtk4",
  "log",
  "serde",
@@ -1002,6 +1253,12 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "muldiv"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "956787520e75e9bd233246045d19f42fb73242759cc57fba9611d940ae96d4b0"
+
 [[package]]
 name = "nix"
 version = "0.28.0"
@@ -1030,6 +1287,35 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.16.0"
@@ -1064,6 +1350,15 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
+[[package]]
+name = "option-operations"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c26d27bb1aeab65138e4bf7666045169d1717febcc9ff870166be8348b223d0"
+dependencies = [
+ "paste",
+]
+
 [[package]]
 name = "ordered-multimap"
 version = "0.6.0"
@@ -1104,6 +1399,12 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
 
+[[package]]
+name = "paste"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
+
 [[package]]
 name = "pathdiff"
 version = "0.2.1"
@@ -1860,7 +2161,16 @@ version = "0.54.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49"
 dependencies = [
- "windows-core",
+ "windows-core 0.54.0",
+ "windows-targets 0.52.4",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+dependencies = [
  "windows-targets 0.52.4",
 ]
 

Cargo.toml

@@ -12,6 +12,8 @@ err-derive = "0.3.1"
 futures-core = "0.3.30"
 futures-util = { version = "0.3.30", features = ["tokio-io"] }
 gilrs = "0.10.6"
+gstreamer = { version = "0.22.4", features = ["v1_22"] }
+gst-plugin-gtk4 = { version = "0.12.2", features = ["gtk_v4_12"] }
 gtk = { version = "0.8.1", package = "gtk4", features = ["v4_12"] }
 log = "0.4.21"
 serde = { version = "1.0.197", features = ["derive"] }

src/coordinator.rs

@@ -1,4 +1,5 @@
 use std::pin::{pin, Pin};
+use std::sync::Mutex;
 use std::sync::{
     atomic::{AtomicBool, Ordering},
     Arc,
@@ -16,9 +17,9 @@ use tokio::runtime::Handle;
 use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream};
 
 use crate::remote_sources;
+use crate::ui::BoxCoords;
 use crate::{joystick_source::joystick_loop, ui::GuiUpdate};
 
-
 const PRIORITY_TIMEOUT: Duration = Duration::from_secs(2);
 
 #[derive(Clone)]
@@ -43,6 +44,7 @@ pub enum ApplicationEvent {
 struct CoordState<'a> {
     pub sck_outbound: Option<SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>>,
     pub sck_alive_server: Arc<AtomicBool>,
+    pub identity_boxes: Arc<Mutex<Vec<BoxCoords>>>,
     pub sck_alive_recvr: Arc<AtomicBool>,
     pub joystick_loop_alive: Arc<AtomicBool>,
 
@@ -61,11 +63,13 @@ impl<'a> CoordState<'a> {
         to_mec: Sender<ApplicationEvent>,
         to_gui: Sender<GuiUpdate>,
         rt: Handle,
+        identity_boxes: Arc<Mutex<Vec<BoxCoords>>>,
     ) -> Self {
         CoordState {
             sck_outbound: None,
             sck_alive_recvr: Arc::new(AtomicBool::new(false)),
             sck_alive_server: Arc::new(AtomicBool::new(false)),
+            identity_boxes,
             joystick_loop_alive: Arc::new(AtomicBool::new(false)),
 
             current_priority: ConnectionType::Local,
@@ -138,6 +142,7 @@ impl<'a> CoordState<'a> {
                 self.rt.clone(),
                 self.to_mec.clone(),
                 self.sck_alive_server.clone(),
+                self.identity_boxes.clone(),
             ));
         }
 
@@ -170,12 +175,13 @@ pub async fn start_coordinator(
     to_mec: Sender<ApplicationEvent>,
     to_gui: Sender<GuiUpdate>,
     runtime: Handle,
+    identity_boxes: Arc<Mutex<Vec<BoxCoords>>>,
 ) {
     info!("Starting coordinator!");
 
     let mec = pin!(mec);
 
-    let mut state = CoordState::new(mec, to_mec, to_gui, runtime);
+    let mut state = CoordState::new(mec, to_mec, to_gui, runtime, identity_boxes);
 
     state.check_states().await;
 
@@ -197,7 +203,9 @@ pub async fn start_coordinator(
             }
             ApplicationEvent::MoveEvent(coord, priority) => {
                 // If Automatic control, but local event happens, override the automatice events for 2 seconds
-                if priority <= state.current_priority || Instant::now() > state.last_update_of_priority + PRIORITY_TIMEOUT {
+                if priority <= state.current_priority
+                    || Instant::now() > state.last_update_of_priority + PRIORITY_TIMEOUT
+                {
                     state.last_update_of_priority = Instant::now();
                     state.current_priority = priority;
 
@@ -216,7 +224,6 @@ pub async fn start_coordinator(
 
                         state.socket_send(Message::Text(message)).await;
                     }
-
                 }
             }
         }

src/main.rs

@@ -22,6 +22,10 @@ fn main() -> glib::ExitCode {
         error!("Failed to init the simplelogger!: {e}");
     }
 
+    gstreamer::init().expect("Unable to start gstreamer");
+
+    gstgtk4::plugin_register_static().expect("Unable to register gtk4 plugin");
+
     let rt = runtime::Runtime::new().expect("Could not start tokio runtime");
     let handle = rt.handle().clone();
 

src/remote_sources/automated_source.rs

@@ -11,7 +11,10 @@ use tokio::net::TcpStream;
 use tokio_tungstenite::{tungstenite::Result, WebSocketStream};
 
 use super::TrackerState;
-use crate::coordinator::{ApplicationEvent, MoveEvent};
+use crate::{
+    coordinator::{ApplicationEvent, MoveEvent},
+    ui::BoxCoords,
+};
 
 const HALF_FRAME_WIDTH: f64 = 320.0;
 
@@ -19,6 +22,7 @@ pub async fn handle_connection(
     mut ws_stream: WebSocketStream<TcpStream>,
     mec: Sender<ApplicationEvent>,
     tracker_state: Arc<Mutex<TrackerState>>,
+    identity_boxes: Arc<Mutex<Vec<BoxCoords>>>,
 ) -> Result<()> {
     if let Ok(mut ts) = tracker_state.lock() {
         ts.has_active_connection = true;
@@ -26,14 +30,17 @@ pub async fn handle_connection(
     while let Some(msg) = ws_stream.next().await {
         let msg = msg?;
         if msg.is_text() {
-            let (x_off, y_off) =
+            let (x_off, y_off) = match process_incoming_string(
-                match process_incoming_string(msg.to_string(), tracker_state.clone()) {
+                msg.to_string(),
-                    Ok(val) => val,
+                tracker_state.clone(),
-                    Err(e) => {
+                &identity_boxes,
-                        error!("{e}");
+            ) {
-                        (0, 0)
+                Ok(val) => val,
-                    }
+                Err(e) => {
-                };
+                    error!("{e}");
+                    (0, 0)
+                }
+            };
 
             if let Err(e) = mec
                 .send(ApplicationEvent::MoveEvent(
@@ -53,18 +60,12 @@ pub async fn handle_connection(
     Ok(())
 }
 
-struct BoxInfo {
-    id: u32,
-    x1: u32,
-    y1: u32,
-    x2: u32,
-}
-
 fn process_incoming_string(
     message: String,
     tracker_state: Arc<Mutex<TrackerState>>,
+    identity_boxes: &Arc<Mutex<Vec<BoxCoords>>>,
 ) -> core::result::Result<(i32, i32), String> {
-    let mut boxes: Vec<BoxInfo> = Vec::new();
+    let mut boxes: Vec<BoxCoords> = Vec::new();
 
     for line in message.lines() {
         let parts: Vec<&str> = line.split(' ').collect();
@@ -91,27 +92,37 @@ fn process_incoming_string(
         }
 
         let x2: u32 = coords2[0].parse().map_err(|_| "Invalid width")?;
+        let y2: u32 = coords2[1].parse().map_err(|_| "Invalid width")?;
 
-        boxes.push(BoxInfo { id, x1, y1, x2 });
+        boxes.push(BoxCoords { id, x1, y1, x2, y2 });
     }
 
+    let ret: core::result::Result<(i32, i32), String>;
+
     if let Ok(mut ts) = tracker_state.lock() {
         if ts.last_detect + Duration::from_secs(2) < Instant::now() && !boxes.is_empty() {
             info!("Setting new target: {}", boxes[0].id);
             ts.tracking_id = boxes[0].id;
         }
 
-        if let Some(target_box) = boxes.into_iter().find(|e| e.id == ts.tracking_id) {
+        if let Some(target_box) = boxes.iter().find(|e| e.id == ts.tracking_id) {
             let x_adjust = calc_x_adjust(target_box.x1, target_box.x2);
             let y_adjust = calc_y_adjust(target_box.y1);
             ts.last_detect = Instant::now();
-            Ok((x_adjust, y_adjust))
+            ret = Ok((x_adjust, y_adjust));
         } else {
-            Err("Couldn't find target in results".to_string())
+            ret = Err("Couldn't find target in results".to_string());
         }
     } else {
-        Err("Couldn't lock tracker state".to_string())
+        ret = Err("Couldn't lock tracker state".to_string());
     }
+
+    if let Ok(mut ib) = identity_boxes.lock() {
+        // Replace the memory address in the mutex guard with that of the created vec above
+        *ib = boxes;
+    }
+
+    return ret;
 }
 
 fn calc_x_adjust(x1: u32, x2: u32) -> i32 {

src/remote_sources/mod.rs

@@ -23,7 +23,10 @@ use tokio_tungstenite::{
 mod automated_source;
 mod remote_source;
 
-use crate::coordinator::{ApplicationEvent, ConnectionType};
+use crate::{
+    coordinator::{ApplicationEvent, ConnectionType},
+    ui::BoxCoords,
+};
 
 pub struct TrackerState {
     pub has_active_connection: bool,
@@ -35,6 +38,7 @@ pub async fn start_socketserver(
     rt: Handle,
     mec: Sender<ApplicationEvent>,
     stay_alive: Arc<AtomicBool>,
+    identity_boxes: Arc<Mutex<Vec<BoxCoords>>>,
 ) {
     let addr = "127.0.0.1:9002";
     let listener = TcpListener::bind(&addr).await.expect("Can't listen");
@@ -57,6 +61,7 @@ pub async fn start_socketserver(
             stream,
             mec.clone(),
             tracker_state.clone(),
+            identity_boxes.clone(),
         ));
     }
 
@@ -68,8 +73,11 @@ async fn accept_connection(
     stream: TcpStream,
     mec: Sender<ApplicationEvent>,
     tracker_state: Arc<Mutex<TrackerState>>,
+    identity_boxes: Arc<Mutex<Vec<BoxCoords>>>,
 ) {
-    if let Err(e) = handle_connection(peer, stream, mec.clone(), tracker_state).await {
+    if let Err(e) =
+        handle_connection(peer, stream, mec.clone(), tracker_state, identity_boxes).await
+    {
         match e {
             Error::ConnectionClosed | Error::Protocol(_) | Error::Utf8 => (),
             err => error!("Error processing connection: {}", err),
@@ -82,6 +90,7 @@ async fn handle_connection(
     stream: TcpStream,
     mec: Sender<ApplicationEvent>,
     tracker_state: Arc<Mutex<TrackerState>>,
+    identity_boxes: Arc<Mutex<Vec<BoxCoords>>>,
 ) -> Result<()> {
     let mut ws_stream = accept_async(stream).await.expect("Failed to accept");
     info!("New WebSocket connection: {}", peer);
@@ -132,7 +141,8 @@ async fn handle_connection(
     if !ws_stream.is_terminated() {
         match connection_type.unwrap() {
             ConnectionType::Automated => {
-                automated_source::handle_connection(ws_stream, mec, tracker_state).await?;
+                automated_source::handle_connection(ws_stream, mec, tracker_state, identity_boxes)
+                    .await?;
             }
             ConnectionType::Remote => {
                 remote_source::handle_connection().await?;

src/ui/mod.rs

@@ -1,4 +1,9 @@
+use std::sync::{Arc, Mutex};
 
+use gstreamer::prelude::{ElementExt, ElementExtManual, GstBinExt};
+use gstreamer::{ElementFactory, Pipeline, State};
+use gtk::cairo::Context;
+use gtk::gdk::Paintable;
 use gtk::{glib, prelude::*, Box, Entry, Label, ListBox};
 use gtk::{Application, ApplicationWindow, Button};
 use log::error;
@@ -7,7 +12,6 @@ use tokio::runtime::Handle;
 use tokio_tungstenite::tungstenite::Message;
 
 use crate::config::{load_config, save_config};
-// use crate::{joystick_loop, JoystickThreadUpdate};
 use crate::coordinator::{start_coordinator, ApplicationEvent, MoveEvent};
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -30,19 +34,39 @@ pub enum GuiUpdate {
     MoveEvent(MoveEvent),
 }
 
+pub struct BoxCoords {
+    pub id: u32,
+    pub x1: u32,
+    pub y1: u32,
+    pub x2: u32,
+    pub y2: u32,
+}
+
 pub fn build_ui(app: &Application, runtime: Handle) {
     let initial_settings = load_config();
-    let main_box = ListBox::new();
+    let main_box = gtk::Box::new(gtk::Orientation::Horizontal, 0);
+    let left_box = ListBox::builder().width_request(200).build();
+
+    // Create a window
+    let window = ApplicationWindow::builder()
+        .application(app)
+        .title("VCC Camera Controller")
+        .default_width(840)
+        .default_height(480)
+        .child(&main_box)
+        .build();
 
     // Main Event Channel
     let (to_mec, mec) = async_channel::unbounded::<ApplicationEvent>();
     let (to_gui, gui_recv) = async_channel::bounded::<GuiUpdate>(10);
+    let identity_boxes: Arc<Mutex<Vec<BoxCoords>>> = Arc::new(Mutex::new(vec![]));
 
     runtime.spawn(start_coordinator(
         mec,
         to_mec.clone(),
         to_gui,
         runtime.clone(),
+        identity_boxes.clone(),
     ));
 
     // let conn_status_label = Label::new(Some(&"No Connection".to_string()));
@@ -83,10 +107,46 @@ pub fn build_ui(app: &Application, runtime: Handle) {
         .css_classes(vec!["JoystickCurrent"])
         .build();
 
-    main_box.append(&conn_status_label);
+    left_box.append(&conn_status_label);
-    main_box.append(&content_box);
+    left_box.append(&content_box);
-    main_box.append(&axis_label);
+    left_box.append(&axis_label);
-    main_box.append(&button2);
+    left_box.append(&button2);
+
+    main_box.append(&left_box);
+
+    let webcam_picture = gtk::Picture::builder()
+        .height_request(480)
+        .width_request(640)
+        .can_focus(false)
+        .build();
+
+    let overlay_box = gtk::Overlay::new();
+    main_box.append(&overlay_box);
+
+    let (pipeline, paintable) = create_pipeline();
+    pipeline
+        .set_state(State::Playing)
+        .expect("Could not set pipeline state to playing");
+
+    webcam_picture.set_paintable(Some(&paintable));
+
+    window.connect_close_request(move |_| {
+        pipeline
+            .set_state(State::Null)
+            .expect("Could not close pipeline during window deconstruction");
+        glib::Propagation::Proceed
+    });
+
+    let drawable = gtk::DrawingArea::builder()
+        .content_height(480)
+        .content_width(640)
+        .build();
+    drawable.set_draw_func(move |_, ctx, _width, _height| {
+        draw_boxes(&ctx, identity_boxes.clone());
+    });
+
+    overlay_box.set_child(Some(&webcam_picture));
+    overlay_box.add_overlay(&drawable);
 
     // Connect to "clicked" signal of `button`
     button.connect_clicked(glib::clone!(@weak ip_entry, @weak port_entry, @strong to_mec => move |_button| {
@@ -157,15 +217,67 @@ pub fn build_ui(app: &Application, runtime: Handle) {
         }),
     );
 
-    // Create a window
-    let window = ApplicationWindow::builder()
-        .application(app)
-        .title("VCC Camera Controller")
-        .child(&main_box)
-        .build();
-
     window.connect_close_request(move |_| glib::Propagation::Proceed);
 
     // Present window
     window.present();
 }
+
+fn draw_boxes(ctx: &Context, boxes: Arc<Mutex<Vec<BoxCoords>>>) {
+    ctx.set_line_width(2.0);
+    ctx.set_source_rgb(0.0, 1.0, 0.0);
+
+    if let Ok(bxs) = boxes.lock() {
+        for b in bxs.iter() {
+            ctx.rectangle(
+                b.x1 as f64,
+                b.y1 as f64,
+                (b.x2 - b.x1) as f64,
+                (b.y2 - b.y1) as f64,
+            );
+            if let Err(e) = ctx.stroke() {
+                error!("Could not draw cairo stroke: {e}");
+            }
+            ctx.stroke()
+                .expect("GTK Cario context did not have a stroke!");
+        }
+    } else {
+        error!("Could not get lock on boxes for drawing on the draw area!");
+    }
+}
+
+// Function to create GStreamer pipeline
+fn create_pipeline() -> (Pipeline, Paintable) {
+    // Create pipeline
+    let pipeline = Pipeline::with_name("webcam_pipeline");
+
+    // Create elements
+    let source = ElementFactory::make("mfvideosrc")
+        .build()
+        .expect("Could not build video source for GStreamer");
+    let convert = ElementFactory::make("videoconvert")
+        .build()
+        .expect("Could not build video convert for GStreamer");
+    let sink = ElementFactory::make("gtk4paintablesink")
+        .build()
+        .expect("Could not build gtk sink for GStreamer");
+
+    let paintable = sink.property::<Paintable>("paintable");
+
+    // Add elements to the pipeline
+    pipeline
+        .add(&source)
+        .expect("Could not add GStreamer webcam source to pipeline");
+    pipeline
+        .add(&convert)
+        .expect("Could not add GStreamer webcam convert to pipeline");
+    pipeline
+        .add(&sink)
+        .expect("Could not add GStreamer gtksink to GStreamer pipeline");
+
+    // Link elements
+    source.link(&convert).unwrap();
+    convert.link(&sink).unwrap();
+
+    (pipeline, paintable)
+}