From af87f9166fbf67e324e74461a3835a4b455d3578 Mon Sep 17 00:00:00 2001 From: Nickiel12 Date: Sat, 24 Aug 2024 22:06:16 +0000 Subject: [PATCH] working webrtc video feed --- .gitignore | 12 +- Cargo.lock | 175 +++++++------- Cargo.toml | 90 ++++---- src/gstreamer_pipeline.rs | 466 +++++++++++++++++++------------------- src/main.rs | 356 +++++++++++++++-------------- src/tauri_functions.rs | 31 ++- src/webrtc_remote.rs | 264 ++++++++++----------- ui/index.html | 105 +++++---- ui/static/index.js | 42 ++-- ui/static/rtc.js | 191 ++++++++-------- 10 files changed, 889 insertions(+), 843 deletions(-) diff --git a/.gitignore b/.gitignore index 46bccdd..f9a3e2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -target -settings.toml -.direnv/* -logs/* -ui/static/node_modules/* -ui/static/@tauri-apps/* +target +settings.toml +.direnv/* +logs/* +ui/static/node_modules/* +ui/static/@tauri-apps/* diff --git a/Cargo.lock b/Cargo.lock index 76371b7..b667076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,7 +160,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "synstructure 0.13.1", ] @@ -183,7 +183,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -206,7 +206,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -228,7 +228,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -239,7 +239,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -424,9 +424,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.5.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -435,9 +435,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -522,9 +522,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.13" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" +checksum = "50d2eb3cd3d1bf4529e31c215ee6f93ec5a3d536d9f578f93d9d33ee19562932" dependencies = [ "shlex", ] @@ -926,7 +926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -936,7 +936,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -971,7 +971,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -995,7 +995,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1006,7 +1006,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1073,7 +1073,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1123,7 +1123,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1265,9 +1265,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fdeflate" @@ -1318,14 +1318,23 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", "miniz_oxide 0.8.0", ] +[[package]] +name = "fluent-uri" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1422,7 +1431,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1756,7 +1765,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -2453,10 +2462,11 @@ dependencies = [ [[package]] name = "json-patch" -version = "1.4.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec9ad60d674508f3ca8f380a928cfe7b096bc729c4e2dbfe3852bc45da3ab30b" +checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc" dependencies = [ + "jsonptr", "serde", "serde_json", "thiserror", @@ -2473,6 +2483,17 @@ dependencies = [ "serde", ] +[[package]] +name = "jsonptr" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" +dependencies = [ + "fluent-uri", + "serde", + "serde_json", +] + [[package]] name = "kuchikiki" version = "0.8.2" @@ -3083,7 +3104,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3201,7 +3222,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3248,7 +3269,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3431,7 +3452,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3454,9 +3475,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -3856,29 +3877,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.208" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.208" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] name = "serde_json" -version = "1.0.125" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" +checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" dependencies = [ "indexmap 2.4.0", "itoa 1.0.11", @@ -3895,7 +3916,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3934,14 +3955,14 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] name = "serialize-to-javascript" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9823f2d3b6a81d98228151fdeaf848206a7855a7a042bbf9bf870449a66cafb" +checksum = "04f3666a07a197cdb77cdf306c32be9b7f598d7060d50cfd4d5aa04bfd92f6c5" dependencies = [ "serde", "serde_json", @@ -3950,13 +3971,13 @@ dependencies = [ [[package]] name = "serialize-to-javascript-impl" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74064874e9f6a15f04c1f3cb627902d0e6b410abbf36668afa873c61889f1763" +checksum = "772ee033c0916d670af7860b6e1ef7d658a4629a6d0b4c8c3e67f09b3765b75d" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.76", ] [[package]] @@ -4070,7 +4091,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4221,9 +4242,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.75" +version = "2.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" +checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" dependencies = [ "proc-macro2", "quote", @@ -4256,7 +4277,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4334,13 +4355,13 @@ dependencies = [ [[package]] name = "tao-macros" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec114582505d158b669b136e6851f85840c109819d77c42bb7c0709f727d18c2" +checksum = "f4e16beb8b2ac17db28eab8bca40e62dbfbb34c0fcdc6d9826b11b7b5d047dfd" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.76", ] [[package]] @@ -4362,9 +4383,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tauri" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336bc661a3f3250853fa83c6e5245449ed1c26dce5dcb28bdee7efedf6278806" +checksum = "0e33e3ba00a3b05eb6c57ef135781717d33728b48acf914bb05629e74d897d29" dependencies = [ "anyhow", "cocoa", @@ -4409,9 +4430,9 @@ dependencies = [ [[package]] name = "tauri-build" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c6ec7a5c3296330c7818478948b422967ce4649094696c985f61d50076d29c" +checksum = "d5fb5a90a64241ddb7217d3210d844149070a911e87e8a107a707a1d4973f164" dependencies = [ "anyhow", "cargo_toml", @@ -4428,9 +4449,9 @@ dependencies = [ [[package]] name = "tauri-codegen" -version = "1.4.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1aed706708ff1200ec12de9cfbf2582b5d8ec05f6a7293911091effbd22036b" +checksum = "93a9e3f5cebf779a63bf24903e714ec91196c307d8249a0008b882424328bcda" dependencies = [ "base64 0.21.7", "brotli", @@ -4453,9 +4474,9 @@ dependencies = [ [[package]] name = "tauri-macros" -version = "1.4.5" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88f831d2973ae4f81a706a0004e67dac87f2e4439973bbe98efbd73825d8ede" +checksum = "d1d0e989f54fe06c5ef0875c5e19cf96453d099a0a774d5192ab47e80471cdab" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -4467,9 +4488,9 @@ dependencies = [ [[package]] name = "tauri-runtime" -version = "0.14.4" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3068ed62b63dedc705558f4248c7ecbd5561f0f8050949859ea0db2326f26012" +checksum = "f33fda7d213e239077fad52e96c6b734cecedb30c2382118b64f94cb5103ff3a" dependencies = [ "gtk", "http 0.2.12", @@ -4488,9 +4509,9 @@ dependencies = [ [[package]] name = "tauri-runtime-wry" -version = "0.14.9" +version = "0.14.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c3db170233096aa30330feadcd895bf9317be97e624458560a20e814db7955" +checksum = "18c447dcd9b0f09c7dc4b752cc33e72788805bfd761fbda5692d30c48289efec" dependencies = [ "cocoa", "gtk", @@ -4508,9 +4529,9 @@ dependencies = [ [[package]] name = "tauri-utils" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2826db448309d382dac14d520f0c0a40839b87b57b977e59cf5f296b3ace6a93" +checksum = "83a0c939e88d82903a0a7dfb28388b12a3c03504d6bd6086550edaa3b6d8beaa" dependencies = [ "brotli", "ctor", @@ -4593,7 +4614,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4697,7 +4718,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4917,7 +4938,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -5120,7 +5141,7 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "vcs-common" version = "0.1.0" -source = "git+https://git.nickiel.net/VCC/vcs-common.git?branch=main#0765f70fa773261f38dddb1819aaad47f88e12d7" +source = "git+https://git.nickiel.net/VCC/vcs-common.git?branch=main#59b0b88c53032514bf5ba68efe17b6f089734bfe" dependencies = [ "async-channel", "bincode", @@ -5269,7 +5290,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "wasm-bindgen-shared", ] @@ -5291,7 +5312,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5720,7 +5741,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -5731,7 +5752,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -6116,7 +6137,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -6136,5 +6157,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] diff --git a/Cargo.toml b/Cargo.toml index 83250b7..8c2fafd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,45 +1,45 @@ -[package] -name = "vcs-controller" -version = "2.0.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[build-dependencies] -tauri-build = { version = "1.5.1", features = [] } - - -[features] -tracker-state-debug = [] -tokio-logging = [] -# this feature is used for production builds or when `devPath` points to the filesystem and the built-in dev server is disabled. -# If you use cargo directly instead of tauri's cli you can use this feature flag to switch between tauri's `dev` and `build` modes. -# DO NOT REMOVE!! -custom-protocol = [ "tauri/custom-protocol" ] - -[dependencies] -async-channel = "2.2.0" -async-recursion = "1.1.1" -config = "0.14.0" -futures-core = "0.3.30" -futures-util = { version = "0.3.30" } -gilrs = "0.10.6" -gstreamer = { version = "0.22.4", features = ["v1_22"] } -gstreamer-app = { version = "0.22.0", features = ["v1_22"] } -log = "0.4.21" -serde = { version = "1.0.197", features = ["derive"] } -serde_json = "1.0" -tokio = { version = "1.37.0", features = ["rt-multi-thread", "time", "sync"] } -tokio-tungstenite = "0.21.0" -toml = "0.8.12" -tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["tracing-log"] } -tracing-appender = "0.2.3" -snafu = "0.8.2" -console-subscriber = "0.3.0" -tauri = { version = "1.6.1", features = [] } -lazy_static = "1.5.0" - -vcs-common = { git = "https://git.nickiel.net/VCC/vcs-common.git", branch = "main" } -bincode = "1.3.3" - - +[package] +name = "vcs-controller" +version = "2.0.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[build-dependencies] +tauri-build = { version = "1.5.1", features = [] } + + +[features] +tracker-state-debug = [] +tokio-logging = [] +# this feature is used for production builds or when `devPath` points to the filesystem and the built-in dev server is disabled. +# If you use cargo directly instead of tauri's cli you can use this feature flag to switch between tauri's `dev` and `build` modes. +# DO NOT REMOVE!! +custom-protocol = [ "tauri/custom-protocol" ] + +[dependencies] +async-channel = "2.2.0" +async-recursion = "1.1.1" +config = "0.14.0" +futures-core = "0.3.30" +futures-util = { version = "0.3.30" } +gilrs = "0.10.6" +gstreamer = { version = "0.22.4", features = ["v1_22"] } +gstreamer-app = { version = "0.22.0", features = ["v1_22"] } +log = "0.4.21" +serde = { version = "1.0.197", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.37.0", features = ["rt-multi-thread", "time", "sync"] } +tokio-tungstenite = "0.21.0" +toml = "0.8.12" +tracing = "0.1.40" +tracing-subscriber = { version = "0.3.18", features = ["tracing-log"] } +tracing-appender = "0.2.3" +snafu = "0.8.2" +console-subscriber = "0.3.0" +tauri = { version = "1.6.1", features = [] } +lazy_static = "1.5.0" + +vcs-common = { git = "https://git.nickiel.net/VCC/vcs-common.git", branch = "main" } +bincode = "1.3.3" + + diff --git a/src/gstreamer_pipeline.rs b/src/gstreamer_pipeline.rs index 5050e06..2c35414 100644 --- a/src/gstreamer_pipeline.rs +++ b/src/gstreamer_pipeline.rs @@ -1,233 +1,233 @@ -use gstreamer::{prelude::*, PadLinkError}; -use gstreamer::{Element, ElementFactory, Pipeline}; -use gstreamer_app::AppSink; -use gstreamer::glib::BoolError; -use snafu::prelude::*; -use std::str::FromStr; -use std::sync::Arc; -use tokio::sync::Mutex; - -#[derive(Debug)] -pub struct WebcamPipeline { - pub pipeline: Pipeline, - - pub sink_frame: Arc>, -} - -impl WebcamPipeline { - pub fn new(jpeg_quality: i32) -> Result { - let pipeline = Pipeline::with_name("webcam_pipeline"); - - // All of the following errors are unrecoverable - - let mut video_src = ""; - if cfg!(windows) { - video_src = "mfvideosrc"; - } else if cfg!(unix) { - video_src = "v4l2src"; - } - let source = ElementFactory::make(video_src) - .build() - .context(BuildSnafu { - element: "mfvideosrc", - })?; - let convert = ElementFactory::make("videoconvert") - .build() - .context(BuildSnafu { - element: "videoconvert", - })?; - let rate = ElementFactory::make("videorate") - .build() - .context(BuildSnafu { - element: "videorate", - })?; - - let tee = ElementFactory::make("tee") - .build() - .context(BuildSnafu { element: "tee" })?; - - let queue_app = ElementFactory::make("queue") - .property("max-size-time", 1u64) - .property("max-size-buffers", 0u32) - .property("max-size-bytes", 0u32) - .build() - .context(BuildSnafu { - element: "paintable queue", - })?; - - let webrtc_sink = ElementFactory::make("webrtcsink") - // .property("meta", "meta,name=gst-stream") - .name("web rtc sink") - .build() - .context(BuildSnafu { - element: "webrtcsink" - })?; - - // queue.connect_closure("overrun", false, glib::closure!(|queue: Element| { - // println!("The queue is full!"); - // })); - - let appsink_queue = ElementFactory::make("queue") - .property("max-size-time", 1u64) - .property("max-size-buffers", 0u32) - .property("max-size-bytes", 0u32) - .build() - .context(BuildSnafu { - element: "appsink queue", - })?; - - let resize = ElementFactory::make("videoscale") - .build() - .context(BuildSnafu { - element: "videoscale", - })?; - - let jpeg_enc = ElementFactory::make("jpegenc") - .property("quality", jpeg_quality) - .build() - .context(BuildSnafu { element: "jpegenc" })?; - - let caps_string = "image/jpeg,width=640,height=640"; - let appsrc_caps = gstreamer::Caps::from_str(caps_string).context(BuildSnafu { - element: "appsink caps", - })?; - - let sink_frame = AppSink::builder() - .name("frame_appsink") - .sync(false) - .max_buffers(1u32) - .drop(true) - .caps(&appsrc_caps) - .build(); - - sink_frame.set_property("caps", &appsrc_caps.to_value()); - - pipeline - .add_many([ - &source, - &convert, - &rate, - &tee, - &queue_app, - &webrtc_sink, - &appsink_queue, - &resize, - &jpeg_enc, - &sink_frame.upcast_ref(), - ]) - .context(LinkSnafu { - from: "all", - to: "pipeline", - })?; - - Element::link_many([&source, &convert, &rate]).context(LinkSnafu { - from: "source et. al.", - to: "rate", - })?; - - // -- BEGIN PAINTABLE SINK PIPELINE - let tee_caps = - // gstreamer::caps::Caps::from_str("video/x-raw,framerate=15/1").context(BuildSnafu { - gstreamer::caps::Caps::from_str("video/x-raw").context(BuildSnafu { - element: "tee caps", - })?; - - rate.link_filtered(&tee, &tee_caps).context(LinkSnafu { - from: "videorate", - to: "tee", - })?; - - let tee_src_1 = tee - .request_pad_simple("src_%u") - .ok_or(PipelineError::PadRequest { - element: "tee pad 1".to_string(), - })?; - let paintable_queue_sinkpad = - queue_app - .static_pad("sink") - .ok_or(PipelineError::PadRequest { - element: "gtk4 sink".to_string(), - })?; - - tee_src_1 - .link(&paintable_queue_sinkpad) - .context(PadLinkSnafu { - from: "tee src pad", - to: "gtk4 paintable queue", - })?; - - queue_app.link(&webrtc_sink).context(LinkSnafu { - from: "gtk4 paintable queue", - to: "gtk4 paintable", - })?; - - // -- END PAINTABLE SINK PIPELINE - - // -- BEGIN APPSINK PIPELINE - let tee_src_2 = tee - .request_pad_simple("src_%u") - .ok_or(PipelineError::PadRequest { - element: "tee pad 2".to_string(), - })?; - let appsink_queue_sinkpad = - appsink_queue - .static_pad("sink") - .ok_or(PipelineError::PadRequest { - element: "appsink queue".to_string(), - })?; - tee_src_2 - .link(&appsink_queue_sinkpad) - .context(PadLinkSnafu { - from: "tee src pad 2", - to: "appsink queue sinkpad", - })?; - - appsink_queue.link(&resize).context(LinkSnafu { - from: "appsink_queue", - to: "resize", - })?; - - let resize_caps = - gstreamer::caps::Caps::from_str("video/x-raw,format=RGB,width=640,height=640") - .context(BuildSnafu { - element: "resize_caps", - })?; - - resize - .link_filtered(&jpeg_enc, &resize_caps) - .context(LinkSnafu { - from: "jpeg_enc", - to: "resize_caps", - })?; - - Element::link_many([&jpeg_enc, &sink_frame.upcast_ref()]).context(LinkSnafu { - from: "jpeg_enc", - to: "appsink", - })?; - - Ok(WebcamPipeline { - pipeline, - sink_frame: Arc::new(Mutex::new(sink_frame)), - }) - } -} - -#[derive(Debug, Snafu)] -pub enum PipelineError { - #[snafu(display("Error during element linking"))] - Link { - source: BoolError, - from: String, - to: String, - }, - #[snafu(display("Error linking pads"))] - PadLink { - source: PadLinkError, - from: String, - to: String, - }, - #[snafu(display("Error creating element"))] - Build { source: BoolError, element: String }, - #[snafu(display("Error getting pad from element"))] - PadRequest { element: String }, -} +use gstreamer::{prelude::*, PadLinkError}; +use gstreamer::{Element, ElementFactory, Pipeline}; +use gstreamer_app::AppSink; +use gstreamer::glib::BoolError; +use snafu::prelude::*; +use std::str::FromStr; +use std::sync::Arc; +use tokio::sync::Mutex; + +#[derive(Debug)] +pub struct WebcamPipeline { + pub pipeline: Pipeline, + + pub sink_frame: Arc>, +} + +impl WebcamPipeline { + pub fn new(jpeg_quality: i32) -> Result { + let pipeline = Pipeline::with_name("webcam_pipeline"); + + // All of the following errors are unrecoverable + + let mut video_src = ""; + if cfg!(windows) { + video_src = "mfvideosrc"; + } else if cfg!(unix) { + video_src = "v4l2src"; + } + let source = ElementFactory::make(video_src) + .build() + .context(BuildSnafu { + element: "mfvideosrc", + })?; + let convert = ElementFactory::make("videoconvert") + .build() + .context(BuildSnafu { + element: "videoconvert", + })?; + let rate = ElementFactory::make("videorate") + .build() + .context(BuildSnafu { + element: "videorate", + })?; + + let tee = ElementFactory::make("tee") + .build() + .context(BuildSnafu { element: "tee" })?; + + let queue_app = ElementFactory::make("queue") + .property("max-size-time", 1u64) + .property("max-size-buffers", 0u32) + .property("max-size-bytes", 0u32) + .build() + .context(BuildSnafu { + element: "paintable queue", + })?; + + let webrtc_sink = ElementFactory::make("webrtcsink") + // .property("meta", "meta,name=gst-stream") + .name("web rtc sink") + .build() + .context(BuildSnafu { + element: "webrtcsink" + })?; + + // queue.connect_closure("overrun", false, glib::closure!(|queue: Element| { + // println!("The queue is full!"); + // })); + + let appsink_queue = ElementFactory::make("queue") + .property("max-size-time", 1u64) + .property("max-size-buffers", 0u32) + .property("max-size-bytes", 0u32) + .build() + .context(BuildSnafu { + element: "appsink queue", + })?; + + let resize = ElementFactory::make("videoscale") + .build() + .context(BuildSnafu { + element: "videoscale", + })?; + + let jpeg_enc = ElementFactory::make("jpegenc") + .property("quality", jpeg_quality) + .build() + .context(BuildSnafu { element: "jpegenc" })?; + + let caps_string = "image/jpeg,width=640,height=640"; + let appsrc_caps = gstreamer::Caps::from_str(caps_string).context(BuildSnafu { + element: "appsink caps", + })?; + + let sink_frame = AppSink::builder() + .name("frame_appsink") + .sync(false) + .max_buffers(1u32) + .drop(true) + .caps(&appsrc_caps) + .build(); + + sink_frame.set_property("caps", &appsrc_caps.to_value()); + + pipeline + .add_many([ + &source, + &convert, + &rate, + &tee, + &queue_app, + &webrtc_sink, + &appsink_queue, + &resize, + &jpeg_enc, + &sink_frame.upcast_ref(), + ]) + .context(LinkSnafu { + from: "all", + to: "pipeline", + })?; + + Element::link_many([&source, &convert, &rate]).context(LinkSnafu { + from: "source et. al.", + to: "rate", + })?; + + // -- BEGIN PAINTABLE SINK PIPELINE + let tee_caps = + // gstreamer::caps::Caps::from_str("video/x-raw,framerate=15/1").context(BuildSnafu { + gstreamer::caps::Caps::from_str("video/x-raw").context(BuildSnafu { + element: "tee caps", + })?; + + rate.link_filtered(&tee, &tee_caps).context(LinkSnafu { + from: "videorate", + to: "tee", + })?; + + let tee_src_1 = tee + .request_pad_simple("src_%u") + .ok_or(PipelineError::PadRequest { + element: "tee pad 1".to_string(), + })?; + let paintable_queue_sinkpad = + queue_app + .static_pad("sink") + .ok_or(PipelineError::PadRequest { + element: "gtk4 sink".to_string(), + })?; + + tee_src_1 + .link(&paintable_queue_sinkpad) + .context(PadLinkSnafu { + from: "tee src pad", + to: "gtk4 paintable queue", + })?; + + queue_app.link(&webrtc_sink).context(LinkSnafu { + from: "gtk4 paintable queue", + to: "gtk4 paintable", + })?; + + // -- END PAINTABLE SINK PIPELINE + + // -- BEGIN APPSINK PIPELINE + let tee_src_2 = tee + .request_pad_simple("src_%u") + .ok_or(PipelineError::PadRequest { + element: "tee pad 2".to_string(), + })?; + let appsink_queue_sinkpad = + appsink_queue + .static_pad("sink") + .ok_or(PipelineError::PadRequest { + element: "appsink queue".to_string(), + })?; + tee_src_2 + .link(&appsink_queue_sinkpad) + .context(PadLinkSnafu { + from: "tee src pad 2", + to: "appsink queue sinkpad", + })?; + + appsink_queue.link(&resize).context(LinkSnafu { + from: "appsink_queue", + to: "resize", + })?; + + let resize_caps = + gstreamer::caps::Caps::from_str("video/x-raw,format=RGB,width=640,height=640") + .context(BuildSnafu { + element: "resize_caps", + })?; + + resize + .link_filtered(&jpeg_enc, &resize_caps) + .context(LinkSnafu { + from: "jpeg_enc", + to: "resize_caps", + })?; + + Element::link_many([&jpeg_enc, &sink_frame.upcast_ref()]).context(LinkSnafu { + from: "jpeg_enc", + to: "appsink", + })?; + + Ok(WebcamPipeline { + pipeline, + sink_frame: Arc::new(Mutex::new(sink_frame)), + }) + } +} + +#[derive(Debug, Snafu)] +pub enum PipelineError { + #[snafu(display("Error during element linking"))] + Link { + source: BoolError, + from: String, + to: String, + }, + #[snafu(display("Error linking pads"))] + PadLink { + source: PadLinkError, + from: String, + to: String, + }, + #[snafu(display("Error creating element"))] + Build { source: BoolError, element: String }, + #[snafu(display("Error getting pad from element"))] + PadRequest { element: String }, +} diff --git a/src/main.rs b/src/main.rs index 5ad9d71..e5cd16b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,169 +1,187 @@ -// Prevents additional console window on Windows in release, DO NOT REMOVE!! -#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] - -use std::{sync::{Arc, Mutex}, time::Duration}; -use async_channel::Sender; -use tokio::{runtime, sync:: RwLock}; -use tracing::{self, debug, info, error}; -use tauri::{AppHandle, Manager, Window}; -use lazy_static::lazy_static; - -#[cfg(not(debug_assertions))] -use tracing_subscriber; -use vcs_common::ApplicationMessage; -use webrtc_remote::start_listener; - -use crate::config::{load_config, AppConfig}; - -mod config; -mod coordinator; -mod gstreamer_pipeline; -mod sources; -mod states; -mod tauri_functions; -mod webrtc_remote; - -use coordinator::{start_coordinator, ApplicationEvent}; - -lazy_static! { - static ref TO_MEC_REF: Mutex>> = Mutex::new(None); - static ref TO_WEBRTC: Mutex>> = Mutex::new(None); - static ref APP_HANDLE: Mutex> = Mutex::new(None); -} - -fn main() { - - #[cfg(not(debug_assertions))] - { - let file_appender = tracing_appender::rolling::daily(".\\logs", "camera-controller"); - let (non_blocking, _gaurd) = tracing_appender::non_blocking(file_appender); - tracing_subscriber::fmt() - .with_writer(non_blocking) - .with_max_level(tracing::Level::DEBUG) - .with_ansi(false) - .init(); - } - #[cfg(all(not(feature = "tokio-debug"), debug_assertions))] - { - let _sub = tracing_subscriber::fmt() - .with_max_level(tracing_subscriber::filter::LevelFilter::DEBUG) - .init(); - } - #[cfg(feature = "tokio-debug")] - { - console_subscriber::init(); - } - - let (to_mec, mec) = async_channel::bounded::(10); - - info!("Logging intialized"); - - let config: Arc> = Arc::new(RwLock::new(load_config())); - - gstreamer::init().expect("Unable to start gstreamer"); - - let rt = runtime::Runtime::new().expect("Could not start tokio runtime"); - let handle = rt.handle().clone(); - - let _coordinator = rt.handle().spawn(start_coordinator( - mec, - to_mec.clone(), - handle, - config - )); - - *TO_MEC_REF.lock().unwrap() = Some(to_mec.clone()); - - let (to_webrtc_send, to_webrtc_recv) = async_channel::bounded::(10); - let (from_webrtc_send, from_webrtc_recv) = async_channel::bounded::(10); - - rt.handle().spawn(start_listener(to_webrtc_recv, from_webrtc_send)); - - *TO_WEBRTC.lock().unwrap() = Some(to_webrtc_send.clone()); - - rt.handle().spawn(async move { - while let Ok(msg) = from_webrtc_recv.recv().await { - let mut do_sleep = false; - { - if let Ok(mut e) = APP_HANDLE.lock() { - if e.is_none() { - do_sleep = true; - } else { - let handle = e.take().unwrap(); - - match msg { - vcs_common::ApplicationMessage::WebRTCPacket(msg) => { - debug!("Got a message from the webrtc connection! {:?}", msg); - handle.emit_all("frontend_message", serde_json::to_string(&msg).unwrap()).unwrap(); - } - } - - *e = Some(handle); - } - } - - } - if do_sleep { - tokio::time::sleep(Duration::from_millis(500)).await; - } - - } - }); - - tauri::Builder::default() - .manage(tauri_functions::TauriState { to_mec: to_mec.clone(), to_webrtc: to_webrtc_send }) - .invoke_handler(tauri::generate_handler![tauri_functions::connect_to_camera]) - .setup(|app| { - *APP_HANDLE.lock().unwrap() = Some(app.handle()); - - let _id2 = app.listen_global("webrtc-message", |event| { - match event.payload() { - Some(payload) => { - if let Ok(e) = TO_WEBRTC.lock() { - match e.as_ref() { - Some(to_webrtc) => { - debug!("Sending message to the webrtc connection"); - if let Err(e) = to_webrtc.send_blocking(ApplicationMessage::WebRTCPacket(serde_json::from_str(payload).expect("Could not decode the browser's sdp"))) { - error!("Could not send to mec! {e}"); - } - }, - None => { - error!("TO_MEC_REF was none!"); - } - } - } - } - None => { - info!("There was an empty payload!"); - } - } - }); - - let _id = app.listen_global("webrtc-event", |event| { - match event.payload() { - Some(payload) => { - if let Ok(e) = TO_MEC_REF.lock() { - match e.as_ref() { - Some(to_mec) => { - if let Err(e) = to_mec.send_blocking(ApplicationEvent::WebRTCMessage(payload.to_string())) { - error!("Could not send to mec! {e}"); - } - }, - None => { - error!("TO_MEC_REF was none!"); - } - } - } - } - None => { - info!("There was an empty payload!"); - } - } - }); - Ok(()) - }) - .run(tauri::generate_context!()) - .expect("error while running tauri application"); - - let _ = to_mec.send_blocking(ApplicationEvent::Close); -} +// Prevents additional console window on Windows in release, DO NOT REMOVE!! +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] + +use std::{sync::{Arc, Mutex}, time::Duration}; +use async_channel::Sender; +use tokio::{runtime, sync:: RwLock}; +use tracing::{self, debug, info, error}; +use tauri::{AppHandle, Manager, Window}; +use lazy_static::lazy_static; + +#[cfg(not(debug_assertions))] +use tracing_subscriber; +use vcs_common::ApplicationMessage; +use webrtc_remote::start_listener; + +use crate::config::{load_config, AppConfig}; + +mod config; +mod coordinator; +mod gstreamer_pipeline; +mod sources; +mod states; +mod tauri_functions; +mod webrtc_remote; + +use coordinator::{start_coordinator, ApplicationEvent}; + +lazy_static! { + static ref TO_MEC_REF: Mutex>> = Mutex::new(None); + static ref TO_WEBRTC: Mutex>> = Mutex::new(None); + static ref APP_HANDLE: Mutex> = Mutex::new(None); +} + +fn main() { + + #[cfg(not(debug_assertions))] + { + let file_appender = tracing_appender::rolling::daily(".\\logs", "camera-controller"); + let (non_blocking, _gaurd) = tracing_appender::non_blocking(file_appender); + tracing_subscriber::fmt() + .with_writer(non_blocking) + .with_max_level(tracing::Level::DEBUG) + .with_ansi(false) + .init(); + } + #[cfg(all(not(feature = "tokio-debug"), debug_assertions))] + { + let _sub = tracing_subscriber::fmt() + .with_max_level(tracing_subscriber::filter::LevelFilter::DEBUG) + .init(); + } + #[cfg(feature = "tokio-debug")] + { + console_subscriber::init(); + } + + let (to_mec, mec) = async_channel::bounded::(10); + + info!("Logging intialized"); + + let config: Arc> = Arc::new(RwLock::new(load_config())); + + gstreamer::init().expect("Unable to start gstreamer"); + + let rt = runtime::Runtime::new().expect("Could not start tokio runtime"); + let handle = rt.handle().clone(); + + let _coordinator = rt.handle().spawn(start_coordinator( + mec, + to_mec.clone(), + handle, + config + )); + + *TO_MEC_REF.lock().unwrap() = Some(to_mec.clone()); + + let (to_webrtc_send, to_webrtc_recv) = async_channel::bounded::(10); + let (from_webrtc_send, from_webrtc_recv) = async_channel::bounded::(10); + + rt.handle().spawn(start_listener(to_webrtc_recv, from_webrtc_send)); + + *TO_WEBRTC.lock().unwrap() = Some(to_webrtc_send.clone()); + + rt.handle().spawn(async move { + while let Ok(msg) = from_webrtc_recv.recv().await { + let mut do_sleep = false; + { + if let Ok(mut e) = APP_HANDLE.lock() { + if e.is_none() { + do_sleep = true; + } else { + let handle = e.take().unwrap(); + + match msg { + vcs_common::ApplicationMessage::WebRTCPacket(msg) => { + debug!("Got a message from the webrtc connection! {:?}", msg); + handle.emit_all("frontend_message", serde_json::to_string(&msg).unwrap()).unwrap(); + } + vcs_common::ApplicationMessage::WebRTCIceCandidateInit(msg) => { + debug!("Got an ICE init candidate from the webrtc connection! {:?}", msg); + handle.emit_all("frontend_message", serde_json::to_string(&msg).unwrap()).unwrap(); + } + vcs_common::ApplicationMessage::WebRTCIceCandidate(msg) => { + + debug!("Got an ICE candidate from the webrtc connection! {:?}", msg); + handle.emit_all("frontend_message", serde_json::to_string(&msg).unwrap()).unwrap(); + } + } + + *e = Some(handle); + } + } + + } + if do_sleep { + tokio::time::sleep(Duration::from_millis(500)).await; + } + + } + }); + + tauri::Builder::default() + .manage(tauri_functions::TauriState { to_mec: to_mec.clone() }) + .invoke_handler(tauri::generate_handler![tauri_functions::connect_to_camera]) + .setup(|app| { + *APP_HANDLE.lock().unwrap() = Some(app.handle()); + + let _id2 = app.listen_global("webrtc-message", |event| { + debug!("Got webrtc-message event from Tauri client! {:#?}", event); + match event.payload() { + Some(payload) => { + if let Ok(e) = TO_WEBRTC.lock() { + match e.as_ref() { + Some(to_webrtc) => { + debug!("Sending message to the webrtc connection"); + let message: Option = match payload { + s if s.starts_with("{\"type") => Some(ApplicationMessage::WebRTCPacket(serde_json::from_str(payload).expect("Could not decode the browser's sdp"))), + s if s.starts_with("{\"candidate") => Some(ApplicationMessage::WebRTCIceCandidateInit(serde_json::from_str(payload).expect("Could not decode the browser's Ice Candidate"))), + _ => None + }; + + if message.is_some() { + if let Err(e) = to_webrtc.send_blocking(message.unwrap()) { + error!("Could not send to mec! {e}"); + } + } + }, + None => { + error!("TO_MEC_REF was none!"); + } + } + } + } + None => { + info!("There was an empty payload!"); + } + } + }); + + // let _id = app.listen_global("webrtc-event", |event| { + // match event.payload() { + // Some(payload) => { + // if let Ok(e) = TO_MEC_REF.lock() { + // match e.as_ref() { + // Some(to_mec) => { + // if let Err(e) = to_mec.send_blocking(ApplicationEvent::WebRTCMessage(payload.to_string())) { + // error!("Could not send to mec! {e}"); + // } + // }, + // None => { + // error!("TO_MEC_REF was none!"); + // } + // } + // } + // } + // None => { + // info!("There was an empty payload!"); + // } + // } + // }); + Ok(()) + }) + .run(tauri::generate_context!()) + .expect("error while running tauri application"); + + let _ = to_mec.send_blocking(ApplicationEvent::Close); +} diff --git a/src/tauri_functions.rs b/src/tauri_functions.rs index e3128d6..309520d 100644 --- a/src/tauri_functions.rs +++ b/src/tauri_functions.rs @@ -1,17 +1,14 @@ - -use std::sync::Arc; - -use async_channel::Sender; -use tauri::State; - -use crate::coordinator::ApplicationEvent; - -pub struct TauriState { - pub to_mec: Sender, - pub to_webrtc: Sender, -} - -#[tauri::command] -pub fn connect_to_camera(state: State<'_, TauriState>) { - let _ = state.to_mec.send_blocking(ApplicationEvent::CameraConnectionPress); -} + +use async_channel::Sender; +use tauri::State; + +use crate::coordinator::ApplicationEvent; + +pub struct TauriState { + pub to_mec: Sender, +} + +#[tauri::command] +pub fn connect_to_camera(state: State<'_, TauriState>) { + let _ = state.to_mec.send_blocking(ApplicationEvent::CameraConnectionPress); +} diff --git a/src/webrtc_remote.rs b/src/webrtc_remote.rs index ff4f2cf..e4bbc1f 100644 --- a/src/webrtc_remote.rs +++ b/src/webrtc_remote.rs @@ -1,132 +1,132 @@ -use std::{cmp::Ordering, sync::{atomic::AtomicBool, Arc}}; - -use futures_util::{StreamExt, SinkExt}; -use tokio::{net::TcpListener, sync::Mutex}; -use tokio_tungstenite::{accept_async, tungstenite::Message}; -use tracing::{error, info, instrument, debug}; - -use vcs_common::{AppReceiver, AppSender, ApplicationMessage}; - - - -#[instrument(skip_all)] -pub async fn start_listener( - from_app: AppReceiver, - to_ui: AppSender, -) { - - info!("starting tcplistener"); - let listener = TcpListener::bind("localhost:7891").await.unwrap(); - - info!("Listening for webrtc connections on localhost:7891"); - - if let Ok((stream, _)) = listener.accept().await { - match accept_async(stream).await { - Err(e) => error!("Could not convert incoming stream to websocket: {e}"), - Ok(ws_stream) => { - let (mut ws_sender, mut ws_receiver) = ws_stream.split(); - - tokio::spawn(async move { - while let Ok(msg) = from_app.recv().await { - #[cfg(debug_assertions)] - { - // serialized message - match serde_json::to_string(&msg) { - Err(e) => error!("Could not serialize ApplicationMessage to JSON! {e}"), - Ok(msg) => { - if let Err(e) = ws_sender.send(Message::text(msg)).await { - error!("Could not send text ApplicationMessage to websocket! Closing websocket\n{e}"); - break; - } - } - } - - } - - #[cfg(not(debug_assertions))] - { - match bincode::serialize(&msg) { - Err(e) => error!("Could not serialize ApplicationMessage into binary! {e}"), - Ok(e) => { - if let Err(e) = sender.send(Message::binary(msg)).await { - error!("Could not send binary ApplicationMessage to websocket! Closing websocket\n{e}"); - break; - } - } - } - } - } - }); - - while let Some(msg) = ws_receiver.next().await { - match msg { - Err(e) => { - error!("There was an error getting a message from the remote! {e}"); - } - Ok(msg) => match msg { - Message::Ping(_) | Message::Pong(_) => {} - Message::Close(_) => { - info!("Received WebSocket close message! Closing the websocket"); - break; - } - Message::Frame(_) => { - info!("Received a Frame websocket message?"); - } - Message::Text(text) => { - debug!("Recieved text from websocket: {text}"); - #[cfg(debug_assertions)] - { - match serde_json::from_str(&text) { - Ok(msg) => { - if let Err(e) = to_ui.send(msg).await { - error!("Could not send message from ws to application! Closing and exiting\n{e}"); - break; - } - } - Err(e) => { - error!("Received a malformed JSON message from the websocket!\n{text}\nmsg: {e}"); - } - } - - } - #[cfg(not(debug_assertions))] - { - warn!("Recieved a `Text` message from the remote while running in release mode! " + - "Was the other endpoint running release mode?\n msg: {text}"); - } - } - Message::Binary(msg) => { - #[cfg(debug_assertions)] - { - match bincode::deserialize::(&msg) { - Ok(m) => { - if let Err(e) = to_ui.send(m).await { - error!("Could not send message to application! Closing and exiting\n{e}"); - break; - } - } - Err(e) => { - error!("Received a malformed binary message from the websocket!\n{e}"); - } - - } - } - - #[cfg(not(debug_assertions))] - { - warn!("Recieved a `Binary` message from the remote while running in debug mode! " + - "Was the other endpoing running debug mode?"); - } - - } - } - } - - } - - - } - } - - } -} +use std::{cmp::Ordering, sync::{atomic::AtomicBool, Arc}}; + +use futures_util::{StreamExt, SinkExt}; +use tokio::{net::TcpListener, sync::Mutex}; +use tokio_tungstenite::{accept_async, tungstenite::Message}; +use tracing::{error, info, instrument, debug}; + +use vcs_common::{AppReceiver, AppSender, ApplicationMessage}; + + + +#[instrument(skip_all)] +pub async fn start_listener( + from_app: AppReceiver, + to_ui: AppSender, +) { + + info!("starting tcplistener"); + let listener = TcpListener::bind("localhost:7891").await.unwrap(); + + info!("Listening for webrtc connections on localhost:7891"); + + if let Ok((stream, _)) = listener.accept().await { + match accept_async(stream).await { + Err(e) => error!("Could not convert incoming stream to websocket: {e}"), + Ok(ws_stream) => { + let (mut ws_sender, mut ws_receiver) = ws_stream.split(); + + tokio::spawn(async move { + while let Ok(msg) = from_app.recv().await { + #[cfg(debug_assertions)] + { + // serialized message + match serde_json::to_string(&msg) { + Err(e) => error!("Could not serialize ApplicationMessage to JSON! {e}"), + Ok(msg) => { + if let Err(e) = ws_sender.send(Message::text(msg)).await { + error!("Could not send text ApplicationMessage to websocket! Closing websocket\n{e}"); + break; + } + } + } + + } + + #[cfg(not(debug_assertions))] + { + match bincode::serialize(&msg) { + Err(e) => error!("Could not serialize ApplicationMessage into binary! {e}"), + Ok(e) => { + if let Err(e) = sender.send(Message::binary(msg)).await { + error!("Could not send binary ApplicationMessage to websocket! Closing websocket\n{e}"); + break; + } + } + } + } + } + }); + + while let Some(msg) = ws_receiver.next().await { + match msg { + Err(e) => { + error!("There was an error getting a message from the remote! {e}"); + } + Ok(msg) => match msg { + Message::Ping(_) | Message::Pong(_) => {} + Message::Close(_) => { + info!("Received WebSocket close message! Closing the websocket"); + break; + } + Message::Frame(_) => { + info!("Received a Frame websocket message?"); + } + Message::Text(text) => { + debug!("Recieved text from websocket: {text}"); + #[cfg(debug_assertions)] + { + match serde_json::from_str(&text) { + Ok(msg) => { + if let Err(e) = to_ui.send(msg).await { + error!("Could not send message from ws to application! Closing and exiting\n{e}"); + break; + } + } + Err(e) => { + error!("Received a malformed JSON message from the websocket!\n{text}\nmsg: {e}"); + } + } + + } + #[cfg(not(debug_assertions))] + { + warn!("Recieved a `Text` message from the remote while running in release mode! " + + "Was the other endpoint running release mode?\n msg: {text}"); + } + } + Message::Binary(msg) => { + #[cfg(debug_assertions)] + { + match bincode::deserialize::(&msg) { + Ok(m) => { + if let Err(e) = to_ui.send(m).await { + error!("Could not send message to application! Closing and exiting\n{e}"); + break; + } + } + Err(e) => { + error!("Received a malformed binary message from the websocket!\n{e}"); + } + + } + } + + #[cfg(not(debug_assertions))] + { + warn!("Recieved a `Binary` message from the remote while running in debug mode! " + + "Was the other endpoing running debug mode?"); + } + + } + } + } + + } + + + } + } + + } +} diff --git a/ui/index.html b/ui/index.html index 86c211d..4be4db7 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1,53 +1,52 @@ - - - - - - - - - - -
- -
- - - - -
-
- - - -
- -
- - - - + + + + + + + + + + +
+ +
+ + + + +
+
+ + + +
+ +
+ + + + diff --git a/ui/static/index.js b/ui/static/index.js index 5b9b1b4..4b09e5c 100644 --- a/ui/static/index.js +++ b/ui/static/index.js @@ -1,21 +1,21 @@ -const { invoke } = window.__TAURI__.tauri; -import { rtc_init } from "./rtc.js"; - -function call_camera_connect() { - invoke("connect_to_camera", {}) - .then(() => { - let cam_button = document.getElementById("camera_connect_button"); - cam_button.innerText = "Connecting to Camera"; - cam_button.classList.remove("text-semibold"); - cam_button.classList.remove("text-white"); - cam_button.classList.add("text-neutral-400") - }) - .catch((e) => console.error(e)); -} - -async function init() { - console.log("Setting up"); - await rtc_init(); -} - -export { init }; +const { invoke } = window.__TAURI__.tauri; +import { rtc_init } from "./rtc.js"; + +function call_camera_connect() { + invoke("connect_to_camera", {}) + .then(() => { + let cam_button = document.getElementById("camera_connect_button"); + cam_button.innerText = "Connecting to Camera"; + cam_button.classList.remove("text-semibold"); + cam_button.classList.remove("text-white"); + cam_button.classList.add("text-neutral-400") + }) + .catch((e) => console.error(e)); +} + +async function init() { + console.log("Setting up"); + await rtc_init(); +} + +export { init }; diff --git a/ui/static/rtc.js b/ui/static/rtc.js index 4ccdb59..ee764e0 100644 --- a/ui/static/rtc.js +++ b/ui/static/rtc.js @@ -1,90 +1,101 @@ - - -import { event } from "./@tauri-apps/api/index.js"; - -async function rtc_init() { - console.log("ding ding!"); - console.log(event); - const videoview = document.getElementById("remoteview"); - - const config = { - iceServers: [{ urls: "stun:localhost" }] - }; - const polite = true; - - const pc = new RTCPeerConnection(config); - - pc.ontrack = ({ track, streams }) => { - track.onunmute = () => { - if (remoteview.srcObject) { - return; - } - remoteview.srcObject = streams[0]; - }; - }; - - let makingOffer = false; - - pc.onnegotionationneeded = async () => { - try { - makingOffer = true; - await pc.setLocalDescription(); - event.emit("webrtc-message", { description: pc.localDescription }); - } catch (err) { - console.error(err); - } finally { - makingOffer = false; - } - }; - - pc.onicecandidate = ({ candidate }) => event.emit("webrtc-message", { candidate }); - pc.oniceconnectionstatechange = () => { - if (pc.iceConnectionState === "failed") { - pc.restartIce(); - } - }; - - let ignoreOffer = false; - - console.log("registering listner"); - const application_message = await event.listen('frontend_message', async (event) => { - console.log("Event: "); - console.log(event); - - const { description, candidate } = event.payload.data; - - try { - if (description) { - const offerCollision = - description.type === "offer" && - (makingOffer || pc.signalingState !== "stable"); - - ignoreOffer = !polite && offerCollision; - - if (ignoreOffer) { - return; - } - - await pc.setRemoteDescription(description); - if (description.type === "offer") { - await pc.setLocalDescription(); - event.emit( "webrtc-message", { description: pc.localDescription }); - } - } else if (candidate) { - try { - await pc.addIceCandidate(candidate); - } catch (err) { - if (!ignoreOffer) { - throw err; - } - } - } - } catch (err) { - console.error(err); - } - - }); -} - - -export { rtc_init }; + + +import { event } from "./@tauri-apps/api/index.js"; + +async function rtc_init() { + const videoview = document.getElementById("remoteview"); + + const config = { + iceServers: [{ urls: "stun:localhost" }] + }; + const polite = true; + + const pc = new RTCPeerConnection(config); + window.pc = pc; + + pc.ontrack = (e) => { + console.log(e); + e.track.onunmute = () => { + console.log("Unmuted?"); + if (remoteview.srcObject) { + console.log("Skipping srcobject"); + return; + } + videoview.srcObject = e.streams[0]; + Object.assign(videoview.style, { "background-color": "black"}); + }; + }; + + let makingOffer = false; + + pc.onnegotionationneeded = async () => { + try { + makingOffer = true; + await pc.setLocalDescription(); + console.log("emitting response webrtc packet"); + event.emit("webrtc-message", pc.localDescription ); + } catch (err) { + console.error(err); + } finally { + makingOffer = false; + } + }; + + pc.onicecandidate = ({ candidate }) => { + console.log("emitting response webrtc packet"); + event.emit("webrtc-message", candidate); + }; + pc.oniceconnectionstatechange = () => { + console.log('ICE state: ',pc.iceConnectionState); + if (pc.iceConnectionState === "failed") { + pc.restartIce(); + } + }; + + let ignoreOffer = false; + + console.log("registering listener"); + const application_message = await event.listen('frontend_message', async (msg) => { + + const payload = JSON.parse(msg.payload); + + console.log(payload); + + try { + if (payload.type) { + const offerCollision = + payload.type === "offer" && + (makingOffer || pc.signalingState !== "stable"); + + ignoreOffer = !polite && offerCollision; + + if (ignoreOffer) { + return; + } + + await pc.setRemoteDescription(payload); + if (payload.type === "offer") { + console.log("Settings local description"); + await pc.setLocalDescription(); + console.log("emitting response webrtc packet"); + event.emit( "webrtc-message", pc.localDescription); + } + } else if (payload.candidate) { + try { + console.log("Adding trickle ICE candidate"); + await pc.addIceCandidate(payload.candidate); + } catch (err) { + if (!ignoreOffer) { + throw err; + } + } + } + } catch (err) { + console.error(err); + } + + }); +} + + +export { rtc_init };