From 506ebde9f5a96aef440b98ffe214446827ba381c Mon Sep 17 00:00:00 2001 From: Nickiel12 <35903114+Nickiel12@users.noreply.github.com> Date: Fri, 26 Apr 2024 22:11:37 -0700 Subject: [PATCH] working pixels over windows pipes --- Cargo.lock | 570 +++++++++++++++++++++ Cargo.toml | 3 + src/{coordinator.rs => coordinator/mod.rs} | 35 +- src/{ui => }/gstreamer_pipeline.rs | 94 ++-- src/main.rs | 1 + src/remote_sources/mod.rs | 1 + src/remote_sources/shared_video_pipe.rs | 97 ++++ src/ui/mod.rs | 33 +- src/ui/shared_video_pipe.rs | 49 -- 9 files changed, 768 insertions(+), 115 deletions(-) rename src/{coordinator.rs => coordinator/mod.rs} (87%) rename src/{ui => }/gstreamer_pipeline.rs (50%) create mode 100644 src/remote_sources/shared_video_pipe.rs delete mode 100644 src/ui/shared_video_pipe.rs diff --git a/Cargo.lock b/Cargo.lock index 7757143..c6ee962 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aligned-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -38,6 +44,29 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "async-channel" version = "2.2.0" @@ -74,6 +103,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "av1-grain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876c75a42f6364451a033496a14c44bffe41f5f4a8236f697391f11024e596d2" +dependencies = [ + "arrayvec", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -95,6 +147,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -110,6 +168,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitstream-io" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c9989a51171e2e81038ab168b6ae22886fe9ded214430dbb4f41c28cf176da" + [[package]] name = "block-buffer" version = "0.10.4" @@ -119,18 +183,36 @@ dependencies = [ "generic-array", ] +[[package]] +name = "built" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41bfbdb21256b87a8b5e80fab81a8eed158178e812fd7ba451907518b2742f16" + [[package]] name = "bumpalo" version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +[[package]] +name = "bytemuck" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "0.4.12" @@ -176,6 +258,10 @@ name = "cc" version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +dependencies = [ + "jobserver", + "libc", +] [[package]] name = "cfg-expr" @@ -211,6 +297,12 @@ dependencies = [ "windows-targets 0.52.4", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "concurrent-queue" version = "2.4.0" @@ -294,6 +386,34 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.19" @@ -397,6 +517,31 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "887d93f60543e9a9362ef8a21beedd0a833c5d9610e18c67abe15a5963dcb1a4" +dependencies = [ + "bit_field", + "flume", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fdeflate" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f9bfee30e4dedf0ab8b422f03af778d9612b63f502710fc500a334ebe2de645" +dependencies = [ + "simd-adler32", +] + [[package]] name = "field-offset" version = "0.3.6" @@ -407,6 +552,25 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flate2" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4556222738635b7a3417ae6130d8f52201e45a0c4d1a907f0826383adb5f85e7" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -602,6 +766,16 @@ dependencies = [ "wasi", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gilrs" version = "0.10.6" @@ -836,6 +1010,34 @@ dependencies = [ "thiserror", ] +[[package]] +name = "gstreamer-app" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50184e88d3462a796a5924fb329839c102b22f9383c1636323fa4ef5255dea92" +dependencies = [ + "futures-core", + "futures-sink", + "glib", + "gstreamer", + "gstreamer-app-sys", + "gstreamer-base", + "libc", +] + +[[package]] +name = "gstreamer-app-sys" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6771c0939f286fb261525494a0aad29435b37e802284756bab24afe3bbca7476" +dependencies = [ + "glib-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + [[package]] name = "gstreamer-base" version = "0.22.0" @@ -990,6 +1192,16 @@ dependencies = [ "system-deps", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.13.2" @@ -1064,6 +1276,45 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd54d660e773627692c524beaad361aca785a4f9f5730ce91f42aabe5bce3d11" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "exr", + "gif", + "image-webp", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d730b085583c4d789dfd07fdcf185be59501666a90c97c40162b37e4fdad272d" +dependencies = [ + "byteorder-lite", + "thiserror", +] + +[[package]] +name = "imgref" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126" + [[package]] name = "indexmap" version = "2.2.5" @@ -1094,6 +1345,17 @@ dependencies = [ "libc", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + [[package]] name = "interprocess" version = "2.0.0" @@ -1141,6 +1403,15 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + [[package]] name = "joystick-controller-client" version = "1.1.0" @@ -1153,7 +1424,10 @@ dependencies = [ "gilrs", "gst-plugin-gtk4", "gstreamer", + "gstreamer-app", + "gstreamer-video", "gtk4", + "image", "interprocess", "log", "serde", @@ -1163,6 +1437,12 @@ dependencies = [ "toml", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.69" @@ -1189,12 +1469,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "libc" version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libfuzzer-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96cfd5557eb82f2b83fed4955246c988d331975a002961b07c81584d107e7f7" +dependencies = [ + "arbitrary", + "cc", + "once_cell", +] + [[package]] name = "libudev-sys" version = "0.1.4" @@ -1211,12 +1508,31 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "mach2" version = "0.4.2" @@ -1226,6 +1542,16 @@ dependencies = [ "libc", ] +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "memchr" version = "2.7.1" @@ -1254,6 +1580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", + "simd-adler32", ] [[package]] @@ -1273,6 +1600,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "956787520e75e9bd233246045d19f42fb73242759cc57fba9611d940ae96d4b0" +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nix" version = "0.28.0" @@ -1295,12 +1628,40 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1317,6 +1678,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ "autocfg", + "num-bigint", "num-integer", "num-traits", ] @@ -1494,6 +1856,19 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "png" +version = "0.17.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e4b0d3d1312775e782c86c91a111aa1f910cbb65e1337f9975b5f9a554b5e1" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1548,6 +1923,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d84d1d7a6ac92673717f9f6d1518374ef257669c24ebc5ac25d5033828be58" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd" +dependencies = [ + "quote", + "syn 2.0.53", +] + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.35" @@ -1587,12 +1996,91 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand", + "rand_chacha", + "simd_helpers", + "system-deps", + "thiserror", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc13288f5ab39e6d7c9d501759712e6969fcc9734220846fc9ed26cae2cc4234" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "recvmsg" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" +[[package]] +name = "rgb" +version = "0.8.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8" +dependencies = [ + "bytemuck", +] + [[package]] name = "ron" version = "0.8.1" @@ -1642,6 +2130,12 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "semver" version = "1.0.22" @@ -1710,6 +2204,21 @@ dependencies = [ "digest", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simplelog" version = "0.12.2" @@ -1746,6 +2255,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "syn" version = "1.0.109" @@ -1828,6 +2346,17 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.34" @@ -2056,6 +2585,17 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +[[package]] +name = "v_frame" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "vec_map" version = "0.8.2" @@ -2144,6 +2684,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "widestring" version = "1.1.0" @@ -2377,3 +2923,27 @@ checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" dependencies = [ "linked-hash-map", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec866b44a2a1fd6133d363f073ca1b179f438f99e7e5bfb1e33f7181facfe448" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index cb31335..f53c28f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,6 @@ tokio-tungstenite = "0.21.0" toml = "0.8.12" # interprocess = { version = "1.2.1", features = ["tokio_support"] } interprocess = { version = "2.0.0", git = "https://github.com/kotauskas/interprocess.git", features = ["tokio"] } +gstreamer-app = { version = "0.22.0", features = ["v1_22"] } +gstreamer-video = { version = "0.22.4", features = ["v1_22"] } +image = "0.25.1" diff --git a/src/coordinator.rs b/src/coordinator/mod.rs similarity index 87% rename from src/coordinator.rs rename to src/coordinator/mod.rs index 6756f97..43359a5 100644 --- a/src/coordinator.rs +++ b/src/coordinator/mod.rs @@ -11,12 +11,14 @@ use futures_util::{ stream::{SplitSink, SplitStream}, SinkExt, StreamExt, }; +use gstreamer::State; +use gstreamer::prelude::ElementExt; use log::{error, info}; use tokio::net::TcpStream; use tokio::runtime::Handle; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; -use crate::remote_sources; +use crate::{gstreamer_pipeline, remote_sources}; use crate::ui::BoxCoords; use crate::{joystick_source::joystick_loop, ui::GuiUpdate}; @@ -55,6 +57,9 @@ struct CoordState<'a> { pub to_mec: Sender, pub to_gui: Sender, pub rt: Handle, + + pub pipeline: gstreamer_pipeline::WebcamPipeline, + pub keep_windows_pipe_alive: Arc, } impl<'a> CoordState<'a> { @@ -65,7 +70,7 @@ impl<'a> CoordState<'a> { rt: Handle, identity_boxes: Arc>>, ) -> Self { - CoordState { + let this = CoordState { sck_outbound: None, sck_alive_recvr: Arc::new(AtomicBool::new(false)), sck_alive_server: Arc::new(AtomicBool::new(false)), @@ -79,7 +84,16 @@ impl<'a> CoordState<'a> { to_mec, to_gui, rt, - } + + pipeline: gstreamer_pipeline::WebcamPipeline::new(), + keep_windows_pipe_alive: Arc::new(AtomicBool::new(true)), + }; + this.rt.spawn(crate::remote_sources::shared_video_pipe::create_outbound_pipe( + this.pipeline.sink_frame.clone(), + this.to_mec.clone(), + this.keep_windows_pipe_alive.clone(), + )); + this } pub async fn socket_send(&mut self, message: Message) { @@ -160,6 +174,7 @@ impl<'a> CoordState<'a> { pub async fn close(&mut self) { info!("closing coord state"); + self.keep_windows_pipe_alive.store(false, Ordering::SeqCst); self.socket_close().await; self.joystick_loop_alive.store(false, Ordering::SeqCst); @@ -183,6 +198,15 @@ pub async fn start_coordinator( let mut state = CoordState::new(mec, to_mec, to_gui, runtime, identity_boxes); + state.pipeline + .pipeline + .set_state(State::Playing) + .expect("Could not set pipeline state to playing"); + + if let Err(e) = state.to_gui.send(GuiUpdate::UpdatePaintable(state.pipeline.sink_paintable.clone())).await { + error!("Could not send new paintable to GUI: {e}"); + } + state.check_states().await; while let Some(msg) = state.mec.next().await { @@ -229,6 +253,11 @@ pub async fn start_coordinator( } } + state.pipeline + .pipeline + .set_state(State::Null) + .expect("Could not set pipeline state to playing"); + info!("Stopping Coordinator"); } diff --git a/src/ui/gstreamer_pipeline.rs b/src/gstreamer_pipeline.rs similarity index 50% rename from src/ui/gstreamer_pipeline.rs rename to src/gstreamer_pipeline.rs index c492df4..d5bc0ac 100644 --- a/src/ui/gstreamer_pipeline.rs +++ b/src/gstreamer_pipeline.rs @@ -1,18 +1,21 @@ -use gstreamer::prelude::{ElementExt, ElementExtManual, GstBinExtManual, PadExt}; -use gstreamer::{Element, ElementFactory, Pipeline, State}; +use gstreamer::prelude::*; +use gstreamer::{Element, ElementFactory, Pipeline}; +use gstreamer_app::AppSink; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; -#[derive(Clone)] pub struct WebcamPipeline { pub pipeline: Pipeline, pub src: Element, pub converter: Element, pub tee: Element, + pub queue_app: Element, pub sink_paintable: Element, + pub queue: Element, pub resize: Element, - pub decode: Element, - pub sink_frame: Element, + pub sink_frame: Arc>, } impl WebcamPipeline { @@ -30,30 +33,56 @@ impl WebcamPipeline { .build() .expect("Could not create tee element"); + let queue_app = ElementFactory::make("queue") + .build() + .expect("Could not create the queue buffer"); let sink_paintable = ElementFactory::make("gtk4paintablesink") + .name("gtk4_output") .build() .expect("Could not build gtk sink for GStreamer"); + let queue = ElementFactory::make("queue") + .build() + .expect("Could not create the queue buffer"); + let resize = ElementFactory::make("videoscale") .build() .expect("Could not build videoscale for GStreamer"); - todo!("Video XRaw is not actually working yet"); - let decode = ElementFactory::make("video/x-raw,width=650,height=480") - .build() - .expect("Could not create decoder"); + let caps_string = String::from("video/x-raw,format=RGB,width=640,height=480,max-buffers=1,drop=true"); + // let caps_string = String::from("video/x-raw,format=RGB,max-buffers=1,drop=true"); + let appsrc_caps = gstreamer::Caps::from_str(&caps_string).expect("Couldn't create appsrc caps"); - let sink_frame = ElementFactory::make("appsink name=frame_sink") - .build() - .expect("Could not build appsrc for GStreamer"); + /* + // let sink_frame = ElementFactory::make("appsink") + // .name("frame_output") + // .property("sync", &false) + // .build() + // .expect("Could not build appsrc for GStreamer"); + let video_info = gstreamer_video::VideoInfo::builder(gstreamer_video::VideoFormat::Rgb, 650, 480) + .build() + .expect("Couldn't build video info!"); + */ + + let sink_frame = AppSink::builder() + .name("frame_output") + .sync(false) + .max_buffers(1u32) + .drop(true) + .caps(&appsrc_caps) + .build(); + + sink_frame.set_property("caps", &appsrc_caps.to_value()); pipeline.add_many(&[ &source, &convert, &tee, + &queue_app, &sink_paintable, &resize, - &sink_frame, + &queue, + &sink_frame.upcast_ref(), ]).expect("Could not link the elements to the pipeline"); source @@ -61,54 +90,47 @@ impl WebcamPipeline { .expect("Could not link video source to converter"); convert.link(&tee).expect("Could not link converter to tee"); - let tee_pad_template = tee - .pad_template("src_%u") - .expect("Could not get pad templates"); - let tee_src_1 = tee - .request_pad(&tee_pad_template, Some("paintable_src_pad"), None) + .request_pad_simple("src_%u") .expect("Could not create src pad 1"); - let sink_paintable_sinkpad = sink_paintable + let sink_paintable_sinkpad = queue_app .static_pad("sink") - .expect("Could not get sink pad for paintablesink"); + .expect("Could not get sink pad for paintablesink queue"); tee_src_1 .link(&sink_paintable_sinkpad) .expect("Could not link tee srcpad 1 to paintablesink pad"); + queue_app + .link(&sink_paintable) + .expect("Could not link app queue to paintable sink"); let tee_src_2 = tee - .request_pad(&tee_pad_template, Some("output_src_pad"), None) + .request_pad_simple("src_%u") .expect("Could not create src pad 2"); - let sink_frameoutput_sinkpad = sink_frame + let sink_frameoutput_sinkpad = queue .static_pad("sink") .expect("Could not get sink pad for frameoutput sink"); tee_src_2 .link(&sink_frameoutput_sinkpad) .expect("Could not link tee srcpad 2 to frame output sink pad"); + queue + .link(&resize) + .expect("Could not link queue to resize"); resize - .link(&decode) - .expect("Could not bind resize to decoder"); - decode .link(&sink_frame) - .expect("Could not bind decoder to webcam frame output"); + .expect("Could not bind resize to appsrc"); + WebcamPipeline { pipeline, src: source, converter: convert, tee, + queue_app, sink_paintable, resize, - decode, - sink_frame, + queue, + sink_frame: Arc::new(Mutex::new(sink_frame)), } } } - -impl Drop for WebcamPipeline { - fn drop(&mut self) { - self.pipeline - .set_state(State::Null) - .expect("Could not close pipeline during window deconstruction"); - } -} diff --git a/src/main.rs b/src/main.rs index 7f5e060..95c6861 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ mod config; mod coordinator; mod joystick_source; mod remote_sources; +mod gstreamer_pipeline; mod ui; const APP_ID: &str = "net.nickiel.joystick-controller-client"; diff --git a/src/remote_sources/mod.rs b/src/remote_sources/mod.rs index 851b0ce..ff3eb39 100644 --- a/src/remote_sources/mod.rs +++ b/src/remote_sources/mod.rs @@ -22,6 +22,7 @@ use tokio_tungstenite::{ mod automated_source; mod remote_source; +pub mod shared_video_pipe; use crate::{ coordinator::{ApplicationEvent, ConnectionType}, diff --git a/src/remote_sources/shared_video_pipe.rs b/src/remote_sources/shared_video_pipe.rs new file mode 100644 index 0000000..603b8d0 --- /dev/null +++ b/src/remote_sources/shared_video_pipe.rs @@ -0,0 +1,97 @@ +use std::sync::{atomic::AtomicBool, Arc, Mutex}; + +use async_channel::Sender; +use gstreamer_app::AppSink; +use gstreamer_video::VideoFrameExt; +use interprocess::os::windows::named_pipe::{ + pipe_mode, + tokio::{DuplexPipeStream, PipeStream}, +}; +use log::{error, info}; +use tokio::io::AsyncWriteExt; + +use crate::coordinator::ApplicationEvent; + +pub async fn create_outbound_pipe(appsink: Arc>, to_mec: Sender, keep_alive: Arc) { + if let Ok(mut pipe) = + DuplexPipeStream::::connect_by_path(r"\\.\pipe\example_pipe").await + { + let video_info = gstreamer_video::VideoInfo::builder(gstreamer_video::VideoFormat::Rgb, 640, 480) + .build() + .expect("Couldn't build video info!"); + + loop { + if !keep_alive.load(std::sync::atomic::Ordering::SeqCst) { + break; + } + + let sample = match appsink.lock() { + Ok(sink) => { + match sink.pull_sample() { + Ok(e) => e, + Err(e) => { + error!("Could not pull appsink sample: {e}"); + break; + } + } + } + Err(e) => { + error!("Could not get a lock on the appsink: {e}"); + break; + } + }; + let buffer = sample.buffer_owned().unwrap(); + let video_frame = match gstreamer_video::VideoFrame::from_buffer_readable(buffer, &video_info) { + Ok(e) => e, + Err(_) => { + error!("Unable to make video frame from buffer!"); + break; + } + }; + + info!("Video frame {}x{} with stride of {}, is this many bytes: {}", video_frame.width(), video_frame.height(), video_frame.plane_stride()[0], video_frame.plane_data(0).unwrap().len()); + + + if let Err(e) = send_to_pipe(&mut pipe, video_frame.plane_data(0).unwrap()).await { + error!("Error in sending to the pipe: {e}"); + break; + } + + + } + + if let Err(e) = pipe.shutdown().await { + error!("Couldn't shut down pipe: {e}"); + } + // send pipeline images as bytes + // let bus = pipeline.get_bus().expect("Pipeline has no bus"); + // let mut frame_count = 0; + + + // while let Some(msg) = bus.timed_pop(gstreamer::ClockTime::from_seconds(1)) { + // use gstreamer::MessageView; + // match msg.view() { + // MessageView::Element(element) => { + // if let Some(buffer) = element.structure().get::("buffer") { + // frame_count += 1; + // println!("Writing frame {}", frame_count); + // let data = buffer.map_readable().expect("Failed to map buffer"); + // // Send the data to the broadcast channel + // tx.send(data.as_slice()).expect("Failed to send data"); + // } + // } + // _ => (), + // } + // } + } +} + +async fn send_to_pipe<'a>( + pipe: &mut PipeStream, + message: &'a [u8] +) -> Result<(), Box> { + pipe.write_all(message).await?; + // pipe.shutdown().await?; + + Ok(()) +} diff --git a/src/ui/mod.rs b/src/ui/mod.rs index ac508b5..b46fb50 100644 --- a/src/ui/mod.rs +++ b/src/ui/mod.rs @@ -1,6 +1,5 @@ use std::sync::{Arc, Mutex}; -use gstreamer::{prelude::ElementExt, State}; use gtk::cairo::Context; use gtk::gdk::Paintable; use gtk::{glib, prelude::*, Box, Entry, Label, ListBox}; @@ -10,8 +9,6 @@ use serde::{Deserialize, Serialize}; use tokio::runtime::Handle; use tokio_tungstenite::tungstenite::Message; -mod gstreamer_pipeline; -mod shared_video_pipe; use crate::config::{load_config, save_config}; use crate::coordinator::{start_coordinator, ApplicationEvent, MoveEvent}; @@ -33,6 +30,7 @@ impl Default for AppState { pub enum GuiUpdate { SocketState(bool), MoveEvent(MoveEvent), + UpdatePaintable(gstreamer::Element), } pub struct BoxCoords { @@ -124,24 +122,6 @@ pub fn build_ui(app: &Application, runtime: Handle) { let overlay_box = gtk::Overlay::new(); main_box.append(&overlay_box); - let pipeline = gstreamer_pipeline::WebcamPipeline::new(); - - pipeline - .pipeline - .set_state(State::Playing) - .expect("Could not set pipeline state to playing"); - - let pipeline2 = pipeline.pipeline.clone(); - window.connect_close_request(move |_| { - pipeline2 - .set_state(State::Null) - .expect("Could not close pipeline during window deconstruction"); - glib::Propagation::Proceed - }); - - let paintable = pipeline.sink_paintable.property::("paintable"); - webcam_picture.set_paintable(Some(&paintable)); - let drawable = gtk::DrawingArea::builder() .content_height(480) .content_width(640) @@ -153,12 +133,6 @@ pub fn build_ui(app: &Application, runtime: Handle) { overlay_box.set_child(Some(&webcam_picture)); overlay_box.add_overlay(&drawable); - let pipeline3 = pipeline.pipeline.clone(); - runtime.spawn(shared_video_pipe::create_outbound_pipe( - pipeline3, - to_mec.clone(), - )); - // Connect to "clicked" signal of `button` button.connect_clicked(glib::clone!(@weak ip_entry, @weak port_entry, @strong to_mec => move |_button| { let ip_text = ip_entry.text(); @@ -223,6 +197,11 @@ pub fn build_ui(app: &Application, runtime: Handle) { button.set_css_classes(&["NoConnection"]); } } + GuiUpdate::UpdatePaintable(sink) => { + let paintable = sink.property::("paintable"); + + webcam_picture.set_paintable(Some(&paintable)); + } } } }), diff --git a/src/ui/shared_video_pipe.rs b/src/ui/shared_video_pipe.rs deleted file mode 100644 index efe45d4..0000000 --- a/src/ui/shared_video_pipe.rs +++ /dev/null @@ -1,49 +0,0 @@ -use async_channel::Sender; -use gstreamer::{prelude::ElementExt, Pipeline}; -use interprocess::os::windows::named_pipe::{ - pipe_mode, - tokio::{DuplexPipeStream, PipeStream}, -}; -use log::error; -use tokio::{io::AsyncWriteExt, runtime::Handle, sync::broadcast}; - -use crate::coordinator::ApplicationEvent; - -pub async fn create_outbound_pipe(pipeline: Pipeline, to_mec: Sender) { - if let Ok(pipe) = - DuplexPipeStream::::connect_by_path(r"\\.\pipe\example_pipe").await - { - // send pipeline images as bytes - // let bus = pipeline.get_bus().expect("Pipeline has no bus"); - // let mut frame_count = 0; - - if let Err(e) = send_to_pipe(pipe).await { - error!("Error in sending to the pipe: {e}"); - } - - // while let Some(msg) = bus.timed_pop(gstreamer::ClockTime::from_seconds(1)) { - // use gstreamer::MessageView; - // match msg.view() { - // MessageView::Element(element) => { - // if let Some(buffer) = element.structure().get::("buffer") { - // frame_count += 1; - // println!("Writing frame {}", frame_count); - // let data = buffer.map_readable().expect("Failed to map buffer"); - // // Send the data to the broadcast channel - // tx.send(data.as_slice()).expect("Failed to send data"); - // } - // } - // _ => (), - // } - // } - } -} - -async fn send_to_pipe( - mut pipe: PipeStream, -) -> Result<(), Box> { - pipe.write_all(b"Hello World").await?; - pipe.shutdown().await?; - - Ok(()) -}