VCC/vcs-controller
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

not-quite-mvp for gstreamer tauri rtc

Browse source
This commit is contained in:
Nickiel12 2024-08-01 02:42:30 +00:00
parent 6488402700
commit b54626ea84
7 changed files with 2581 additions and 337 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2607
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

15
src/gstreamer_pipeline.rs
View file

@ -11,8 +11,6 @@ use tokio::sync::Mutex;
pub struct WebcamPipeline { pub struct WebcamPipeline {
pub pipeline: Pipeline, pub pipeline: Pipeline,
pub sink_paintable: Element,
pub sink_frame: Arc<Mutex<AppSink>>, pub sink_frame: Arc<Mutex<AppSink>>,
} }
@ -56,11 +54,13 @@ impl WebcamPipeline {
.context(BuildSnafu { .context(BuildSnafu {
element: "paintable queue", element: "paintable queue",
})?; })?;
let sink_paintable = ElementFactory::make("gtk4paintablesink")
.name("gtk4_output") let webrtc_sink = ElementFactory::make("webrtcsink")
.property("meta", "meta,name=gst-stream")
.name("web rtc sink")
.build() .build()
.context(BuildSnafu { .context(BuildSnafu {
element: "gtkpaintablesink", element: "webrtcsink"
})?; })?;
// queue.connect_closure("overrun", false, glib::closure!(|queue: Element| { // queue.connect_closure("overrun", false, glib::closure!(|queue: Element| {
@ -109,7 +109,7 @@ impl WebcamPipeline {
&rate, &rate,
&tee, &tee,
&queue_app, &queue_app,
&sink_paintable, &webrtc_sink,
&appsink_queue, &appsink_queue,
&resize, &resize,
&jpeg_enc, &jpeg_enc,
@ -156,7 +156,7 @@ impl WebcamPipeline {
to: "gtk4 paintable queue", to: "gtk4 paintable queue",
})?; })?;
queue_app.link(&sink_paintable).context(LinkSnafu { queue_app.link(&webrtc_sink).context(LinkSnafu {
from: "gtk4 paintable queue", from: "gtk4 paintable queue",
to: "gtk4 paintable", to: "gtk4 paintable",
})?; })?;
@ -207,7 +207,6 @@ impl WebcamPipeline {
Ok(WebcamPipeline { Ok(WebcamPipeline {
pipeline, pipeline,
sink_paintable,
sink_frame: Arc::new(Mutex::new(sink_frame)), sink_frame: Arc::new(Mutex::new(sink_frame)),
}) })
} }

5
ui/index.html
View file

@ -1,8 +1,10 @@
<!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<link href="/static/main.css" rel="stylesheet"/> <link href="/static/main.css" rel="stylesheet"/>
<script src="/static/feather.min.js"></script> <script src="/static/feather.min.js"></script>
<script src="/static/index.js"></script> <script src="/static/index.js"></script>
<script src="/static/gstwebrtc-api-2-0-0.js"></script>
</head> </head>
<body> <body>
@ -36,6 +38,8 @@
</div> </div>
<div class="bg-emerald-700 h-dvh w-7/8"> <div class="bg-emerald-700 h-dvh w-7/8">
<video style="width: 320px; height: 240px;" id="remoteview"></video>
<video style="width: 320px; height: 240px;" id="capture"></video>
</div> </div>
@ -43,6 +47,7 @@
<script> <script>
feather.replace(); feather.replace();
</script> </script>
<script src="/static/gstreamer-rtc.js"></script>
</body> </body>
</html> </html>

206
ui/static/gstreamer-rtc.js Normal file
View file

@ -0,0 +1,206 @@
function initCapture(api) {
const captureSection = document.getElementById("capture");
const clientIdElement = captureSection.querySelector(".client-id");
const videoElement = captureSection.getElementsByTagName("video")[0];
const listener = {
connected: function(clientId) { clientIdElement.textContent = clientId; },
disconnected: function() { clientIdElement.textContent = "none"; }
};
api.registerConnectionListener(listener);
document.getElementById("capture-button").addEventListener("click", (event) => {
event.preventDefault();
if (captureSection._producerSession) {
captureSection._producerSession.close();
} else if (!captureSection.classList.contains("starting")) {
captureSection.classList.add("starting");
const constraints = {
video: { width: 1280, height: 720 }
};
navigator.mediaDevices.getUserMedia(constraints).then((stream) => {
const session = api.createProducerSession(stream);
if (session) {
captureSection._producerSession = session;
session.addEventListener("error", (event) => {
if (captureSection._producerSession === session) {
console.error(event.message, event.error);
}
});
session.addEventListener("closed", () => {
if (captureSection._producerSession === session) {
videoElement.pause();
videoElement.srcObject = null;
captureSection.classList.remove("has-session", "starting");
delete captureSection._producerSession;
}
});
session.addEventListener("stateChanged", (event) => {
if ((captureSection._producerSession === session) &&
(event.target.state === GstWebRTCAPI.SessionState.streaming)) {
videoElement.srcObject = stream;
videoElement.play().catch(() => {});
captureSection.classList.remove("starting");
}
});
session.addEventListener("clientConsumerAdded", (event) => {
if (captureSection._producerSession === session) {
console.info(`client consumer added: ${event.detail.peerId}`);
}
});
session.addEventListener("clientConsumerRemoved", (event) => {
if (captureSection._producerSession === session) {
console.info(`client consumer removed: ${event.detail.peerId}`);
}
});
captureSection.classList.add("has-session");
session.start();
} else {
for (const track of stream.getTracks()) {
track.stop();
}
captureSection.classList.remove("starting");
}
}).catch((error) => {
console.error("cannot have access to webcam and microphone", error);
captureSection.classList.remove("starting");
});
}
});
}
function initRemoteStreams(api) {
const remoteStreamsElement = document.getElementById("remote-streams");
const listener = {
producerAdded: function(producer) {
const producerId = producer.id
if (!document.getElementById(producerId)) {
remoteStreamsElement.insertAdjacentHTML("beforeend",
`<li id="${producerId}">
<div class="button">${producer.meta.name || producerId}</div>
<div class="video">
<div class="spinner">
<div></div>
<div></div>
<div></div>
<div></div>
</div>
<span class="remote-control">&#xA9;</span>
<video></video>
<div class="fullscreen"><span title="Toggle fullscreen">&#x25A2;</span></div>
</div>
</li>`);
const entryElement = document.getElementById(producerId);
const videoElement = entryElement.getElementsByTagName("video")[0];
videoElement.addEventListener("playing", () => {
if (entryElement.classList.contains("has-session")) {
entryElement.classList.add("streaming");
}
});
entryElement.addEventListener("click", (event) => {
event.preventDefault();
if (!event.target.classList.contains("button")) {
return;
}
if (entryElement._consumerSession) {
entryElement._consumerSession.close();
} else {
const session = api.createConsumerSession(producerId);
if (session) {
entryElement._consumerSession = session;
session.addEventListener("error", (event) => {
if (entryElement._consumerSession === session) {
console.error(event.message, event.error);
}
});
session.addEventListener("closed", () => {
if (entryElement._consumerSession === session) {
videoElement.pause();
videoElement.srcObject = null;
entryElement.classList.remove("has-session", "streaming", "has-remote-control");
delete entryElement._consumerSession;
}
});
session.addEventListener("streamsChanged", () => {
if (entryElement._consumerSession === session) {
const streams = session.streams;
if (streams.length > 0) {
videoElement.srcObject = streams[0];
videoElement.play().catch(() => {});
}
}
});
session.addEventListener("remoteControllerChanged", () => {
if (entryElement._consumerSession === session) {
const remoteController = session.remoteController;
if (remoteController) {
entryElement.classList.add("has-remote-control");
remoteController.attachVideoElement(videoElement);
} else {
entryElement.classList.remove("has-remote-control");
}
}
});
entryElement.classList.add("has-session");
session.connect();
}
}
});
}
},
producerRemoved: function(producer) {
const element = document.getElementById(producer.id);
if (element) {
if (element._consumerSession) {
element._consumerSession.close();
}
element.remove();
}
}
};
api.registerProducersListener(listener);
for (const producer of api.getAvailableProducers()) {
listener.producerAdded(producer);
}
}
window.addEventListener("DOMContentLoaded", () => {
document.addEventListener("click", (event) => {
if (event.target.matches("div.video>div.fullscreen:hover>span")) {
event.preventDefault();
event.target.parentNode.previousElementSibling.requestFullscreen();
}
});
const signalingProtocol = window.location.protocol.startsWith("https") ? "wss" : "ws";
const gstWebRTCConfig = {
meta: { name: `WebClient-${Date.now()}` },
signalingServerUrl: `ws://127.0.0.1:8443`,
};
const api = new GstWebRTCAPI(gstWebRTCConfig);
// initCapture(api);
initRemoteStreams(api);
});

5
ui/static/gstwebrtc-api-2-0-0.js Normal file
View file

File diff suppressed because one or more lines are too long

1
ui/static/gstwebrtc-api-2-0-0.js.map Normal file
View file

File diff suppressed because one or more lines are too long

79
ui/static/rtc.js Normal file
View file

@ -0,0 +1,79 @@
const videoview = document.getElementById("remoteview");
const config = {
iceServers: [{ urls: "stun:localhost" }]
};
const polite = true;
const signaler = new SignalingChannel();
const pc = new RTCPeerConnection(config);
const constraints = { audio: false, video: true };
pc.ontrack = ({ track, streams }) => {
track.onunmute = () => {
if (remoteview.srcObject) {
return;
}
remoteview.srcObject = streams[0];
};
};
let makingOffer = false;
pc.onnegotionationneeded = async () => {
try {
makingOffer = true;
await pc.setLocalDescription();
signaler.send({ description: pc.localDescription });
} catch (err) {
console.error(err);
} finally {
makingOffer = false;
}
};
pc.onicecandidate = ({ candidate }) => signaler.send({ candidate });
pc.oniceconnectionstatechange = () => {
if (pc.iceConnectionState === "failed") {
pc.restartIce();
}
};
let ignoreOffer = false;
signaler.onmessage = async ({ data: { description, candidate } }) => {
try {
if (description) {
const offerCollision =
description.type === "offer" &&
(makingOffer || pc.signalingState !== "stable");
ignoreOffer = !polite && offerCollision;
if (ignoreOffer) {
return;
}
await pc.setRemoteDescription(description);
if (description.type === "offer") {
await pc.setLocalDescription();
signaler.send({ description: pc.localDescription });
}
} else if (candidate) {
try {
await pc.addIceCandidate(candidate);
} catch (err) {
if (!ignoreOffer) {
throw err;
}
}
}
} catch (err) {
console.error(err);
}
};
async function start_rtc_connection() {
}