Live stream a gstreamer UDP video pipeline to a react application

Question

I have a camera which I access through a gstreamer pipeline. The pipeline sends the udp video to a local port 5000.

A nodejs server picks up the udp, handles offer/awnser and ice candidate, and video data with sockets.

A react application connects to the socket, handles the video data and uses mediasource api to play the live video.

The video is not showing in the web app and I don't have any apparant errors. The front end receives the video data as ArrayBuffer objects I use Chrome as browser and the video format and mime type seems to be correct.

Below is the code I am using :

Gstreamer pipeline I run in my terminal(mac) : gst-launch-1.0 -v avfvideosrc ! videoconvert ! x264enc tune=zerolatency ! h264parse ! queue ! rtph264pay ! udpsink host=127.0.0.1 port=5000

Note that I am using the computers webcam for the moment, the equivalent pipeline for pc would be : gst-launch-1.0 -v mfvideosrc ! videoconvert ! x264enc tune=zerolatency ! h264parse ! queue ! rtph264pay ! udpsink host=127.0.0.1 port=5000

My signaling server (server.js) :

const express = require("express");
const http = require("http");
const socketIO = require("socket.io");
const dgram = require("dgram");
const cors = require("cors");

const app = express();
const server = http.createServer(app);
const udpServer = dgram.createSocket("udp4");



// Use the cors middleware to allow cross-origin requests for HTTP requests
app.use(
  cors({
    origin: "http://localhost:3001", // Allow requests only from React app's URL
    methods: ["GET", "POST"],
  })
);

const io = socketIO(server, {
  cors: {
    origin: "http://localhost:3001", // Allow WebSocket connections only from React app's URL
    methods: ["GET", "POST"],
  },
});

udpServer.on("message", (message, remote) => {
    console.log("Received video data:", message.length, "bytes");
    io.emit("video-data", message);
  });

udpServer.bind(5000); // port 5000 where gstreamer is sending UDP


io.on("connection", (socket) => {
  console.log("A user connected");

  socket.on("offer", (offer) => {
    // Broadcast the offer to all other connected clients
    console.log("offer:", offer);
    socket.broadcast.emit("offer", offer);
  });

  socket.on("answer", (answer) => {
    // Broadcast the answer to all other connected clients
    console.log("answer:", answer);
    socket.broadcast.emit("answer", answer);
  });

  socket.on("ice-candidate", (candidate) => {
    // Broadcast the ICE candidate to all other connected clients
    console.log("ice-candidate", candidate);
    socket.broadcast.emit("ice-candidate", candidate);
  });

  socket.on("disconnect", () => {
    console.log("A user disconnected");
  });

  socket.on("error", (error) => {
    console.error("WebSocket error:", error);
  });
});

server.listen(3000, () => {
  console.log("Signaling server is running on port 3000");
});

My react application :

import React, { useEffect, useRef } from "react";
import io from "socket.io-client";

const App = () => {
  const videoRef = useRef();
  const socket = io.connect("http://localhost:3000");

  const mediaSource = useRef(null);
  const sourceBuffer = useRef(null);

  useEffect(() => {
    const videoElement = videoRef.current;

    mediaSource.current = new MediaSource();

    mediaSource.current.addEventListener("sourceopen", () => {
      sourceBuffer.current = mediaSource.current.addSourceBuffer("video/mp4; codecs=avc1.42E01E");

      sourceBuffer.current.addEventListener("updateend", () => {
        if (mediaSource.current.readyState === "open") {
          mediaSource.current.endOfStream();
        }
      });

      videoElement.srcObject = mediaSource.current;
      videoElement.play().catch((error) => {
        console.error("Autoplay error:", error);
      });
    });

    mediaSource.current.addEventListener("error", (e) => {
      console.error("MediaSource error:", e);
    });

    socket.on("video-data", (videoData) => {
      if (mediaSource.current.readyState === "open") {
        if (!sourceBuffer.current) {
          sourceBuffer.current = mediaSource.current.addSourceBuffer("video/mp4; codecs=avc1.42E01E");
          sourceBuffer.current.addEventListener("updateend", () => {
            if (mediaSource.current.readyState === "open") {
              mediaSource.current.endOfStream();
            }
          });
        }

        sourceBuffer.current.appendBuffer(new Uint8Array(videoData));
      }
    });

    return () => {
      if (mediaSource.current.readyState === "open") {
        mediaSource.current.endOfStream();
      }
    };
  }, []);

  return (
    <div>
      <h1>Video Stream</h1>
      <video ref={videoRef} playsInline />
    </div>
  );
};

export default App;