openvidu-live-captions: remove bug with participantInfo on transcriptions

ai-services/openvidu-live-captions/README.md

@@ -0,0 +1,5 @@
+# OpenVidu Captions
+
+This is the basic JavaScript tutorial extended to show live captions. It uses OpenVidu AI Services to generate captions from the audio Tracks of Participants in the Rooms.
+
+Visit [Live Captions tutorial](https://openvidu.io/latest/docs/tutorials/ai-services/openvidu-live-captions/)

ai-services/openvidu-live-captions/src/app.js

@@ -0,0 +1,221 @@
+// When running OpenVidu locally, leave these variables empty
+// For other deployment type, configure them with correct URLs depending on your deployment
+var APPLICATION_SERVER_URL = "";
+var LIVEKIT_URL = "";
+configureUrls();
+
+const LivekitClient = window.LivekitClient;
+var room;
+
+function configureUrls() {
+  // If APPLICATION_SERVER_URL is not configured, use default value from OpenVidu Local deployment
+  if (!APPLICATION_SERVER_URL) {
+    if (window.location.hostname === "localhost") {
+      APPLICATION_SERVER_URL = "http://localhost:6080/";
+    } else {
+      APPLICATION_SERVER_URL = "https://" + window.location.hostname + ":6443/";
+    }
+  }
+
+  // If LIVEKIT_URL is not configured, use default value from OpenVidu Local deployment
+  if (!LIVEKIT_URL) {
+    if (window.location.hostname === "localhost") {
+      LIVEKIT_URL = "ws://localhost:7880/";
+    } else {
+      LIVEKIT_URL = "wss://" + window.location.hostname + ":7443/";
+    }
+  }
+}
+
+async function joinRoom() {
+  // Disable 'Join' button
+  document.getElementById("join-button").disabled = true;
+  document.getElementById("join-button").innerText = "Joining...";
+
+  // Initialize a new Room object
+  room = new LivekitClient.Room();
+
+  // Specify the actions when events take place in the room
+  // On every new Track received...
+  room.on(
+    LivekitClient.RoomEvent.TrackSubscribed,
+    (track, _publication, participant) => {
+      addTrack(track, participant.identity);
+    }
+  );
+
+  // On every new Track destroyed...
+  room.on(
+    LivekitClient.RoomEvent.TrackUnsubscribed,
+    (track, _publication, participant) => {
+      track.detach();
+      document.getElementById(track.sid)?.remove();
+
+      if (track.kind === "video") {
+        removeVideoContainer(participant.identity);
+      }
+    }
+  );
+
+  room.registerTextStreamHandler("lk.transcription", async (reader, participantInfo) => {
+      const message = await reader.readAll();
+      const isFinal = reader.info.attributes["lk.transcription_final"] === "true";
+      const trackId = reader.info.attributes["lk.transcribed_track_id"];
+
+      if (isFinal) {
+        const speaker = participantInfo.identity == room.localParticipant.identity
+            ? "You" : participantInfo.identity;
+        const timestamp = new Date().toLocaleTimeString();
+        const captionsTextarea = document.getElementById("captions");
+        captionsTextarea.value += `[${timestamp}] ${speaker}: ${message}\n`;
+        captionsTextarea.scrollTop = captionsTextarea.scrollHeight;
+      }
+    }
+  );
+
+  try {
+    // Get the room name and participant name from the form
+    const roomName = document.getElementById("room-name").value;
+    const userName = document.getElementById("participant-name").value;
+
+    // Get a token from your application server with the room name and participant name
+    const token = await getToken(roomName, userName);
+
+    // Connect to the room with the LiveKit URL and the token
+    await room.connect(LIVEKIT_URL, token);
+
+    // Hide the 'Join room' page and show the 'Room' page
+    document.getElementById("room-title").innerText = roomName;
+    document.getElementById("join").hidden = true;
+    document.getElementById("room").hidden = false;
+
+    // Publish your camera and microphone
+    await room.localParticipant.enableCameraAndMicrophone();
+    const localVideoTrack = this.room.localParticipant.videoTrackPublications
+      .values()
+      .next().value.track;
+    addTrack(localVideoTrack, userName, true);
+  } catch (error) {
+    console.log("There was an error connecting to the room:", error.message);
+    await leaveRoom();
+  }
+}
+
+function addTrack(track, participantIdentity, local = false) {
+  const element = track.attach();
+  element.id = track.sid;
+
+  /* If the track is a video track, we create a container and append the video element to it 
+    with the participant's identity */
+  if (track.kind === "video") {
+    const videoContainer = createVideoContainer(participantIdentity, local);
+    videoContainer.append(element);
+    appendParticipantData(
+      videoContainer,
+      participantIdentity + (local ? " (You)" : "")
+    );
+  } else {
+    document.getElementById("layout-container").append(element);
+  }
+}
+
+async function leaveRoom() {
+  // Leave the room by calling 'disconnect' method over the Room object
+  await room.disconnect();
+
+  // Remove all HTML elements inside the layout container
+  removeAllLayoutElements();
+
+  // Clear the captions textarea
+  document.getElementById("captions").value = "";
+
+  // Back to 'Join room' page
+  document.getElementById("join").hidden = false;
+  document.getElementById("room").hidden = true;
+
+  // Enable 'Join' button
+  document.getElementById("join-button").disabled = false;
+  document.getElementById("join-button").innerText = "Join!";
+}
+
+window.onbeforeunload = () => {
+  room?.disconnect();
+};
+
+window.onload = generateFormValues;
+
+function generateFormValues() {
+  document.getElementById("room-name").value = "Test Room";
+  document.getElementById("participant-name").value =
+    "Participant" + Math.floor(Math.random() * 100);
+}
+
+function createVideoContainer(participantIdentity, local = false) {
+  const videoContainer = document.createElement("div");
+  videoContainer.id = `camera-${participantIdentity}`;
+  videoContainer.className = "video-container";
+  const layoutContainer = document.getElementById("layout-container");
+
+  if (local) {
+    layoutContainer.prepend(videoContainer);
+  } else {
+    layoutContainer.append(videoContainer);
+  }
+
+  return videoContainer;
+}
+
+function appendParticipantData(videoContainer, participantIdentity) {
+  const dataElement = document.createElement("div");
+  dataElement.className = "participant-data";
+  dataElement.innerHTML = `<p>${participantIdentity}</p>`;
+  videoContainer.prepend(dataElement);
+}
+
+function removeVideoContainer(participantIdentity) {
+  const videoContainer = document.getElementById(
+    `camera-${participantIdentity}`
+  );
+  videoContainer?.remove();
+}
+
+function removeAllLayoutElements() {
+  const layoutElements = document.getElementById("layout-container").children;
+  Array.from(layoutElements).forEach((element) => {
+    element.remove();
+  });
+}
+
+/**
+ * --------------------------------------------
+ * GETTING A TOKEN FROM YOUR APPLICATION SERVER
+ * --------------------------------------------
+ * The method below request the creation of a token to
+ * your application server. This prevents the need to expose
+ * your LiveKit API key and secret to the client side.
+ *
+ * In this sample code, there is no user control at all. Anybody could
+ * access your application server endpoints. In a real production
+ * environment, your application server must identify the user to allow
+ * access to the endpoints.
+ */
+async function getToken(roomName, participantName) {
+  const response = await fetch(APPLICATION_SERVER_URL + "token", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      roomName,
+      participantName,
+    }),
+  });
+
+  if (!response.ok) {
+    const error = await response.json();
+    throw new Error(`Failed to get token: ${error.errorMessage}`);
+  }
+
+  const token = await response.json();
+  return token.token;
+}

ai-services/openvidu-live-captions/src/index.html

@@ -0,0 +1,94 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <title>OpenVidu Live Captions</title>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1" />
+        <link rel="shortcut icon" href="resources/images/favicon.ico" type="image/x-icon" />
+
+        <!-- Bootstrap -->
+        <link
+            href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css"
+            rel="stylesheet"
+            integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH"
+            crossorigin="anonymous"
+        />
+        <script
+            src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"
+            integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz"
+            crossorigin="anonymous"
+        ></script>
+
+        <!-- Font Awesome -->
+        <link
+            rel="stylesheet"
+            href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.2/css/all.min.css"
+            integrity="sha512-SnH5WK+bZxgPHs44uWIX+LLJAJ9/2PkPKZ5QiAj6Ta86w+fsb2TkcmfRyVX3pBnMFcV7oQPJkl9QevSCWr3W6A=="
+            crossorigin="anonymous"
+            referrerpolicy="no-referrer"
+        />
+
+        <link rel="stylesheet" href="styles.css" type="text/css" media="screen" />
+        <script src="https://cdn.jsdelivr.net/npm/livekit-client@2.14.0/dist/livekit-client.umd.js"></script>
+        <script src="app.js"></script>
+    </head>
+
+    <body>
+        <header>
+            <a href="/" title="Home"><h1>OpenVidu Live Captions</h1></a>
+            <div id="links">
+                <a
+                    href="https://github.com/OpenVidu/openvidu-livekit-tutorials/tree/master/application-client/openvidu-js"
+                    title="GitHub Repository"
+                    target="_blank"
+                >
+                    <i class="fa-brands fa-github"></i>
+                </a>
+                <a
+                    href="https://livekit-tutorials.openvidu.io/tutorials/application-client/javascript/"
+                    title="Documentation"
+                    target="_blank"
+                >
+                    <i class="fa-solid fa-book"></i>
+                </a>
+            </div>
+        </header>
+
+        <main>
+            <div id="join">
+                <div id="join-dialog">
+                    <h2>Join a Video Room</h2>
+                    <form onsubmit="joinRoom(); return false">
+                        <div>
+                            <label for="participant-name">Participant</label>
+                            <input id="participant-name" class="form-control" type="text" required />
+                        </div>
+                        <div>
+                            <label for="room-name">Room</label>
+                            <input id="room-name" class="form-control" type="text" required />
+                        </div>
+                        <button id="join-button" class="btn btn-lg btn-success" type="submit">Join!</button>
+                    </form>
+                </div>
+            </div>
+
+            <div id="room" hidden>
+                <div id="room-header">
+                    <h2 id="room-title"></h2>
+                    <button class="btn btn-danger" id="leave-room-button" onclick="leaveRoom()">Leave Room</button>
+                </div>
+                <div id="layout-container"></div>
+                <div id="captions-container">
+                    <textarea id="captions" class="form-control" rows="3"></textarea>
+                </div>
+            </div>
+        </main>
+
+        <footer>
+            <p class="text">Made with love by <span>OpenVidu Team</span></p>
+            <a href="http://www.openvidu.io/" target="_blank">
+                <img id="openvidu-logo" src="resources/images/openvidu_logo.png" alt="OpenVidu logo" />
+            </a>
+        </footer>
+    </body>
+</html>

ai-services/openvidu-live-captions/src/styles.css

@@ -0,0 +1,283 @@
+html {
+    height: 100%;
+}
+
+body {
+    margin: 0;
+    padding: 0;
+    padding-top: 50px;
+    display: flex;
+    flex-direction: column;
+    height: 100%;
+}
+
+header {
+    height: 50px;
+    width: 100%;
+    position: fixed;
+    top: 0;
+    left: 0;
+    z-index: 1;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 10px 30px;
+    background-color: #4d4d4d;
+}
+
+header h1 {
+    margin: 0;
+    font-size: 1.5em;
+    font-weight: bold;
+}
+
+header a {
+    color: #ccc;
+    text-decoration: none;
+}
+
+header a:hover {
+    color: #a9a9a9;
+}
+
+header i {
+    padding: 5px 5px;
+    font-size: 2em;
+}
+
+main {
+    flex: 1;
+    padding: 20px;
+}
+
+#join {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+    height: 100%;
+}
+
+#join-dialog {
+    width: 70%;
+    max-width: 900px;
+    padding: 60px;
+    border-radius: 6px;
+    background-color: #f0f0f0;
+}
+
+#join-dialog h2 {
+    color: #4d4d4d;
+    font-size: 60px;
+    font-weight: bold;
+    text-align: center;
+}
+
+#join-dialog form {
+    text-align: left;
+}
+
+#join-dialog label {
+    display: block;
+    margin-bottom: 10px;
+    color: #0088aa;
+    font-weight: bold;
+    font-size: 20px;
+}
+
+.form-control {
+    width: 100%;
+    padding: 8px;
+    margin-bottom: 10px;
+    box-sizing: border-box;
+    color: #0088aa;
+    font-weight: bold;
+}
+
+.form-control:focus {
+    color: #0088aa;
+    border-color: #0088aa;
+    -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(0, 136, 170, 0.6);
+    box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(0, 136, 170, 0.6);
+}
+
+#join-dialog button {
+    display: block;
+    margin: 20px auto 0;
+}
+
+.btn {
+    font-weight: bold;
+}
+
+.btn-success {
+    background-color: #06d362;
+    border-color: #06d362;
+}
+
+.btn-success:hover {
+    background-color: #1abd61;
+    border-color: #1abd61;
+}
+
+#room {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+}
+
+#room-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    width: 100%;
+    max-width: 1000px;
+    padding: 0 20px;
+    margin-bottom: 20px;
+}
+
+#room-title {
+    font-size: 2em;
+    font-weight: bold;
+    margin: 0;
+}
+
+#layout-container {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+    gap: 10px;
+    justify-content: center;
+    align-items: center;
+    width: 100%;
+    max-width: 1000px;
+    height: 100%;
+}
+
+.video-container {
+    position: relative;
+    background: #3b3b3b;
+    aspect-ratio: 16/9;
+    border-radius: 6px;
+    overflow: hidden;
+}
+
+.video-container video {
+    width: 100%;
+    height: 100%;
+}
+
+.video-container .participant-data {
+    position: absolute;
+    top: 0;
+    left: 0;
+}
+
+.participant-data p {
+    background: #f8f8f8;
+    margin: 0;
+    padding: 0 5px;
+    color: #777777;
+    font-weight: bold;
+    border-bottom-right-radius: 4px;
+}
+
+footer {
+    height: 60px;
+    width: 100%;
+    padding: 10px 30px;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    background-color: #4d4d4d;
+}
+
+footer a {
+    color: #ffffff;
+    text-decoration: none;
+}
+
+footer .text {
+    color: #ccc;
+    margin: 0;
+}
+
+footer .text span {
+    color: white;
+    font-weight: bold;
+}
+
+#openvidu-logo {
+    height: 35px;
+    -webkit-transition: all 0.1s ease-in-out;
+    -moz-transition: all 0.1s ease-in-out;
+    -o-transition: all 0.1s ease-in-out;
+    transition: all 0.1s ease-in-out;
+}
+
+#openvidu-logo:hover {
+    -webkit-filter: grayscale(0.5);
+    filter: grayscale(0.5);
+}
+
+/* Media Queries */
+@media screen and (max-width: 768px) {
+    header {
+        padding: 10px 15px;
+    }
+
+    #join-dialog {
+        width: 90%;
+        padding: 30px;
+    }
+
+    #join-dialog h2 {
+        font-size: 50px;
+    }
+
+    #layout-container {
+        grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+    }
+
+    footer {
+        padding: 10px 15px;
+    }
+}
+
+@media screen and (max-width: 480px) {
+    header {
+        padding: 10px;
+    }
+
+    #join-dialog {
+        width: 100%;
+        padding: 20px;
+    }
+
+    #join-dialog h2 {
+        font-size: 40px;
+    }
+
+    #layout-container {
+        grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+    }
+
+    .video-container {
+        aspect-ratio: 9/16;
+    }
+
+    footer {
+        padding: 10px;
+    }
+}
+
+#captions-container {
+    width: 100%;
+    padding: 25px;
+}
+
+textarea {
+    width: 100%;
+    min-height: 100px;
+    padding: 10px;
+}