squash for release

Co-authored-by: Justin Haaheim <[email protected]>
Co-authored-by: Mark Duppenthaler <[email protected]>
Co-authored-by: Radamés Ajna <[email protected]>

.dockerignore

@@ -0,0 +1 @@
+seamless_server/models/*.pt

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.whl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+.vscode/settings.json
+__pycache__/
+*.pt
+*.model
+venv/

Dockerfile

@@ -0,0 +1,77 @@
+# build frontend with node
+FROM node:20-alpine AS frontend
+RUN apk add --no-cache libc6-compat
+WORKDIR /app
+
+COPY streaming-react-app .
+RUN \
+    if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
+    elif [ -f package-lock.json ]; then npm ci; \
+    elif [ -f pnpm-lock.yaml ]; then yarn global add pnpm && pnpm i --frozen-lockfile; \
+    else echo "Lockfile not found." && exit 1; \
+    fi
+
+RUN npm run build
+
+# build backend on CUDA 
+FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04 AS backend
+WORKDIR /app
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV NODE_MAJOR=20
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    git \
+    git-lfs \
+    wget \
+    curl \
+    # python build dependencies \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    sox libsox-fmt-all \
+    # gradio dependencies \
+    ffmpeg \
+    # fairseq2 dependencies \
+    libsndfile-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+
+RUN curl https://pyenv.run | bash
+ENV PATH=$HOME/.pyenv/shims:$HOME/.pyenv/bin:$PATH
+ARG PYTHON_VERSION=3.10.12
+RUN pyenv install $PYTHON_VERSION && \
+    pyenv global $PYTHON_VERSION && \
+    pyenv rehash && \
+    pip install --no-cache-dir -U pip setuptools wheel
+
+COPY --chown=user:user ./seamless_server ./seamless_server
+# change dir since pip needs to seed whl folder
+RUN cd seamless_server && pip install --no-cache-dir --upgrade -r requirements.txt
+COPY --from=frontend /app/dist ./streaming-react-app/dist
+
+WORKDIR $HOME/app/seamless_server
+USER root
+RUN ln -s /usr/lib/x86_64-linux-gnu/libsox.so.3 /usr/lib/x86_64-linux-gnu/libsox.so
+USER user
+CMD [ "uvicorn", "app_pubsub:app", "--host", "0.0.0.0", "--port", "7860" ]
+
+

README.md

@@ -0,0 +1,61 @@
+---
+title: Seamless Streaming Backend/Frontend
+emoji: 📞
+colorFrom: blue
+colorTo: yellow
+sdk: docker
+pinned: false
+suggested_hardware: t4-medium
+---
+
+# Seamless Streaming demo
+## Running on HF spaces
+You can simply duplicate the space to run it.
+
+## Running locally
+### Install backend seamless_server dependencies
+
+Please note: we *strongly* recommend running the model on GPU. 
+
+If running for the first time, create conda environment and install the desired torch version. The example below is for PyTorch 2.1.1 and variant cu118. Check [here](https://pytorch.org/get-started/locally/) to find the command for your variant. Then install the rest of the requirements:
+```
+cd seamless_server
+conda create --name smlss_server python=3.8
+conda activate smlss_server
+conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
+pip install -r requirements.txt
+conda install -c conda-forge libsndfile==1.0.31
+```
+
+### Install frontend streaming-react-app dependencies
+```
+conda install -c conda-forge nodejs
+cd streaming-react-app
+npm install
+npm run build  # this will create the dist/ folder
+```
+
+
+### Running the server
+
+The server can be run locally with uvicorn below.
+Run the server in dev mode:
+
+```
+cd seamless_server
+uvicorn app_pubsub:app --reload --host localhost
+```
+
+Run the server in prod mode:
+
+```
+cd seamless_server
+uvicorn app_pubsub:app --host 0.0.0.0
+```
+
+To enable additional logging from uvicorn pass `--log-level debug` or `--log-level trace`.
+
+
+### Debuging
+
+If you enable "Server Debug Flag" when starting streaming from the client, this enables extensive debug logging and it saves audio files in /debug folder. 

seamless_server/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+src/__pycache__/
+debug/
+.vscode/

seamless_server/app_pubsub.py

@@ -0,0 +1,797 @@
+from operator import itemgetter
+import os
+from typing import Any, Optional, Tuple, Dict, TypedDict
+from urllib import parse
+from uuid import uuid4
+import colorlog
+import io
+import logging
+from pprint import pformat
+import socketio
+import sys
+import time
+import random
+import string
+from starlette.applications import Starlette
+from starlette.routing import Mount, Route
+from starlette.staticfiles import StaticFiles
+
+
+from src.room import Room, Member
+from src.simuleval_agent_directory import NoAvailableAgentException
+from src.simuleval_agent_directory import SimulevalAgentDirectory
+from src.simuleval_transcoder import SimulevalTranscoder
+from src.transcoder_helpers import get_transcoder_output_events
+
+###############################################
+# Constants
+###############################################
+
+DEBUG = True
+
+ALL_ROOM_ID = "ALL"
+
+ROOM_ID_USABLE_CHARACTERS = string.ascii_uppercase
+ROOM_ID_LENGTH = 4
+
+ROOM_LISTENERS_SUFFIX = "_listeners"
+ROOM_SPEAKERS_SUFFIX = "_speakers"
+
+ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME = "remove_server_lock"
+
+###############################################
+# Configure logger
+###############################################
+
+logger = logging.getLogger("socketio_server_pubsub")
+logger.propagate = False
+
+handler = colorlog.StreamHandler(stream=sys.stdout)
+
+formatter = colorlog.ColoredFormatter(
+    "%(log_color)s[%(asctime)s][%(levelname)s][%(module)s]:%(reset)s %(message)s",
+    reset=True,
+    log_colors={
+        "DEBUG": "cyan",
+        "INFO": "green",
+        "WARNING": "yellow",
+        "ERROR": "red",
+        "CRITICAL": "red,bg_white",
+    },
+)
+
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+logger.setLevel(logging.WARNING)
+
+print("")
+print("")
+print("=" * 20 + " ⭐️ Starting Server... ⭐️ " + "=" * 20)
+
+###############################################
+# Configure socketio server
+###############################################
+
+CLIENT_BUILD_PATH = "../streaming-react-app/dist/"
+static_files = {
+    "/": CLIENT_BUILD_PATH,
+    "/assets/seamless-db6a2555.svg": {
+        "filename": CLIENT_BUILD_PATH + "assets/seamless-db6a2555.svg",
+        "content_type": "image/svg+xml",
+    },
+}
+
+# sio is the main socket.io entrypoint
+sio = socketio.AsyncServer(
+    async_mode="asgi",
+    cors_allowed_origins="*",
+    logger=logger,
+    # engineio_logger=logger,
+)
+# sio.logger.setLevel(logging.DEBUG)
+socketio_app = socketio.ASGIApp(sio)
+
+app_routes = [
+    Mount("/ws", app=socketio_app),  # Mount Socket.IO server under /app
+    Mount(
+        "/", app=StaticFiles(directory=CLIENT_BUILD_PATH, html=True)
+    ),  # Serve static files from root
+]
+app = Starlette(debug=True, routes=app_routes)
+
+# rooms is indexed by room_id
+rooms: Dict[str, Room] = {}
+
+
+class MemberDirectoryObject(TypedDict):
+    room: Room
+    member_object: Member
+
+
+# member_directory is indexed by client_id
+# NOTE: client_id is really "client session id", meaning that it is unique to a single browser session.
+# If a user opens a new tab, they will have a different client_id and can join another room, join
+# the same room with different roles, etc.
+# NOTE: For a long-running production server we would want to clean up members after a certain timeout
+# but for this limited application we can just keep them around
+member_directory: Dict[str, MemberDirectoryObject] = {}
+
+
+class ServerLock(TypedDict):
+    name: str
+    client_id: str
+    member_object: Member
+
+
+server_lock: Optional[ServerLock] = None
+
+server_id = str(uuid4())
+
+# Specify specific models to load (some environments have issues loading multiple models)
+# See AgentWithInfo with JSON format details.
+models_override = os.environ.get("MODELS_OVERRIDE")
+
+available_agents = SimulevalAgentDirectory()
+logger.info("Building and adding agents...")
+if models_override is not None:
+    logger.info(f"MODELS_OVERRIDE supplied from env vars: {models_override}")
+available_agents.build_and_add_agents(models_override)
+
+agents_capabilities_for_json = available_agents.get_agents_capabilities_list_for_json()
+
+
+###############################################
+# Helpers
+###############################################
+
+
+def catch_and_log_exceptions_for_sio_event_handlers(func):
+    # wrapper should have the same signature as the original function
+    async def catch_exception_wrapper(*args, **kwargs):
+        try:
+            return await func(*args, **kwargs)
+        except Exception as e:
+            message = f"[app_pubsub] Caught exception in '{func.__name__}' event handler:\n\n{e}"
+            logger.exception(message, stack_info=True)
+
+            try:
+                exception_data = {
+                    "message": message,
+                    "timeEpochMs": int(time.time() * 1000),
+                }
+
+                try:
+                    # Let's try to add as much useful metadata as possible to the server_exception event
+                    sid = args[0]
+                    if isinstance(sid, str) and len(sid) > 0:
+                        session_data = await get_session_data(sid)
+                        if session_data:
+                            client_id = session_data.get("client_id")
+                            member = session_data.get("member_object")
+                            room = session_data.get("room_object")
+
+                            exception_data["room"] = str(room)
+                            exception_data["member"] = str(member)
+                            exception_data["clientID"] = str(client_id)
+                except Exception as inner_e:
+                    # We expect there will be times when clientID or other values aren't present, so just log this as a warning
+                    logger.warn(
+                        f"[app_pubsub] Caught exception while trying add additional_data to server_exception:\n\n{inner_e}"
+                    )
+
+                # For now let's emit this to all clients. We ultimatley may want to emit it just to the room it's happening in.
+                await sio.emit("server_exception", exception_data)
+            except Exception as inner_e:
+                logger.exception(
+                    f"[app_pubsub] Caught exception while trying to emit server_exception event:\n{inner_e}"
+                )
+
+            # Re-raise the exception so it's handled normally by the server
+            raise e
+
+    # Set the name of the wrapper to the name of the original function so that the socketio server can associate it with the right event
+    catch_exception_wrapper.__name__ = func.__name__
+    return catch_exception_wrapper
+
+
+async def emit_room_state_update(room):
+    await sio.emit(
+        "room_state_update",
+        room.to_json(),
+        room=room.room_id,
+    )
+
+
+async def emit_server_state_update():
+    room_statuses = {
+        room_id: room.get_room_status_dict() for room_id, room in rooms.items()
+    }
+    total_active_connections = sum(
+        [room_status["activeConnections"] for room_status in room_statuses.values()]
+    )
+    total_active_transcoders = sum(
+        [room_status["activeTranscoders"] for room_status in room_statuses.values()]
+    )
+    logger.info(
+        f"[Server Status]: {total_active_connections} active connections (in rooms); {total_active_transcoders} active transcoders"
+    )
+    logger.info(f"[Server Status]: server_lock={server_lock}")
+    server_lock_object_for_js = (
+        {
+            "name": server_lock.get("name"),
+            "clientID": server_lock.get("client_id"),
+            "isActive": server_lock.get("member_object")
+            and server_lock.get("member_object").transcoder is not None,
+        }
+        if server_lock
+        else None
+    )
+    await sio.emit(
+        "server_state_update",
+        {
+            "statusByRoom": room_statuses,
+            "totalActiveConnections": total_active_connections,
+            "totalActiveTranscoders": total_active_transcoders,
+            "agentsCapabilities": agents_capabilities_for_json,
+            "serverLock": server_lock_object_for_js,
+        },
+        room=ALL_ROOM_ID,
+    )
+
+
+async def get_session_data(sid):
+    session = await sio.get_session(sid)
+    # It seems like if the session has not been set that get_session may return None, so let's provide a fallback empty dictionary here
+    return session or {}
+
+
+async def set_session_data(sid, client_id, room_id, room_object, member_object):
+    await sio.save_session(
+        sid,
+        {
+            "client_id": client_id,
+            "room_id": room_id,
+            "room_object": room_object,
+            "member_object": member_object,
+        },
+    )
+
+
+def get_random_room_id():
+    return "".join(random.choices(ROOM_ID_USABLE_CHARACTERS, k=ROOM_ID_LENGTH))
+
+
+def get_random_unused_room_id():
+    room_id = get_random_room_id()
+    while room_id in rooms:
+        room_id = get_random_room_id()
+    return room_id
+
+
+###############################################
+# Socket.io Basic Event Handlers
+###############################################
+
+
+@sio.on("connect")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def connect(sid, environ):
+    logger.info(f"📥 [event: connected] sid={sid}")
+
+    # TODO: Sanitize/validate query param input
+    query_params = dict(parse.parse_qsl(environ["QUERY_STRING"]))
+    client_id = query_params.get("clientID")
+
+    logger.debug(f"query_params:\n{pformat(query_params)}")
+
+    if client_id is None:
+        logger.info("No clientID provided. Disconnecting...")
+        await sio.disconnect(sid)
+        return
+
+    # On reconnect we need to rejoin rooms and reset session data
+    if member_directory.get(client_id):
+        room = member_directory[client_id].get("room")
+        room_id = room.room_id
+        # Note: We could also get this from room.members[client_id]
+        member = member_directory[client_id].get("member_object")
+
+        member.connection_status = "connected"
+        member.session_id = sid
+
+        logger.info(
+            f"[event: connect] {member} reconnected. Attempting to re-add them to socketio rooms and reset session data."
+        )
+
+        if room is None or member is None:
+            logger.error(
+                f"[event: connect] {client_id} is reconnecting, but room or member is None. This should not happen."
+            )
+            await sio.disconnect(sid)
+            return
+
+        sio.enter_room(sid, room_id)
+        sio.enter_room(sid, ALL_ROOM_ID)
+
+        if client_id in room.listeners:
+            sio.enter_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        if client_id in room.speakers:
+            sio.enter_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+
+        # Save the room_id to the socketio client session
+        await set_session_data(
+            sid,
+            client_id=client_id,
+            room_id=room.room_id,
+            room_object=room,
+            member_object=member,
+        )
+        await emit_room_state_update(room)
+    else:
+        # Save the client id to the socketio client session
+        await set_session_data(
+            sid, client_id=client_id, room_id=None, room_object=None, member_object=None
+        )
+
+    await sio.emit("server_id", server_id, to=sid)
+    await emit_server_state_update()
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def disconnect(sid):
+    global server_lock
+    session_data = await get_session_data(sid)
+    # logger.info("session_data", session_data)
+
+    client_id = None
+    member = None
+    room = None
+
+    if session_data:
+        client_id = session_data.get("client_id")
+        member = session_data.get("member_object")
+        room = session_data.get("room_object")
+
+    logger.info(
+        f"[event: disconnect][{room or 'NOT_IN_ROOM'}] member: {member or 'NO_MEMBER_OBJECT'} disconnected"
+    )
+
+    # Release the lock if this is the client that holds the current server lock
+    if server_lock and server_lock.get("client_id") == client_id:
+        server_lock = None
+
+    if member:
+        member.connection_status = "disconnected"
+
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+            member.requested_output_type = None
+
+        if room:
+            logger.info(
+                f"[event: disconnect] {member} disconnected from room {room.room_id}"
+            )
+            await emit_room_state_update(room)
+        else:
+            logger.info(
+                f"[event: disconnect] {member} disconnected, but no room object present. This should not happen."
+            )
+    else:
+        logger.info(
+            f"[event: disconnect] client_id {client_id or 'NO_CLIENT_ID'} with sid {sid} in rooms {str(sio.rooms(sid))} disconnected"
+        )
+
+    await emit_server_state_update()
+
+
+@sio.on("*")
+async def catch_all(event, sid, data):
+    logger.info(f"[unhandled event: {event}] sid={sid} data={data}")
+
+
+###############################################
+# Socket.io Streaming Event handlers
+###############################################
+
+
+@sio.on("join_room")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def join_room(sid, client_id, room_id_from_client, config_dict):
+    global server_lock
+
+    args = {
+        "sid": sid,
+        "client_id": client_id,
+        "room_id": room_id_from_client,
+        "config_dict": config_dict,
+    }
+    logger.info(f"[event: join_room] {args}")
+    session_data = await get_session_data(sid)
+    logger.info(f"session_data: {session_data}")
+
+    room_id = room_id_from_client
+    if room_id is None:
+        room_id = get_random_unused_room_id()
+        logger.info(
+            f"No room_id provided. Generating a random, unused room_id: {room_id}"
+        )
+
+    # Create the room if it doesn't already exist
+    if room_id not in rooms:
+        rooms[room_id] = Room(room_id)
+
+    room = rooms[room_id]
+
+    member = None
+
+    name = "[NO_NAME]"
+
+    # If the client is reconnecting use their existing member object. Otherwise create a new one.
+    if client_id in room.members:
+        member = room.members[client_id]
+        logger.info(f"{member} is rejoining room {room_id}.")
+    else:
+        member_number = len(room.members) + 1
+        name = f"Member {member_number}"
+        member = Member(
+            client_id=client_id,
+            session_id=sid,
+            name=name,
+        )
+        logger.info(f"Created a new Member object: {member}")
+        logger.info(f"Adding {member} to room {room_id}")
+        room.members[client_id] = member
+
+    # Also add them to the member directory
+    member_directory[client_id] = {"room": room, "member_object": member}
+
+    # Join the socketio room, which enables broadcasting to all members of the room
+    sio.enter_room(sid, room_id)
+    # Join the room for all clients
+    sio.enter_room(sid, ALL_ROOM_ID)
+
+    if "listener" in config_dict["roles"]:
+        sio.enter_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        if client_id not in room.listeners:
+            room.listeners.append(client_id)
+    else:
+        sio.leave_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        room.listeners = [
+            listener_id for listener_id in room.listeners if listener_id != client_id
+        ]
+
+    if "speaker" in config_dict["roles"]:
+        sio.enter_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+        if client_id not in room.speakers:
+            room.speakers.append(client_id)
+    else:
+        sio.leave_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+        # If the person is no longer a speaker they should no longer be able to lock the server
+        if server_lock and server_lock.get("client_id") == client_id:
+            logger.info(
+                f"🔓 Server is now unlocked from client {server_lock.get('client_id')} with name/info: {server_lock.get('name')}"
+            )
+            server_lock = None
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+        room.speakers = [
+            speaker_id for speaker_id in room.speakers if speaker_id != client_id
+        ]
+
+    # If we currently own the server lock and are updating roles and we no longer have server lock specified, release it
+    if (
+        server_lock is not None
+        and server_lock["client_id"] == client_id
+        and config_dict.get("lockServerName") is None
+    ):
+        logger.info(f"[join_room] Releasing server lock: {pformat(server_lock)}")
+        server_lock = None
+
+    # Only speakers should be able to lock the server
+    if config_dict.get("lockServerName") is not None and "speaker" in config_dict.get(
+        "roles", {}
+    ):
+        # If something goes wrong and the server gets stuck in a locked state the client can
+        # force the server to remove the lock by passing the special name ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
+        if (
+            server_lock is not None
+            and config_dict.get("lockServerName")
+            == ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
+        ):
+            server_lock = None
+            logger.info(
+                f"🔓 Server lock has been reset by {client_id} using the escape hatch name {ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME}"
+            )
+
+        # If the server is not locked, set a lock. If it's already locked to this client, update the lock object
+        elif server_lock is None or server_lock.get("client_id") == client_id:
+            # TODO: Add some sort of timeout as a backstop in case someone leaves the browser tab open after locking the server
+            server_lock = {
+                "name": config_dict.get("lockServerName"),
+                "client_id": client_id,
+                "member_object": member,
+            }
+            logger.info(
+                f"🔒 Server is now locked to client {server_lock.get('client_id')} with name/info: {server_lock.get('name')}\nThis client will have priority over all others until they disconnect."
+            )
+        # If the server is already locked to someone else, don't allow this client to lock it
+        elif server_lock is not None and server_lock.get("client_id") != client_id:
+            logger.warn(
+                f"⚠️  Server is already locked to client {server_lock.get('client_id')}. Ignoring request to lock to client {client_id}."
+            )
+            # TODO: Maybe throw an error here?
+
+    # Save the room_id to the socketio client session
+    await set_session_data(
+        sid,
+        client_id=client_id,
+        room_id=room_id,
+        room_object=room,
+        member_object=member,
+    )
+
+    await emit_room_state_update(room)
+    await emit_server_state_update()
+
+    return {"roomsJoined": sio.rooms(sid), "roomID": room_id}
+
+
+# TODO: Add code to prevent more than one speaker from connecting/streaming at a time
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def configure_stream(sid, config):
+    session_data = await get_session_data(sid)
+    client_id, member, room = itemgetter("client_id", "member_object", "room_object")(
+        session_data
+    )
+
+    logger.debug(
+        f"[event: configure_stream][{room}] Received stream config from {member}\n{pformat(config)}"
+    )
+
+    if member is None or room is None:
+        logger.error(
+            f"Received stream config from {member}, but member or room is None. This should not happen."
+        )
+        return {"status": "error", "message": "member_or_room_is_none"}
+
+    # If there is a server lock WITH an active transcoder session, prevent other users from configuring and starting a stream
+    # If the server lock client does NOT have an active transcoder session allow this to proceed, knowing that
+    # this stream will be interrupted if the server lock client starts streaming
+    if (
+        server_lock is not None
+        and server_lock.get("client_id") != client_id
+        and server_lock.get("member_object")
+        and server_lock.get("member_object").transcoder is not None
+    ):
+        logger.warn(
+            f"Server is locked to client {server_lock.get('client_id')}. Ignoring request to configure stream from client {client_id}."
+        )
+        return {"status": "error", "message": "server_locked"}
+
+    debug = config.get("debug")
+    async_processing = config.get("async_processing")
+
+    # Currently s2s, s2t or s2s&t
+    model_type = config.get("model_type")
+    member.requested_output_type = model_type
+
+    model_name = config.get("model_name")
+
+    try:
+        agent = available_agents.get_agent_or_throw(model_name)
+    except NoAvailableAgentException as e:
+        logger.warn(f"Error while getting agent: {e}")
+        # await sio.emit("error", str(e), to=sid)
+        await sio.disconnect(sid)
+        return {"status": "error", "message": str(e)}
+
+    if member.transcoder:
+        logger.warn(
+            "Member already has a transcoder configured. Closing it, and overwriting with a new transcoder..."
+        )
+        member.transcoder.close = True
+
+    t0 = time.time()
+    try:
+        member.transcoder = SimulevalTranscoder(
+            agent,
+            config["rate"],
+            debug=debug,
+            buffer_limit=int(config["buffer_limit"]),
+        )
+    except Exception as e:
+        logger.warn(f"Got exception while initializing agents: {e}")
+        # await sio.emit("error", str(e), to=sid)
+        await sio.disconnect(sid)
+        return {"status": "error", "message": str(e)}
+
+    t1 = time.time()
+    logger.debug(f"Booting up VAD and transcoder took {t1-t0} sec")
+
+    # TODO: if async_processing is false, then we need to run transcoder.process_pipeline_once() whenever we receive audio, or at some other sensible interval
+    if async_processing:
+        member.transcoder.start()
+
+    # We need to emit a room state update here since room state now includes # of active transcoders
+    await emit_room_state_update(room)
+    await emit_server_state_update()
+
+    return {"status": "ok", "message": "server_ready"}
+
+
+# The config here is a partial config, meaning it may not contain all the config values -- only the ones the user
+# wants to change
+@sio.on("set_dynamic_config")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def set_dynamic_config(
+    sid,
+    # partial_config's type is defined in StreamingTypes.ts
+    partial_config,
+):
+    session_data = await get_session_data(sid)
+
+    member = None
+
+    if session_data:
+        member = session_data.get("member_object")
+
+    if member:
+        new_dynamic_config = {
+            **(member.transcoder_dynamic_config or {}),
+            **partial_config,
+        }
+        logger.info(
+            f"[set_dynamic_config] Setting new dynamic config:\n\n{pformat(new_dynamic_config)}\n"
+        )
+        member.transcoder_dynamic_config = new_dynamic_config
+
+    return {"status": "ok", "message": "dynamic_config_set"}
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def incoming_audio(sid, blob):
+    session_data = await get_session_data(sid)
+
+    client_id = None
+    member = None
+    room = None
+
+    if session_data:
+        client_id = session_data.get("client_id")
+        member = session_data.get("member_object")
+        room = session_data.get("room_object")
+
+    logger.debug(f"[event: incoming_audio] from member {member}")
+
+    # If the server is locked by someone else, kill our transcoder and ignore incoming audio
+    # If the server lock client does NOT have an active transcoder session allow this incoming audio pipeline to proceed,
+    # knowing that this stream will be interrupted if the server lock client starts streaming
+    if (
+        server_lock is not None
+        and server_lock.get("client_id") != client_id
+        and server_lock.get("member_object")
+        and server_lock.get("member_object").transcoder is not None
+    ):
+        # TODO: Send an event to the client to let them know their streaming session has been killed
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+            # Update both room state and server state given that the number of active transcoders has changed
+            if room:
+                await emit_room_state_update(room)
+            await emit_server_state_update()
+        logger.warn(
+            f"[incoming_audio] Server is locked to client {server_lock.get('client_id')}. Ignoring incoming audio from client {client_id}."
+        )
+        return
+
+    if member is None or room is None:
+        logger.error(
+            f"[incoming_audio] Received incoming_audio from {member}, but member or room is None. This should not happen."
+        )
+        return
+
+    # NOTE: bytes and bytearray are very similar, but bytes is immutable, and is what is returned by socketio
+    if not isinstance(blob, bytes):
+        logger.error(
+            f"[incoming_audio] Received audio from {member}, but it was not of type `bytes`. type(blob) = {type(blob)}"
+        )
+        return
+
+    if member.transcoder is None:
+        logger.error(
+            f"[incoming_audio] Received audio from {member}, but no transcoder configured to process it (member.transcoder is None). This should not happen."
+        )
+        return
+
+    member.transcoder.process_incoming_bytes(
+        blob, dynamic_config=member.transcoder_dynamic_config
+    )
+
+    # Send back any available model output
+    # NOTE: In theory it would make sense remove this from the incoming_audio handler and
+    # handle this in a dedicated thread that checks for output and sends it right away,
+    # but in practice for our limited demo use cases this approach didn't add noticeable
+    # latency, so we're keeping it simple for now.
+    events = get_transcoder_output_events(member.transcoder)
+    logger.debug(f"[incoming_audio] transcoder output events: {len(events)}")
+
+    if len(events) == 0:
+        logger.debug("[incoming_audio] No transcoder output to send")
+    else:
+        for e in events:
+            if e["event"] == "translation_speech" and member.requested_output_type in [
+                "s2s",
+                "s2s&t",
+            ]:
+                logger.debug("[incoming_audio] Sending translation_speech event")
+                await sio.emit(
+                    "translation_speech", e, room=f"{room.room_id}_listeners"
+                )
+            elif e["event"] == "translation_text" and member.requested_output_type in [
+                "s2t",
+                "s2s&t",
+            ]:
+                logger.debug("[incoming_audio] Sending translation_text event")
+                await sio.emit("translation_text", e, room=f"{room.room_id}_listeners")
+            else:
+                logger.error(f"[incoming_audio] Unexpected event type: {e['event']}")
+
+    return
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def stop_stream(sid):
+    session_data = await get_session_data(sid)
+    client_id, member, room = itemgetter("client_id", "member_object", "room_object")(
+        session_data
+    )
+
+    logger.debug(f"[event: stop_stream][{room}] Attempting to stop stream for {member}")
+
+    if member is None or room is None:
+        message = f"Received stop_stream from {member}, but member or room is None. This should not happen."
+        logger.error(message)
+        return {"status": "error", "message": message}
+
+    # In order to stop the stream and end the transcoder thread, set close to True and unset it for the member
+    if member.transcoder:
+        member.transcoder.close = True
+        member.transcoder = None
+    else:
+        message = f"Received stop_stream from {member}, but member.transcoder is None. This should not happen."
+        logger.warn(message)
+
+    # We need to emit a room state update here since room state now includes # of active transcoders
+    await emit_room_state_update(room)
+    # Emit a server state update now that we've changed the number of active transcoders
+    await emit_server_state_update()
+
+    return {"status": "ok", "message": "Stream stopped"}
+
+
+@sio.on("clear_transcript_for_all")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def clear_transcript_for_all(sid):
+    session_data = await get_session_data(sid)
+
+    room = session_data.get("room_object")
+
+    if room:
+        await sio.emit("clear_transcript", room=f"{room.room_id}")
+    else:
+        logger.error("[clear_transcript] room is None. This should not happen.")
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def set_name(sid, name):
+    logger.info(f"[Event: set_name] name={name}")
+    await sio.save_session(sid, {"name": name})

seamless_server/debug/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

seamless_server/models/SeamlessStreaming/vad_s2st_sc_24khz_main.yaml

@@ -0,0 +1,21 @@
+agent_class: seamless_communication.streaming.agents.seamless_streaming_s2st.SeamlessStreamingS2STJointVADAgent
+monotonic_decoder_model_name: seamless_streaming_monotonic_decoder
+unity_model_name: seamless_streaming_unity
+sentencepiece_model: spm_256k_nllb100.model
+
+task: s2st
+tgt_lang: "eng"
+min_unit_chunk_size: 50
+decision_threshold: 0.7
+no_early_stop: True
+block_ngrams: True
+vocoder_name: vocoder_v2
+wav2vec_yaml: wav2vec.yaml
+min_starting_wait_w2vbert: 192
+
+config_yaml: cfg_fbank_u2t.yaml
+upstream_idx: 1
+detokenize_only: True
+device: cuda:0
+max_len_a: 0
+max_len_b: 1000

seamless_server/requirements.txt

@@ -0,0 +1,30 @@
+--pre --extra-index-url https://fair.pkg.atmeta.com/fairseq2/pt2.1.1/cu118
+simuleval==1.1.3
+# seamless_communication
+./whl/seamless_communication-1.0.0-py3-none-any.whl
+Flask==2.1.3
+Flask_Sockets==0.2.1
+g2p_en==2.1.0
+gevent==22.10.2
+gevent_websocket==0.10.1
+librosa==0.9.2
+numpy==1.24.4
+openai_whisper==20230124
+protobuf==4.24.2
+psola==0.0.1
+pydub==0.25.1
+silero==0.4.1
+# simuleval==1.1.1
+soundfile==0.11.0
+stable_ts==1.4.0
+# torch  # to be installed by user for desired PyTorch version
+Werkzeug==2.0.3
+whisper==1.1.10
+colorlog==6.7.0
+python-socketio==5.9.0
+uvicorn[standard]==0.23.2
+parallel-wavegan==0.5.5
+python-jose[cryptography]==3.3.0
+starlette==0.32.0.post1 
+hf_transfer==0.1.4
+huggingface_hub==0.19.4

seamless_server/src/room.py

@@ -0,0 +1,64 @@
+# import json
+import uuid
+
+
+class Room:
+    def __init__(self, room_id) -> None:
+        self.room_id = room_id
+        # members is a dict from client_id to Member
+        self.members = {}
+
+        # listeners and speakers are lists of client_id's
+        self.listeners = []
+        self.speakers = []
+
+    def __str__(self) -> str:
+        return f"Room {self.room_id} ({len(self.members)} member{'s' if len(self.members) == 1 else ''})"
+
+    def to_json(self):
+        varsResult = vars(self)
+        # Remember: result is just a shallow copy, so result.members === self.members
+        # Because of that, we need to jsonify self.members without writing over result.members,
+        # which we do here via dictionary unpacking (the ** operator)
+        result = {
+            **varsResult,
+            "members": {key: value.to_json() for (key, value) in self.members.items()},
+            "activeTranscoders": self.get_active_transcoders(),
+        }
+
+        return result
+
+    def get_active_connections(self):
+        return len(
+            [m for m in self.members.values() if m.connection_status == "connected"]
+        )
+
+    def get_active_transcoders(self):
+        return len([m for m in self.members.values() if m.transcoder is not None])
+
+    def get_room_status_dict(self):
+        return {
+            "activeConnections": self.get_active_connections(),
+            "activeTranscoders": self.get_active_transcoders(),
+        }
+
+
+class Member:
+    def __init__(self, client_id, session_id, name) -> None:
+        self.client_id = client_id
+        self.session_id = session_id
+        self.name = name
+        self.connection_status = "connected"
+        self.transcoder = None
+        self.requested_output_type = None
+        self.transcoder_dynamic_config = None
+
+    def __str__(self) -> str:
+        return f"{self.name} (id: {self.client_id[:4]}...) ({self.connection_status})"
+
+    def to_json(self):
+        self_vars = vars(self)
+        return {
+            **self_vars,
+            "transcoder": self.transcoder is not None,
+        }

seamless_server/src/simuleval_agent_directory.py

@@ -0,0 +1,150 @@
+# Creates a directory in which to look up available agents
+
+from typing import List
+from src.simuleval_transcoder import SimulevalTranscoder
+import json
+import logging
+
+logger = logging.getLogger("socketio_server_pubsub")
+
+# fmt: off
+M4T_P0_LANGS = [
+    "eng",
+    "arb", "ben", "cat", "ces", "cmn", "cym", "dan",
+    "deu", "est", "fin", "fra", "hin", "ind", "ita",
+    "jpn", "kor", "mlt", "nld", "pes", "pol", "por",
+    "ron", "rus", "slk", "spa", "swe", "swh", "tel",
+    "tgl", "tha", "tur", "ukr", "urd", "uzn", "vie",
+]
+# fmt: on
+
+
+class NoAvailableAgentException(Exception):
+    pass
+
+
+class AgentWithInfo:
+    def __init__(
+        self,
+        agent,
+        name: str,
+        modalities: List[str],
+        target_langs: List[str],
+        # Supported dynamic params are defined in StreamingTypes.ts
+        dynamic_params: List[str] = [],
+        description="",
+    ):
+        self.agent = agent
+        self.name = name
+        self.description = description
+        self.modalities = modalities
+        self.target_langs = target_langs
+        self.dynamic_params = dynamic_params
+
+    def get_capabilities_for_json(self):
+        return {
+            "name": self.name,
+            "description": self.description,
+            "modalities": self.modalities,
+            "targetLangs": self.target_langs,
+            "dynamicParams": self.dynamic_params,
+        }
+
+    @classmethod
+    def load_from_json(cls, config: str):
+        """
+        Takes in JSON array of models to load in, e.g.
+        [{"name": "s2s_m4t_emma-unity2_multidomain_v0.1", "description": "M4T model that supports simultaneous S2S and S2T", "modalities": ["s2t", "s2s"], "targetLangs": ["en"]},
+        {"name": "s2s_m4t_expr-emma_v0.1", "description": "ES-EN expressive model that supports S2S and S2T", "modalities": ["s2t", "s2s"], "targetLangs": ["en"]}]
+        """
+        configs = json.loads(config)
+        agents = []
+        for config in configs:
+            agent = SimulevalTranscoder.build_agent(config["name"])
+            agents.append(
+                AgentWithInfo(
+                    agent=agent,
+                    name=config["name"],
+                    modalities=config["modalities"],
+                    target_langs=config["targetLangs"],
+                )
+            )
+        return agents
+
+
+class SimulevalAgentDirectory:
+    # Available models. These are the directories where the models can be found, and also serve as an ID for the model.
+    seamless_streaming_agent = "SeamlessStreaming"
+
+    def __init__(self):
+        self.agents = []
+        self.did_build_and_add_agents = False
+
+    def add_agent(self, agent: AgentWithInfo):
+        self.agents.append(agent)
+
+    def build_agent_if_available(self, model_id, config_name=None):
+        agent = None
+        try:
+            if config_name is not None:
+                agent = SimulevalTranscoder.build_agent(
+                    model_id,
+                    config_name=config_name,
+                )
+            else:
+                agent = SimulevalTranscoder.build_agent(
+                    model_id,
+                )
+        except Exception as e:
+            logger.warning("Failed to build agent %s: %s" % (model_id, e))
+            raise e
+
+        return agent
+
+    def build_and_add_agents(self, models_override=None):
+        if self.did_build_and_add_agents:
+            return
+
+        if models_override is not None:
+            agent_infos = AgentWithInfo.load_from_json(models_override)
+            for agent_info in agent_infos:
+                self.add_agent(agent_info)
+        else:
+            s2s_m4t_expr_agent = self.build_agent_if_available(
+                SimulevalAgentDirectory.seamless_streaming_agent,
+                config_name="vad_s2st_sc_24khz_main.yaml",
+            )
+
+            if s2s_m4t_expr_agent:
+                self.add_agent(
+                    AgentWithInfo(
+                        agent=s2s_m4t_expr_agent,
+                        name=SimulevalAgentDirectory.seamless_streaming_agent,
+                        modalities=["s2t", "s2s"],
+                        target_langs=M4T_P0_LANGS,
+                        dynamic_params=["expressive"],
+                        description="multilingual expressive model that supports S2S and S2T",
+                    )
+                )
+
+        if len(self.agents) == 0:
+            logger.error(
+                "No agents were loaded. This likely means you are missing the actual model files specified in simuleval_agent_directory."
+            )
+
+        self.did_build_and_add_agents = True
+
+    def get_agent(self, name):
+        for agent in self.agents:
+            if agent.name == name:
+                return agent.agent
+        return None
+
+    def get_agent_or_throw(self, name):
+        agent = self.get_agent(name)
+        if agent is None:
+            raise NoAvailableAgentException("No agent found with name= %s" % (name))
+        return agent
+
+    def get_agents_capabilities_list_for_json(self):
+        return [agent.get_capabilities_for_json() for agent in self.agents]

seamless_server/src/simuleval_transcoder.py

@@ -0,0 +1,423 @@
+from simuleval.utils.agent import build_system_from_dir
+from typing import Any, List, Optional, Tuple, Union
+import numpy as np
+import soundfile
+import io
+import asyncio
+from simuleval.agents.pipeline import TreeAgentPipeline
+from simuleval.agents.states import AgentStates
+from simuleval.data.segments import Segment, EmptySegment, SpeechSegment
+import threading
+import math
+import logging
+import sys
+from pathlib import Path
+import time
+from g2p_en import G2p
+import torch
+import traceback
+import time
+import random
+import colorlog
+
+from .speech_and_text_output import SpeechAndTextOutput
+
+MODEL_SAMPLE_RATE = 16_000
+
+logger = logging.getLogger(__name__)
+# logger.propagate = False
+handler = colorlog.StreamHandler(stream=sys.stdout)
+formatter = colorlog.ColoredFormatter(
+    "%(log_color)s[%(asctime)s][%(levelname)s][%(module)s]:%(reset)s %(message)s",
+    reset=True,
+    log_colors={
+        "DEBUG": "cyan",
+        "INFO": "green",
+        "WARNING": "yellow",
+        "ERROR": "red",
+        "CRITICAL": "red,bg_white",
+    },
+)
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+logger.setLevel(logging.WARNING)
+
+
+class OutputSegments:
+    def __init__(self, segments: Union[List[Segment], Segment]):
+        if isinstance(segments, Segment):
+            segments = [segments]
+        self.segments: List[Segment] = [s for s in segments]
+
+    @property
+    def is_empty(self):
+        return all(segment.is_empty for segment in self.segments)
+
+    @property
+    def finished(self):
+        return all(segment.finished for segment in self.segments)
+
+    def compute_length(self, g2p):
+        lengths = []
+        for segment in self.segments:
+            if segment.data_type == "text":
+                lengths.append(len([x for x in g2p(segment.content) if x != " "]))
+            elif segment.data_type == "speech":
+                lengths.append(len(segment.content) / MODEL_SAMPLE_RATE)
+            elif isinstance(segment, EmptySegment):
+                continue
+            else:
+                logger.warning(
+                    f"Unexpected data_type: {segment.data_type} not in 'speech', 'text'"
+                )
+        return max(lengths)
+
+    @classmethod
+    def join_output_buffer(
+        cls, buffer: List[List[Segment]], output: SpeechAndTextOutput
+    ):
+        num_segments = len(buffer[0])
+        for i in range(num_segments):
+            segment_list = [
+                buffer[j][i]
+                for j in range(len(buffer))
+                if buffer[j][i].data_type is not None
+            ]
+            if len(segment_list) == 0:
+                continue
+            if len(set(segment.data_type for segment in segment_list)) != 1:
+                logger.warning(
+                    f"Data type mismatch at {i}: {set(segment.data_type for segment in segment_list)}"
+                )
+                continue
+            data_type = segment_list[0].data_type
+            if data_type == "text":
+                if output.text is not None:
+                    logger.warning("Multiple text outputs, overwriting!")
+                output.text = " ".join([segment.content for segment in segment_list])
+            elif data_type == "speech":
+                if output.speech_samples is not None:
+                    logger.warning("Multiple speech outputs, overwriting!")
+                speech_out = []
+                for segment in segment_list:
+                    speech_out += segment.content
+                output.speech_samples = speech_out
+                output.speech_sample_rate = segment.sample_rate
+            elif isinstance(segment_list[0], EmptySegment):
+                continue
+            else:
+                logger.warning(
+                    f"Invalid output buffer data type: {data_type}, expected 'speech' or 'text"
+                )
+
+        return output
+
+    def __repr__(self) -> str:
+        repr_str = str(self.segments)
+        return f"{self.__class__.__name__}(\n\t{repr_str}\n)"
+
+
+class SimulevalTranscoder:
+    def __init__(self, agent, sample_rate, debug, buffer_limit):
+        self.agent = agent
+        self.input_queue = asyncio.Queue()
+        self.output_queue = asyncio.Queue()
+        self.states = self.agent.build_states()
+        if debug:
+            self.get_states_root().debug = True
+        self.incoming_sample_rate = sample_rate
+        self.close = False
+        self.g2p = G2p()
+
+        # buffer all outgoing translations within this amount of time
+        self.output_buffer_idle_ms = 5000
+        self.output_buffer_size_limit = (
+            buffer_limit  # phonemes for text, seconds for speech
+        )
+        self.output_buffer_cur_size = 0
+        self.output_buffer: List[List[Segment]] = []
+        self.speech_output_sample_rate = None
+
+        self.last_output_ts = time.time() * 1000
+        self.timeout_ms = (
+            30000  # close the transcoder thread after this amount of silence
+        )
+        self.first_input_ts = None
+        self.first_output_ts = None
+        self.debug = debug
+        self.debug_ts = f"{time.time()}_{random.randint(1000, 9999)}"
+        if self.debug:
+            debug_folder = Path(__file__).resolve().parent.parent / "debug"
+            self.test_incoming_wav = soundfile.SoundFile(
+                debug_folder / f"{self.debug_ts}_test_incoming.wav",
+                mode="w+",
+                format="WAV",
+                subtype="PCM_16",
+                samplerate=self.incoming_sample_rate,
+                channels=1,
+            )
+            self.get_states_root().test_input_segments_wav = soundfile.SoundFile(
+                debug_folder / f"{self.debug_ts}_test_input_segments.wav",
+                mode="w+",
+                format="WAV",
+                samplerate=MODEL_SAMPLE_RATE,
+                channels=1,
+            )
+
+    def get_states_root(self) -> AgentStates:
+        if isinstance(self.agent, TreeAgentPipeline):
+            # self.states is a dict
+            return self.states[self.agent.source_module]
+        else:
+            # self.states is a list
+            return self.states[0]
+
+    def reset_states(self):
+        if isinstance(self.agent, TreeAgentPipeline):
+            states_iter = self.states.values()
+        else:
+            states_iter = self.states
+        for state in states_iter:
+            state.reset()
+
+    def debug_log(self, *args):
+        if self.debug:
+            logger.info(*args)
+
+    @classmethod
+    def build_agent(cls, model_path, config_name="vad_s2st_main.yaml"):
+        logger.info(f"Building simuleval agent: {model_path}, {config_name}")
+        agent = build_system_from_dir(
+            Path(__file__).resolve().parent.parent / f"models/{model_path}",
+            config_name=config_name,
+        )
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        agent.to(device, fp16=True)
+        logger.info(
+            f"Successfully built simuleval agent {model_path} on device {device}"
+        )
+
+        return agent
+
+    def process_incoming_bytes(self, incoming_bytes, dynamic_config):
+        # TODO: We probably want to do some validation on dynamic_config to ensure it has what we needs
+        segment, sr = self._preprocess_wav(incoming_bytes)
+        segment = SpeechSegment(
+            content=segment,
+            sample_rate=sr,
+            tgt_lang=dynamic_config.get("targetLanguage"),
+            config=dynamic_config,
+        )
+        # # segment is array([0, 0, 0, ..., 0, 0, 0], dtype=int16)
+        self.input_queue.put_nowait(segment)
+
+    def get_input_segment(self):
+        if self.input_queue.empty():
+            return None
+        chunk = self.input_queue.get_nowait()
+        self.input_queue.task_done()
+        return chunk
+
+    def convert_waveform(
+        self,
+        waveform: Union[np.ndarray, torch.Tensor],
+        sample_rate: int,
+        normalize_volume: bool = False,
+        to_mono: bool = False,
+        to_sample_rate: Optional[int] = None,
+    ) -> Tuple[Union[np.ndarray, torch.Tensor], int]:
+        """convert a waveform:
+        - to a target sample rate
+        - from multi-channel to mono channel
+        - volume normalization
+
+        Args:
+            waveform (numpy.ndarray or torch.Tensor): 2D original waveform
+                (channels x length)
+            sample_rate (int): original sample rate
+            normalize_volume (bool): perform volume normalization
+            to_mono (bool): convert to mono channel if having multiple channels
+            to_sample_rate (Optional[int]): target sample rate
+        Returns:
+            waveform (numpy.ndarray): converted 2D waveform (channels x length)
+            sample_rate (float): target sample rate
+        """
+        try:
+            import torchaudio.sox_effects as ta_sox
+        except ImportError:
+            raise ImportError("Please install torchaudio: pip install torchaudio")
+
+        effects = []
+        if normalize_volume:
+            effects.append(["gain", "-n"])
+        if to_sample_rate is not None and to_sample_rate != sample_rate:
+            effects.append(["rate", f"{to_sample_rate}"])
+        if to_mono and waveform.shape[0] > 1:
+            effects.append(["channels", "1"])
+        if len(effects) > 0:
+            is_np_input = isinstance(waveform, np.ndarray)
+            _waveform = torch.from_numpy(waveform) if is_np_input else waveform
+            converted, converted_sample_rate = ta_sox.apply_effects_tensor(
+                _waveform, sample_rate, effects
+            )
+            if is_np_input:
+                converted = converted.numpy()
+            return converted, converted_sample_rate
+        return waveform, sample_rate
+
+    def _preprocess_wav(self, data: Any) -> Tuple[np.ndarray, int]:
+        segment, sample_rate = soundfile.read(
+            io.BytesIO(data),
+            dtype="float32",
+            always_2d=True,
+            frames=-1,
+            start=0,
+            format="RAW",
+            subtype="PCM_16",
+            samplerate=self.incoming_sample_rate,
+            channels=1,
+        )
+        if self.debug:
+            self.test_incoming_wav.seek(0, soundfile.SEEK_END)
+            self.test_incoming_wav.write(segment)
+
+        segment = segment.T
+        segment, new_sample_rate = self.convert_waveform(
+            segment,
+            sample_rate,
+            normalize_volume=False,
+            to_mono=True,
+            to_sample_rate=MODEL_SAMPLE_RATE,
+        )
+
+        assert MODEL_SAMPLE_RATE == new_sample_rate
+        segment = segment.squeeze(axis=0)
+        return segment, new_sample_rate
+
+    def process_pipeline_impl(self, input_segment):
+        try:
+            with torch.no_grad():
+                output_segment = OutputSegments(
+                    self.agent.pushpop(input_segment, self.states)
+                )
+            if (
+                self.get_states_root().first_input_ts is not None
+                and self.first_input_ts is None
+            ):
+                # TODO: this is hacky
+                self.first_input_ts = self.get_states_root().first_input_ts
+
+            if not output_segment.is_empty:
+                self.output_queue.put_nowait(output_segment)
+
+            if output_segment.finished:
+                self.debug_log("OUTPUT SEGMENT IS FINISHED. Resetting states.")
+
+                self.reset_states()
+
+                if self.debug:
+                    # when we rebuild states, this value is reset to whatever
+                    # is in the system dir config, which defaults debug=False.
+                    self.get_states_root().debug = True
+        except Exception as e:
+            logger.error(f"Got exception while processing pipeline: {e}")
+            traceback.print_exc()
+        return input_segment
+
+    def process_pipeline_loop(self):
+        if self.close:
+            return  # closes the thread
+
+        self.debug_log("processing_pipeline")
+        while not self.close:
+            input_segment = self.get_input_segment()
+            if input_segment is None:
+                if self.get_states_root().is_fresh_state:  # TODO: this is hacky
+                    time.sleep(0.3)
+                else:
+                    time.sleep(0.03)
+                continue
+            self.process_pipeline_impl(input_segment)
+        self.debug_log("finished processing_pipeline")
+
+    def process_pipeline_once(self):
+        if self.close:
+            return
+
+        self.debug_log("processing pipeline once")
+        input_segment = self.get_input_segment()
+        if input_segment is None:
+            return
+        self.process_pipeline_impl(input_segment)
+        self.debug_log("finished processing_pipeline_once")
+
+    def get_output_segment(self):
+        if self.output_queue.empty():
+            return None
+
+        output_chunk = self.output_queue.get_nowait()
+        self.output_queue.task_done()
+        return output_chunk
+
+    def start(self):
+        self.debug_log("starting transcoder in a thread")
+        threading.Thread(target=self.process_pipeline_loop).start()
+
+    def first_translation_time(self):
+        return round((self.first_output_ts - self.first_input_ts) / 1000, 2)
+
+    def get_buffered_output(self) -> SpeechAndTextOutput:
+        now = time.time() * 1000
+        self.debug_log(f"get_buffered_output queue size: {self.output_queue.qsize()}")
+        while not self.output_queue.empty():
+            tmp_out = self.get_output_segment()
+            if tmp_out and tmp_out.compute_length(self.g2p) > 0:
+                if len(self.output_buffer) == 0:
+                    self.last_output_ts = now
+                self._populate_output_buffer(tmp_out)
+                self._increment_output_buffer_size(tmp_out)
+
+                if tmp_out.finished:
+                    self.debug_log("tmp_out.finished")
+                    res = self._gather_output_buffer_data(final=True)
+                    self.debug_log(f"gathered output data: {res}")
+                    self.output_buffer = []
+                    self.increment_output_buffer_size = 0
+                    self.last_output_ts = now
+                    self.first_output_ts = now
+                    return res
+            else:
+                self.debug_log("tmp_out.compute_length is not > 0")
+
+        if len(self.output_buffer) > 0 and (
+            now - self.last_output_ts >= self.output_buffer_idle_ms
+            or self.output_buffer_cur_size >= self.output_buffer_size_limit
+        ):
+            self.debug_log(
+                "[get_buffered_output] output_buffer is not empty. getting res to return."
+            )
+            self.last_output_ts = now
+            res = self._gather_output_buffer_data(final=False)
+            self.debug_log(f"gathered output data: {res}")
+            self.output_buffer = []
+            self.output_buffer_phoneme_count = 0
+            self.first_output_ts = now
+            return res
+        else:
+            self.debug_log("[get_buffered_output] output_buffer is empty...")
+            return None
+
+    def _gather_output_buffer_data(self, final):
+        output = SpeechAndTextOutput()
+        output.final = final
+        output = OutputSegments.join_output_buffer(self.output_buffer, output)
+        return output
+
+    def _increment_output_buffer_size(self, segment: OutputSegments):
+        self.output_buffer_cur_size += segment.compute_length(self.g2p)
+
+    def _populate_output_buffer(self, segment: OutputSegments):
+        self.output_buffer.append(segment.segments)
+
+    def _compute_phoneme_count(self, string: str) -> int:
+        return len([x for x in self.g2p(string) if x != " "])

seamless_server/src/speech_and_text_output.py

@@ -0,0 +1,15 @@
+# Provides a container to return both speech and text output from our model at the same time
+
+
+class SpeechAndTextOutput:
+    def __init__(
+        self,
+        text: str = None,
+        speech_samples: list = None,
+        speech_sample_rate: float = None,
+        final: bool = False,
+    ):
+        self.text = text
+        self.speech_samples = speech_samples
+        self.speech_sample_rate = speech_sample_rate
+        self.final = final

seamless_server/src/transcoder_helpers.py

@@ -0,0 +1,43 @@
+import logging
+
+logger = logging.getLogger("socketio_server_pubsub")
+
+
+def get_transcoder_output_events(transcoder) -> list:
+    speech_and_text_output = transcoder.get_buffered_output()
+    if speech_and_text_output is None:
+        logger.debug("No output from transcoder.get_buffered_output()")
+        return []
+
+    logger.debug(f"We DID get output from the transcoder! {speech_and_text_output}")
+
+    lat = None
+
+    events = []
+
+    if speech_and_text_output.speech_samples:
+        events.append(
+            {
+                "event": "translation_speech",
+                "payload": speech_and_text_output.speech_samples,
+                "sample_rate": speech_and_text_output.speech_sample_rate,
+            }
+        )
+
+    if speech_and_text_output.text:
+        events.append(
+            {
+                "event": "translation_text",
+                "payload": speech_and_text_output.text,
+            }
+        )
+
+    for e in events:
+        e["eos"] = speech_and_text_output.final
+
+    # if not latency_sent:
+    #     lat = transcoder.first_translation_time()
+    #     latency_sent = True
+    #     to_send["latency"] = lat
+
+    return events

seamless_server/whl/seamless_communication-1.0.0-py3-none-any.whl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1df10e0c85ee0ffbc9f2e1bf8896850a52c551383df0332a94d26d9d39770c85
+size 201552

streaming-react-app/.eslintrc.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  root: true,
+  env: {browser: true, es2020: true},
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      {allowConstantExport: true},
+    ],
+  },
+};

streaming-react-app/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

streaming-react-app/README.md

@@ -0,0 +1,14 @@
+# 🚀 Streaming React App
+
+## Getting Started
+
+This project uses the [Yarn Package Manager](https://yarnpkg.com/).
+
+1. `yarn` - Install project dependencies
+2. `yarn run dev` - Run the app with a development server that supports hot module reloading
+
+NOTE: You will either need to provide the server URL via environment variable (you can use the `.env` file for this) or via a url param when you load the react app (example: `http://localhost:5173/?serverURL=localhost:8000`)
+
+## URL Parameters
+
+You can provide URL parameters in order to change the behavior of the app. Those are documented in [URLParams.ts](src/URLParams.ts).

streaming-react-app/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/src/assets/seamless.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Seamless Translation</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

streaming-react-app/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "streaming-react-app",
+  "private": true,
+  "version": "0.0.13",
+  "type": "module",
+  "scripts": {
+    "dev": "vite --host --strictPort",
+    "build": "tsc && vite build",
+    "preview": "vite preview",
+    "clean:node-modules": "rm -rf node_modules/",
+    "ts-check": "tsc --noEmit",
+    "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "prettier-check": "cd ../ && yarn run prettier-base --check streaming-react-app",
+    "signal": "concurrently --names \"TS,LINT,PRETTIER\" -c \"bgBlack.bold,bgRed.bold,bgCyan.bold\" \"yarn run ts-check\" \"yarn run lint\" \"yarn run prettier-check\""
+  },
+  "dependencies": {
+    "@emotion/react": "11.11.1",
+    "@emotion/styled": "11.11.0",
+    "@mui/icons-material": "5.14.3",
+    "@mui/material": "5.14.5",
+    "@react-three/drei": "^9.83.9",
+    "@react-three/fiber": "^8.14.1",
+    "@react-three/xr": "^5.7.1",
+    "amazon-cognito-identity-js": "^6.3.6",
+    "audiobuffer-to-wav": "^1.0.0",
+    "aws-sdk": "^2.1472.0",
+    "iso-639-1": "^3.1.0",
+    "js-cookie": "^3.0.5",
+    "lodash": "4.17.21",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-google-charts": "^4.0.1",
+    "socket.io-client": "^4.7.2",
+    "three": "^0.156.1",
+    "three-mesh-ui": "^6.5.4",
+    "uuid": "^9.0.0",
+    "zustand": "^4.4.3"
+  },
+  "devDependencies": {
+    "@types/node": "^20.5.3",
+    "@types/react": "^18.2.15",
+    "@types/react-dom": "^18.2.7",
+    "@types/uuid": "^9.0.2",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "@vitejs/plugin-react": "^4.0.3",
+    "concurrently": "8.2.1",
+    "eslint": "^8.45.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.3",
+    "typescript": "5.1.6",
+    "vite": "^4.4.5"
+  }
+}

streaming-react-app/src/App.tsx

@@ -0,0 +1,57 @@
+import SocketWrapper from './SocketWrapper';
+import {ThemeProvider} from '@mui/material/styles';
+import theme from './theme';
+import StreamingInterface from './StreamingInterface';
+import CssBaseline from '@mui/material/CssBaseline';
+import {createContext, useCallback, useState} from 'react';
+import packageJson from '../package.json';
+
+console.log(`Streaming React App version: ${packageJson?.version}`);
+
+// Roboto font for mui ui library
+// import '@fontsource/roboto/300.css';
+// import '@fontsource/roboto/400.css';
+// import '@fontsource/roboto/500.css';
+// import '@fontsource/roboto/700.css';
+
+export const AppResetKeyContext = createContext<(newKey: string) => void>(
+  () => {
+    throw new Error('AppResetKeyContext not initialized');
+  },
+);
+
+function App() {
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <SocketWrapper>
+        <StreamingInterface />
+      </SocketWrapper>
+    </ThemeProvider>
+  );
+}
+
+function AppWrapper() {
+  const [appResetKey, setAppResetKey] = useState<string>('[initial value]');
+  const setAppResetKeyHandler = useCallback((newKey: string) => {
+    setAppResetKey((prev) => {
+      console.warn(
+        `Resetting the app with appResetKey: ${newKey}; prevKey: ${prev}`,
+      );
+      if (prev === newKey) {
+        console.error(
+          `The appResetKey was the same as the previous key, so the app will not reset.`,
+        );
+      }
+      return newKey;
+    });
+  }, []);
+
+  return (
+    <AppResetKeyContext.Provider value={setAppResetKeyHandler}>
+      <App key={appResetKey} />
+    </AppResetKeyContext.Provider>
+  );
+}
+
+export default AppWrapper;

streaming-react-app/src/Blink.tsx

@@ -0,0 +1,41 @@
+import Box from '@mui/material/Box';
+import {useEffect, useState} from 'react';
+
+type Props = {
+  intervalMs: number;
+  children: React.ReactNode;
+  shouldBlink: boolean;
+  // display?: 'block' | 'inline' | 'inline-block';
+};
+
+export default function Blink({
+  // display = 'inline-block',
+  shouldBlink,
+  intervalMs,
+  children,
+}: Props): React.ReactElement {
+  const [cursorBlinkOn, setCursorBlinkOn] = useState(false);
+
+  useEffect(() => {
+    if (shouldBlink) {
+      const interval = setInterval(() => {
+        setCursorBlinkOn((prev) => !prev);
+      }, intervalMs);
+
+      return () => clearInterval(interval);
+    } else {
+      setCursorBlinkOn(false);
+    }
+  }, [intervalMs, shouldBlink]);
+
+  return (
+    <Box
+      component="span"
+      sx={{
+        display: 'inline-block',
+        visibility: cursorBlinkOn ? 'visible' : 'hidden',
+      }}>
+      {children}
+    </Box>
+  );
+}

streaming-react-app/src/DebugSection.tsx

@@ -0,0 +1,62 @@
+import {Chart} from 'react-google-charts';
+import debug from './debug';
+import {
+  Accordion,
+  AccordionDetails,
+  AccordionSummary,
+  Button,
+  Typography,
+} from '@mui/material';
+import {useState} from 'react';
+import ArrowDropDownIcon from '@mui/icons-material/ArrowDropDown';
+
+export default function DebugChart() {
+  const [showDebugTimings, setShowDebugTimings] = useState<boolean>(false);
+
+  const data = debug()?.getChartData();
+  const options = {
+    timeline: {
+      groupByRowLabel: true,
+    },
+  };
+
+  return (
+    <div className="horizontal-padding-sra text-chunk-sra">
+      <Accordion
+        expanded={showDebugTimings}
+        onChange={() => setShowDebugTimings(!showDebugTimings)}
+        elevation={0}
+        sx={{border: 1, borderColor: 'rgba(0, 0, 0, 0.3)'}}>
+        <AccordionSummary
+          expandIcon={<ArrowDropDownIcon />}
+          className="debug-section">
+          Debug Info
+        </AccordionSummary>
+        <AccordionDetails>
+          {data && data.length > 1 ? (
+            <>
+              <Chart
+                chartType="Timeline"
+                data={data}
+                width="100%"
+                height="400px"
+                options={options}
+              />
+              <Button
+                variant="contained"
+                sx={{marginBottom: 1}}
+                onClick={() => {
+                  debug()?.downloadInputAudio();
+                  debug()?.downloadOutputAudio();
+                }}>
+                Download Input / Ouput Audio
+              </Button>
+            </>
+          ) : (
+            <Typography>No input / output detected</Typography>
+          )}
+        </AccordionDetails>
+      </Accordion>
+    </div>
+  );
+}

streaming-react-app/src/RoomConfig.tsx

@@ -0,0 +1,262 @@
+import Stack from '@mui/material/Stack';
+import TextField from '@mui/material/TextField';
+import {isValidRoomID, isValidPartialRoomID} from './generateNewRoomID';
+import {useCallback, useEffect, useState} from 'react';
+import Button from '@mui/material/Button';
+import {useSocket} from './useSocket';
+import FormGroup from '@mui/material/FormGroup';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import Checkbox from '@mui/material/Checkbox';
+import {RoomState} from './types/RoomState';
+import setURLParam from './setURLParam';
+import {getURLParams} from './URLParams';
+import {
+  JoinRoomConfig,
+  Roles,
+  ServerState,
+  StreamingStatus,
+} from './types/StreamingTypes';
+import Alert from '@mui/material/Alert';
+
+function capitalize(str: string): string {
+  return str.charAt(0).toUpperCase() + str.slice(1);
+}
+
+type Props = {
+  roomState: RoomState | null;
+  serverState: ServerState | null;
+  onJoinRoomOrUpdateRoles?: () => void;
+  streamingStatus: StreamingStatus;
+};
+
+export default function RoomConfig({
+  roomState,
+  serverState,
+  onJoinRoomOrUpdateRoles,
+  streamingStatus,
+}: Props) {
+  const {socket, clientID} = useSocket();
+
+  const urlParams = getURLParams();
+  const roomIDParam = urlParams.roomID;
+  const autoJoinRoom = urlParams.autoJoin;
+
+  const [roomID, setRoomID] = useState<string>(
+    (roomIDParam ?? '').toUpperCase(),
+  );
+  const [roomIDError, setRoomIDError] = useState<boolean>(false);
+  const [roles, setRoles] = useState<{speaker: boolean; listener: boolean}>({
+    speaker: true,
+    listener: true,
+  });
+  const [lockServer, setLockServer] = useState<boolean>(false);
+  const [lockServerName, setLockServerName] = useState<string>('');
+
+  const [joinInProgress, setJoinInProgress] = useState<boolean>(false);
+  const [didAttemptAutoJoin, setDidAttemptAutoJoin] = useState<boolean>(false);
+
+  const isValidServerLock =
+    lockServer === false ||
+    (lockServerName != null && lockServerName.length > 0);
+  const isValidRoles = Object.values(roles).filter(Boolean).length > 0;
+  const isValidAllInputs =
+    isValidRoomID(roomID) && isValidRoles && isValidServerLock;
+  const roomIDFromServer = roomState?.room_id ?? null;
+
+  const onJoinRoom = useCallback(
+    (createNewRoom: boolean) => {
+      if (socket == null) {
+        console.error('Socket is null, cannot join room');
+        return;
+      }
+      console.debug(`Attempting to join roomID ${roomID}...`);
+
+      const lockServerValidated: string | null =
+        lockServer && roles['speaker'] ? lockServerName : null;
+
+      setJoinInProgress(true);
+
+      const configObject: JoinRoomConfig = {
+        roles: (Object.keys(roles) as Array<Roles>).filter(
+          (role) => roles[role] === true,
+        ),
+        lockServerName: lockServerValidated,
+      };
+
+      socket.emit(
+        'join_room',
+        clientID,
+        createNewRoom ? null : roomID,
+        configObject,
+        (result) => {
+          console.log('join_room result:', result);
+          if (createNewRoom) {
+            setRoomID(result.roomID);
+          }
+          if (onJoinRoomOrUpdateRoles != null) {
+            onJoinRoomOrUpdateRoles();
+          }
+          setURLParam('roomID', result.roomID);
+          setJoinInProgress(false);
+        },
+      );
+    },
+    [
+      clientID,
+      lockServer,
+      lockServerName,
+      onJoinRoomOrUpdateRoles,
+      roles,
+      roomID,
+      socket,
+    ],
+  );
+
+  useEffect(() => {
+    if (
+      autoJoinRoom === true &&
+      didAttemptAutoJoin === false &&
+      socket != null
+    ) {
+      // We want to consider this an attempt whether or not we actually try to join, because
+      // we only want auto-join to happen on initial load
+      setDidAttemptAutoJoin(true);
+      if (
+        isValidAllInputs &&
+        joinInProgress === false &&
+        roomIDFromServer == null
+      ) {
+        console.debug('Attempting to auto-join room...');
+
+        onJoinRoom(false);
+      } else {
+        console.debug('Unable to auto-join room', {
+          isValidAllInputs,
+          joinInProgress,
+          roomIDFromServer,
+        });
+      }
+    }
+  }, [
+    autoJoinRoom,
+    didAttemptAutoJoin,
+    isValidAllInputs,
+    joinInProgress,
+    onJoinRoom,
+    roomIDFromServer,
+    socket,
+  ]);
+
+  return (
+    <Stack direction="column" spacing="12px">
+      <Stack direction="row" spacing="12px" sx={{alignItems: 'center'}}>
+        <TextField
+          size="small"
+          label="Room Code"
+          variant="outlined"
+          disabled={roomState?.room_id != null}
+          value={roomID}
+          error={roomIDError}
+          onChange={(e) => {
+            const id = e.target.value.toUpperCase();
+            if (isValidPartialRoomID(id)) {
+              setRoomIDError(false);
+              setRoomID(id);
+            } else {
+              setRoomIDError(true);
+            }
+          }}
+          sx={{width: '8em'}}
+        />
+
+        <div>
+          <Button
+            variant="contained"
+            disabled={
+              isValidAllInputs === false ||
+              joinInProgress ||
+              streamingStatus !== 'stopped'
+            }
+            onClick={() => onJoinRoom(false)}>
+            {roomState?.room_id != null ? 'Update Roles' : 'Join Room'}
+          </Button>
+        </div>
+
+        {roomState?.room_id == null && (
+          <div>
+            <Button
+              variant="contained"
+              disabled={
+                roomState?.room_id != null ||
+                joinInProgress ||
+                streamingStatus !== 'stopped'
+              }
+              onClick={() => onJoinRoom(true)}>
+              {'Create New Room'}
+            </Button>
+          </div>
+        )}
+      </Stack>
+
+      <FormGroup>
+        {Object.keys(roles).map((role) => {
+          return (
+            <FormControlLabel
+              disabled={streamingStatus !== 'stopped'}
+              key={role}
+              control={
+                <Checkbox
+                  checked={roles[role]}
+                  onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+                    setRoles((prevRoles) => ({
+                      ...prevRoles,
+                      [role]: event.target.checked,
+                    }));
+                  }}
+                />
+              }
+              label={capitalize(role)}
+            />
+          );
+        })}
+
+        {urlParams.enableServerLock && roles['speaker'] === true && (
+          <>
+            <FormControlLabel
+              disabled={streamingStatus !== 'stopped'}
+              control={
+                <Checkbox
+                  checked={lockServer}
+                  onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+                    setLockServer(event.target.checked);
+                  }}
+                />
+              }
+              label="Lock Server (prevent other users from streaming)"
+            />
+          </>
+        )}
+      </FormGroup>
+
+      {urlParams.enableServerLock &&
+        roles['speaker'] === true &&
+        lockServer && (
+          <TextField
+            disabled={streamingStatus !== 'stopped'}
+            label="Enter Your Name + Expected Lock End Time"
+            variant="outlined"
+            value={lockServerName}
+            onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+              setLockServerName(event.target.value);
+            }}
+            helperText="Locking the server will prevent anyone else from using it until you close the page, in order to maximize server performance. Please only use this for live demos."
+          />
+        )}
+
+      {serverState?.serverLock != null &&
+        serverState.serverLock.clientID === clientID && (
+          <Alert severity="success">{`The server is now locked for your use (${serverState?.serverLock?.name}). Close this window to release the lock so that others may use the server.`}</Alert>
+        )}
+    </Stack>
+  );
+}

streaming-react-app/src/SocketWrapper.tsx

@@ -0,0 +1,218 @@
+import {useContext, useEffect, useMemo, useRef, useState} from 'react';
+import socketIOClient, {Socket} from 'socket.io-client';
+import useStable from './useStable';
+import {v4 as uuidv4} from 'uuid';
+import {SocketContext} from './useSocket';
+import {AppResetKeyContext} from './App';
+import Backdrop from '@mui/material/Backdrop';
+import CircularProgress from '@mui/material/CircularProgress';
+import Typography from '@mui/material/Typography';
+import {getURLParams} from './URLParams';
+
+// The time to wait before showing a "disconnected" screen upon initial app load
+const INITIAL_DISCONNECT_SCREEN_DELAY = 2000;
+const SERVER_URL_DEFAULT = `${window.location.protocol === "https:" ? "wss" : "ws"
+                    }://${window.location.host}`;
+                    
+export default function SocketWrapper({children}) {
+  const [socket, setSocket] = useState<Socket | null>(null);
+  const [connected, setConnected] = useState<boolean | null>(null);
+  // Default to true:
+  const [willAttemptReconnect] = useState<boolean>(true);
+  const serverIDRef = useRef<string | null>(null);
+
+  const setAppResetKey = useContext(AppResetKeyContext);
+
+  /**
+   * Previously we had stored the clientID in local storage, but in that case
+   * if a user refreshes their page they'll still have the same clientID, and
+   * will be put back into the same room, which may be confusing if they're trying
+   * to join a new room or reset the app interface. So now clientIDs persist only as
+   * long as the react app full lifecycle
+   */
+  const clientID = useStable<string>(() => {
+    const newID = uuidv4();
+    // Set the clientID in session storage so if the page reloads the person
+    // still retains their member/room config
+    return newID;
+  });
+
+  const socketObject = useMemo(
+    () => ({socket, clientID, connected: connected ?? false}),
+    [socket, clientID, connected],
+  );
+
+  useEffect(() => {
+    const queryParams = {
+      clientID: clientID,
+    };
+
+    const serverURLFromParams = getURLParams().serverURL;
+    const serverURL = serverURLFromParams ?? SERVER_URL_DEFAULT;
+
+    console.log(
+      `Opening socket connection to ${
+        serverURL?.length === 0 ? 'window.location.host' : serverURL
+      } with query params:`,
+      queryParams,
+    );
+
+    const newSocket: Socket = socketIOClient(serverURL, {
+      query: queryParams,
+      // Normally socket.io will fallback to http polling, but we basically never
+      // want that because that'd mean awful performance. It'd be better for the app
+      // to simply break in that case and not connect.
+      transports: ['websocket'],
+      path: '/ws/socket.io'
+    });
+
+    const onServerID = (serverID: string) => {
+      console.debug('Received server ID:', serverID);
+      if (serverIDRef.current != null) {
+        if (serverIDRef.current !== serverID) {
+          console.error(
+            'Server ID changed. Resetting the app using the app key',
+          );
+          setAppResetKey(serverID);
+        }
+      }
+      serverIDRef.current = serverID;
+    };
+
+    newSocket.on('server_id', onServerID);
+
+    setSocket(newSocket);
+
+    return () => {
+      newSocket.off('server_id', onServerID);
+      console.log(
+        'Closing socket connection in the useEffect cleanup function...',
+      );
+      newSocket.disconnect();
+      setSocket(null);
+    };
+  }, [clientID, setAppResetKey]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onAny = (eventName: string, ...args) => {
+        console.debug(`[event: ${eventName}] args:`, ...args);
+      };
+
+      socket.onAny(onAny);
+
+      return () => {
+        socket.offAny(onAny);
+      };
+    }
+    return () => {};
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onConnect = (...args) => {
+        console.debug('Connected to server with args:', ...args);
+        setConnected(true);
+      };
+
+      const onConnectError = (err) => {
+        console.error(`Connection error due to ${err.message}`);
+      };
+
+      const onDisconnect = (reason) => {
+        setConnected(false);
+        console.log(`Disconnected due to ${reason}`);
+      };
+
+      socket.on('connect', onConnect);
+      socket.on('connect_error', onConnectError);
+      socket.on('disconnect', onDisconnect);
+
+      return () => {
+        socket.off('connect', onConnect);
+        socket.off('connect_error', onConnectError);
+        socket.off('disconnect', onDisconnect);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onReconnectError = (err) => {
+        console.log(`Reconnect error due to ${err.message}`);
+      };
+
+      socket.io.on('reconnect_error', onReconnectError);
+
+      const onError = (err) => {
+        console.log(`General socket error with message ${err.message}`);
+      };
+      socket.io.on('error', onError);
+
+      const onReconnect = (attempt) => {
+        console.log(`Reconnected after ${attempt} attempt(s)`);
+      };
+      socket.io.on('reconnect', onReconnect);
+
+      const disconnectOnBeforeUnload = () => {
+        console.log('Disconnecting due to beforeunload event...');
+        socket.disconnect();
+        setSocket(null);
+      };
+      window.addEventListener('beforeunload', disconnectOnBeforeUnload);
+
+      return () => {
+        socket.io.off('reconnect_error', onReconnectError);
+        socket.io.off('error', onError);
+        socket.io.off('reconnect', onReconnect);
+        window.removeEventListener('beforeunload', disconnectOnBeforeUnload);
+      };
+    }
+  }, [clientID, setAppResetKey, socket]);
+
+  /**
+   * Wait to show the disconnected screen on initial app load
+   */
+  useEffect(() => {
+    window.setTimeout(() => {
+      setConnected((prev) => {
+        if (prev === null) {
+          return false;
+        }
+        return prev;
+      });
+    }, INITIAL_DISCONNECT_SCREEN_DELAY);
+  }, []);
+
+  return (
+    <SocketContext.Provider value={socketObject}>
+      {children}
+
+      <Backdrop
+        open={connected === false && willAttemptReconnect === true}
+        sx={{
+          color: '#fff',
+          zIndex: (theme) => theme.zIndex.drawer + 1,
+        }}>
+        <div
+          style={{
+            alignItems: 'center',
+            flexDirection: 'column',
+            textAlign: 'center',
+          }}>
+          <CircularProgress color="inherit" />
+          <Typography
+            align="center"
+            fontSize={{sm: 18, xs: 16}}
+            sx={{
+              fontFamily:
+                'ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace',
+              fontWeight: 'bold',
+            }}>
+            {'Disconnected. Attempting to reconnect...'}
+          </Typography>
+        </div>
+      </Backdrop>
+    </SocketContext.Provider>
+  );
+}

streaming-react-app/src/StreamingInterface.css

@@ -0,0 +1,56 @@
+.app-wrapper-sra {
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+}
+
+.main-container-sra {
+  background-color: white;
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+  text-align: left;
+  margin: 16px;
+  margin-bottom: 36px;
+  border-radius: 8px;
+  box-shadow: 0px 24px 30px rgba(0, 0, 0, 0.3);
+  border: 1px solid rgba(0, 0, 0, 0.05);
+  overflow: hidden;
+}
+
+.top-section-sra {
+  padding-top: 24px;
+  margin-bottom: 24px;
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+}
+
+.horizontal-padding-sra {
+  padding-left: 20px;
+  padding-right: 20px;
+}
+
+.header-container-sra {
+  display: flex;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: center;
+  margin-bottom: 24px;
+}
+
+.header-icon-sra {
+  display: block;
+  margin-right: 12px;
+}
+
+.translation-text-container-sra {
+  background-color: #f8f8f8;
+  padding-top: 12px;
+  padding-bottom: 4px;
+}
+
+.text-chunk-sra {
+  margin-bottom: 12px;
+}

streaming-react-app/src/StreamingInterface.tsx

@@ -0,0 +1,1165 @@
+import {useCallback, useEffect, useLayoutEffect, useRef, useState} from 'react';
+import Button from '@mui/material/Button';
+import Typography from '@mui/material/Typography';
+import InputLabel from '@mui/material/InputLabel';
+import FormControl from '@mui/material/FormControl';
+import Select, {SelectChangeEvent} from '@mui/material/Select';
+import MenuItem from '@mui/material/MenuItem';
+import Stack from '@mui/material/Stack';
+import seamlessLogoUrl from './assets/seamless.svg';
+import {
+  AgentCapabilities,
+  BaseResponse,
+  BrowserAudioStreamConfig,
+  DynamicConfig,
+  PartialDynamicConfig,
+  SUPPORTED_INPUT_SOURCES,
+  SUPPORTED_OUTPUT_MODES,
+  ServerExceptionData,
+  ServerSpeechData,
+  ServerState,
+  ServerTextData,
+  StartStreamEventConfig,
+  StreamingStatus,
+  SupportedInputSource,
+  SupportedOutputMode,
+  TranslationSentences,
+} from './types/StreamingTypes';
+import FormLabel from '@mui/material/FormLabel';
+import RadioGroup from '@mui/material/RadioGroup';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import Radio from '@mui/material/Radio';
+import './StreamingInterface.css';
+import RoomConfig from './RoomConfig';
+import Divider from '@mui/material/Divider';
+import {useSocket} from './useSocket';
+import {RoomState} from './types/RoomState';
+import useStable from './useStable';
+import float32To16BitPCM from './float32To16BitPCM';
+import createBufferedSpeechPlayer from './createBufferedSpeechPlayer';
+import Checkbox from '@mui/material/Checkbox';
+import Alert from '@mui/material/Alert';
+import isScrolledToDocumentBottom from './isScrolledToDocumentBottom';
+import Box from '@mui/material/Box';
+import Slider from '@mui/material/Slider';
+import VolumeDown from '@mui/icons-material/VolumeDown';
+import VolumeUp from '@mui/icons-material/VolumeUp';
+import Mic from '@mui/icons-material/Mic';
+import MicOff from '@mui/icons-material/MicOff';
+import XRDialog from './react-xr/XRDialog';
+import getTranslationSentencesFromReceivedData from './getTranslationSentencesFromReceivedData';
+import {
+  sliceTranslationSentencesUpToIndex,
+  getTotalSentencesLength,
+} from './sliceTranslationSentencesUtils';
+import Blink from './Blink';
+import {CURSOR_BLINK_INTERVAL_MS} from './cursorBlinkInterval';
+import {getURLParams} from './URLParams';
+import debug from './debug';
+import DebugSection from './DebugSection';
+import {Grid} from '@mui/material';
+import {getLanguageFromThreeLetterCode} from './languageLookup';
+
+const AUDIO_STREAM_DEFAULTS: {
+  [key in SupportedInputSource]: BrowserAudioStreamConfig;
+} = {
+  userMedia: {
+    echoCancellation: false,
+    noiseSuppression: true,
+  },
+  displayMedia: {
+    echoCancellation: false,
+    noiseSuppression: false,
+  },
+};
+
+async function requestUserMediaAudioStream(
+  config: BrowserAudioStreamConfig = {
+    echoCancellation: false,
+    noiseSuppression: true,
+  },
+) {
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: {...config, channelCount: 1},
+  });
+  console.debug(
+    '[requestUserMediaAudioStream] stream created with settings:',
+    stream.getAudioTracks()?.[0]?.getSettings(),
+  );
+  return stream;
+}
+
+async function requestDisplayMediaAudioStream(
+  config: BrowserAudioStreamConfig = {
+    echoCancellation: false,
+    noiseSuppression: false,
+  },
+) {
+  const stream = await navigator.mediaDevices.getDisplayMedia({
+    audio: {...config, channelCount: 1},
+  });
+  console.debug(
+    '[requestDisplayMediaAudioStream] stream created with settings:',
+    stream.getAudioTracks()?.[0]?.getSettings(),
+  );
+  return stream;
+}
+
+const buttonLabelMap: {[key in StreamingStatus]: string} = {
+  stopped: 'Start Streaming',
+  running: 'Stop Streaming',
+  starting: 'Starting...',
+};
+
+const BUFFER_LIMIT = 1;
+
+const SCROLLED_TO_BOTTOM_THRESHOLD_PX = 36;
+
+const GAIN_MULTIPLIER_OVER_1 = 3;
+
+const getGainScaledValue = (value) =>
+  value > 1 ? (value - 1) * GAIN_MULTIPLIER_OVER_1 + 1 : value;
+
+const TOTAL_ACTIVE_TRANSCODER_WARNING_THRESHOLD = 2;
+
+const MAX_SERVER_EXCEPTIONS_TRACKED = 500;
+
+export const TYPING_ANIMATION_DELAY_MS = 6;
+
+export default function StreamingInterface() {
+  const urlParams = getURLParams();
+  const debugParam = urlParams.debug;
+  const [animateTextDisplay, setAnimateTextDisplay] = useState<boolean>(
+    urlParams.animateTextDisplay,
+  );
+
+  const socketObject = useSocket();
+  const {socket, clientID} = socketObject;
+
+  const [serverState, setServerState] = useState<ServerState | null>(null);
+  const [agent, setAgent] = useState<AgentCapabilities | null>(null);
+  const model = agent?.name ?? null;
+  const agentsCapabilities: Array<AgentCapabilities> =
+    serverState?.agentsCapabilities ?? [];
+  const currentAgent: AgentCapabilities | null =
+    agentsCapabilities.find((agent) => agent.name === model) ?? null;
+
+  const [serverExceptions, setServerExceptions] = useState<
+    Array<ServerExceptionData>
+  >([]);
+  const [roomState, setRoomState] = useState<RoomState | null>(null);
+  const roomID = roomState?.room_id ?? null;
+  const isSpeaker =
+    (clientID != null && roomState?.speakers.includes(clientID)) ?? false;
+  const isListener =
+    (clientID != null && roomState?.listeners.includes(clientID)) ?? false;
+
+  const [streamingStatus, setStreamingStatus] =
+    useState<StreamingStatus>('stopped');
+
+  const isStreamConfiguredRef = useRef<boolean>(false);
+
+  const [outputMode, setOutputMode] = useState<SupportedOutputMode>('s2s&t');
+  const [inputSource, setInputSource] =
+    useState<SupportedInputSource>('userMedia');
+  const [enableNoiseSuppression, setEnableNoiseSuppression] = useState<
+    boolean | null
+  >(null);
+  const [enableEchoCancellation, setEnableEchoCancellation] = useState<
+    boolean | null
+  >(null);
+
+  // Dynamic Params:
+  const [targetLang, setTargetLang] = useState<string | null>(null);
+
+  const [serverDebugFlag, setServerDebugFlag] = useState<boolean>(
+    debugParam ?? false,
+  );
+
+  const [receivedData, setReceivedData] = useState<Array<ServerTextData>>([]);
+  const [
+    translationSentencesAnimatedIndex,
+    setTranslationSentencesAnimatedIndex,
+  ] = useState<number>(0);
+
+  const lastTranslationResultRef = useRef<HTMLDivElement | null>(null);
+
+  const [inputStream, setInputStream] = useState<MediaStream | null>(null);
+  const [inputStreamSource, setInputStreamSource] =
+    useState<MediaStreamAudioSourceNode | null>(null);
+  const audioContext = useStable<AudioContext>(() => new AudioContext());
+  const [scriptNodeProcessor, setScriptNodeProcessor] =
+    useState<ScriptProcessorNode | null>(null);
+
+  const [muted, setMuted] = useState<boolean>(false);
+  // The onaudioprocess script needs an up-to-date reference to the muted state, so
+  // we use a ref here and keep it in sync via useEffect
+  const mutedRef = useRef<boolean>(muted);
+  useEffect(() => {
+    mutedRef.current = muted;
+  }, [muted]);
+
+  const [gain, setGain] = useState<number>(1);
+
+  const isScrolledToBottomRef = useRef<boolean>(isScrolledToDocumentBottom());
+
+  // Some config options must be set when starting streaming and cannot be chaned dynamically.
+  // This controls whether they are disabled or not
+  const streamFixedConfigOptionsDisabled =
+    streamingStatus !== 'stopped' || roomID == null;
+
+  const bufferedSpeechPlayer = useStable(() => {
+    const player = createBufferedSpeechPlayer({
+      onStarted: () => {
+        console.debug('📢 PLAYBACK STARTED 📢');
+      },
+      onEnded: () => {
+        console.debug('🛑 PLAYBACK ENDED 🛑');
+      },
+    });
+
+    // Start the player now so it eagerly plays audio when it arrives
+    player.start();
+    return player;
+  });
+
+  const translationSentencesBase: TranslationSentences =
+    getTranslationSentencesFromReceivedData(receivedData);
+
+  const translationSentencesBaseTotalLength = getTotalSentencesLength(
+    translationSentencesBase,
+  );
+
+  const translationSentences: TranslationSentences = animateTextDisplay
+    ? sliceTranslationSentencesUpToIndex(
+        translationSentencesBase,
+        translationSentencesAnimatedIndex,
+      )
+    : translationSentencesBase;
+
+  // We want the blinking cursor to show before any text has arrived, so let's add an empty string so that the cursor shows up
+  const translationSentencesWithEmptyStartingString =
+    streamingStatus === 'running' && translationSentences.length === 0
+      ? ['']
+      : translationSentences;
+
+  /******************************************
+   * Event Handlers
+   ******************************************/
+
+  const setAgentAndUpdateParams = useCallback(
+    (newAgent: AgentCapabilities | null) => {
+      setAgent((prevAgent) => {
+        if (prevAgent?.name !== newAgent?.name) {
+          setTargetLang(newAgent?.targetLangs[0] ?? null);
+        }
+        return newAgent;
+      });
+    },
+    [],
+  );
+
+  const onSetDynamicConfig = useCallback(
+    async (partialConfig: PartialDynamicConfig) => {
+      return new Promise<void>((resolve, reject) => {
+        if (socket == null) {
+          reject(new Error('[onSetDynamicConfig] socket is null '));
+          return;
+        }
+
+        socket.emit(
+          'set_dynamic_config',
+          partialConfig,
+          (result: BaseResponse) => {
+            console.log('[emit result: set_dynamic_config]', result);
+            if (result.status === 'ok') {
+              resolve();
+            } else {
+              reject();
+            }
+          },
+        );
+      });
+    },
+    [socket],
+  );
+
+  const configureStreamAsync = ({sampleRate}: {sampleRate: number}) => {
+    return new Promise<void>((resolve, reject) => {
+      if (socket == null) {
+        reject(new Error('[configureStreamAsync] socket is null '));
+        return;
+      }
+      const modelName = agent?.name ?? null;
+      if (modelName == null) {
+        reject(new Error('[configureStreamAsync] modelName is null '));
+        return;
+      }
+
+      const config: StartStreamEventConfig = {
+        event: 'config',
+        rate: sampleRate,
+        model_name: modelName,
+        debug: serverDebugFlag,
+        // synchronous processing isn't implemented on the v2 pubsub server, so hardcode this to true
+        async_processing: true,
+        buffer_limit: BUFFER_LIMIT,
+        model_type: outputMode,
+      };
+
+      console.log('[configureStreamAsync] sending config', config);
+
+      socket.emit('configure_stream', config, (statusObject) => {
+        if (statusObject.status === 'ok') {
+          isStreamConfiguredRef.current = true;
+          console.debug(
+            '[configureStreamAsync] stream configured!',
+            statusObject,
+          );
+          resolve();
+        } else {
+          isStreamConfiguredRef.current = false;
+          reject(
+            new Error(
+              `[configureStreamAsync] configure_stream returned status: ${statusObject.status}`,
+            ),
+          );
+          return;
+        }
+      });
+    });
+  };
+
+  const startStreaming = async () => {
+    if (streamingStatus !== 'stopped') {
+      console.warn(
+        `Attempting to start stream when status is ${streamingStatus}`,
+      );
+      return;
+    }
+
+    setStreamingStatus('starting');
+
+    if (audioContext.state === 'suspended') {
+      console.warn('audioContext was suspended! resuming...');
+      await audioContext.resume();
+    }
+
+    let stream: MediaStream | null = null;
+
+    try {
+      if (inputSource === 'userMedia') {
+        stream = await requestUserMediaAudioStream({
+          noiseSuppression:
+            enableNoiseSuppression ??
+            AUDIO_STREAM_DEFAULTS['userMedia'].noiseSuppression,
+          echoCancellation:
+            enableEchoCancellation ??
+            AUDIO_STREAM_DEFAULTS['userMedia'].echoCancellation,
+        });
+      } else if (inputSource === 'displayMedia') {
+        stream = await requestDisplayMediaAudioStream({
+          noiseSuppression:
+            enableNoiseSuppression ??
+            AUDIO_STREAM_DEFAULTS['displayMedia'].noiseSuppression,
+          echoCancellation:
+            enableEchoCancellation ??
+            AUDIO_STREAM_DEFAULTS['displayMedia'].echoCancellation,
+        });
+      } else {
+        throw new Error(`Unsupported input source requested: ${inputSource}`);
+      }
+      setInputStream(stream);
+    } catch (e) {
+      console.error('[startStreaming] media stream request failed:', e);
+      setStreamingStatus('stopped');
+      return;
+    }
+
+    const mediaStreamSource = audioContext.createMediaStreamSource(stream);
+    setInputStreamSource(mediaStreamSource);
+    /**
+     * NOTE: This currently uses a deprecated way of processing the audio (createScriptProcessor), but
+     * which is easy and convenient for our purposes.
+     *
+     * Documentation for the deprecated way of doing it is here: https://developer.mozilla.org/en-US/docs/Web/API/BaseAudioContext/createScriptProcessor
+     *
+     * In an ideal world this would be migrated to something like this SO answer: https://stackoverflow.com/a/65448287
+     */
+    const scriptProcessor = audioContext.createScriptProcessor(16384, 1, 1);
+    setScriptNodeProcessor(scriptProcessor);
+
+    scriptProcessor.onaudioprocess = (event) => {
+      if (isStreamConfiguredRef.current === false) {
+        console.debug('[onaudioprocess] stream is not configured yet!');
+        return;
+      }
+      if (socket == null) {
+        console.warn('[onaudioprocess] socket is null in onaudioprocess');
+        return;
+      }
+
+      if (mutedRef.current) {
+        // We still want to send audio to the server when we're muted to ensure we
+        // get any remaining audio back from the server, so let's pass an array length 1 with a value of 0
+        const mostlyEmptyInt16Array = new Int16Array(1);
+        socket.emit('incoming_audio', mostlyEmptyInt16Array);
+      } else {
+        const float32Audio = event.inputBuffer.getChannelData(0);
+        const pcm16Audio = float32To16BitPCM(float32Audio);
+        socket.emit('incoming_audio', pcm16Audio);
+      }
+
+      debug()?.sentAudio(event);
+    };
+
+    mediaStreamSource.connect(scriptProcessor);
+    scriptProcessor.connect(audioContext.destination);
+
+    bufferedSpeechPlayer.start();
+
+    try {
+      if (targetLang == null) {
+        throw new Error('[startStreaming] targetLang cannot be nullish');
+      }
+
+      // When we are starting the stream we want to pass all the dynamic config values
+      // available before actually configuring and starting the stream
+      const fullDynamicConfig: DynamicConfig = {
+        targetLanguage: targetLang,
+      };
+
+      await onSetDynamicConfig(fullDynamicConfig);
+
+      // NOTE: this needs to be the *audioContext* sample rate, not the sample rate of the input stream. Not entirely sure why.
+      await configureStreamAsync({
+        sampleRate: audioContext.sampleRate,
+      });
+    } catch (e) {
+      console.error('configureStreamAsync failed', e);
+      setStreamingStatus('stopped');
+      return;
+    }
+
+    setStreamingStatus('running');
+  };
+
+  const stopStreaming = useCallback(async () => {
+    if (streamingStatus === 'stopped') {
+      console.warn(
+        `Attempting to stop stream when status is ${streamingStatus}`,
+      );
+      return;
+    }
+
+    // Stop the speech playback right away
+    bufferedSpeechPlayer.stop();
+
+    if (inputStreamSource == null || scriptNodeProcessor == null) {
+      console.error(
+        'inputStreamSource || scriptNodeProcessor is null in stopStreaming',
+      );
+    } else {
+      inputStreamSource.disconnect(scriptNodeProcessor);
+      scriptNodeProcessor.disconnect(audioContext.destination);
+
+      // Release the mic input so we stop showing the red recording icon in the browser
+      inputStream?.getTracks().forEach((track) => track.stop());
+    }
+
+    if (socket == null) {
+      console.warn('Unable to emit stop_stream because socket is null');
+    } else {
+      socket.emit('stop_stream', (result) => {
+        console.debug('[emit result: stop_stream]', result);
+      });
+    }
+
+    setStreamingStatus('stopped');
+  }, [
+    audioContext.destination,
+    bufferedSpeechPlayer,
+    inputStream,
+    inputStreamSource,
+    scriptNodeProcessor,
+    socket,
+    streamingStatus,
+  ]);
+
+  const onClearTranscriptForAll = useCallback(() => {
+    if (socket != null) {
+      socket.emit('clear_transcript_for_all');
+    }
+  }, [socket]);
+
+  /******************************************
+   * Effects
+   ******************************************/
+
+  useEffect(() => {
+    if (socket == null) {
+      return;
+    }
+
+    const onRoomStateUpdate = (roomState: RoomState) => {
+      setRoomState(roomState);
+    };
+
+    socket.on('room_state_update', onRoomStateUpdate);
+
+    return () => {
+      socket.off('room_state_update', onRoomStateUpdate);
+    };
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onTranslationText = (data: ServerTextData) => {
+        setReceivedData((prev) => [...prev, data]);
+        debug()?.receivedText(data.payload);
+      };
+
+      const onTranslationSpeech = (data: ServerSpeechData) => {
+        bufferedSpeechPlayer.addAudioToBuffer(data.payload, data.sample_rate);
+      };
+
+      socket.on('translation_text', onTranslationText);
+      socket.on('translation_speech', onTranslationSpeech);
+
+      return () => {
+        socket.off('translation_text', onTranslationText);
+        socket.off('translation_speech', onTranslationSpeech);
+      };
+    }
+  }, [bufferedSpeechPlayer, socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onServerStateUpdate = (newServerState: ServerState) => {
+        setServerState(newServerState);
+
+        // If a client creates a server lock, we want to stop streaming if we're not them
+        if (
+          newServerState.serverLock?.isActive === true &&
+          newServerState.serverLock?.clientID !== clientID &&
+          streamingStatus === 'running'
+        ) {
+          stopStreaming();
+        }
+
+        const firstAgentNullable = newServerState.agentsCapabilities[0];
+        if (agent == null && firstAgentNullable != null) {
+          setAgentAndUpdateParams(firstAgentNullable);
+        }
+      };
+
+      socket.on('server_state_update', onServerStateUpdate);
+
+      return () => {
+        socket.off('server_state_update', onServerStateUpdate);
+      };
+    }
+  }, [
+    agent,
+    clientID,
+    setAgentAndUpdateParams,
+    socket,
+    stopStreaming,
+    streamingStatus,
+  ]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onServerException = (
+        exceptionDataWithoutClientTime: ServerExceptionData,
+      ) => {
+        const exceptionData = {
+          ...exceptionDataWithoutClientTime,
+          timeStringClient: new Date(
+            exceptionDataWithoutClientTime['timeEpochMs'],
+          ).toLocaleString(),
+        };
+
+        setServerExceptions((prev) =>
+          [exceptionData, ...prev].slice(0, MAX_SERVER_EXCEPTIONS_TRACKED),
+        );
+        console.error(
+          `[server_exception] The server encountered an exception: ${exceptionData['message']}`,
+          exceptionData,
+        );
+      };
+
+      socket.on('server_exception', onServerException);
+
+      return () => {
+        socket.off('server_exception', onServerException);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onClearTranscript = () => {
+        setReceivedData([]);
+        setTranslationSentencesAnimatedIndex(0);
+      };
+
+      socket.on('clear_transcript', onClearTranscript);
+
+      return () => {
+        socket.off('clear_transcript', onClearTranscript);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    const onScroll = () => {
+      if (isScrolledToDocumentBottom(SCROLLED_TO_BOTTOM_THRESHOLD_PX)) {
+        isScrolledToBottomRef.current = true;
+        return;
+      }
+      isScrolledToBottomRef.current = false;
+      return;
+    };
+
+    document.addEventListener('scroll', onScroll);
+
+    return () => {
+      document.removeEventListener('scroll', onScroll);
+    };
+  }, []);
+
+  useLayoutEffect(() => {
+    if (
+      lastTranslationResultRef.current != null &&
+      isScrolledToBottomRef.current
+    ) {
+      // Scroll the div to the most recent entry
+      lastTranslationResultRef.current.scrollIntoView();
+    }
+    // Run the effect every time data is received, so that
+    // we scroll to the bottom even if we're just adding text to
+    // a pre-existing chunk
+  }, [receivedData]);
+
+  useEffect(() => {
+    if (!animateTextDisplay) {
+      return;
+    }
+
+    if (
+      translationSentencesAnimatedIndex < translationSentencesBaseTotalLength
+    ) {
+      const timeout = setTimeout(() => {
+        setTranslationSentencesAnimatedIndex((prev) => prev + 1);
+        debug()?.startRenderText();
+      }, TYPING_ANIMATION_DELAY_MS);
+
+      return () => clearTimeout(timeout);
+    } else {
+      debug()?.endRenderText();
+    }
+  }, [
+    animateTextDisplay,
+    translationSentencesAnimatedIndex,
+    translationSentencesBaseTotalLength,
+  ]);
+
+  /******************************************
+   * Sub-components
+   ******************************************/
+
+  const volumeSliderNode = (
+    <Stack
+      spacing={2}
+      direction="row"
+      sx={{mb: 1, width: '100%'}}
+      alignItems="center">
+      <VolumeDown color="primary" />
+      <Slider
+        aria-label="Volume"
+        defaultValue={1}
+        scale={getGainScaledValue}
+        min={0}
+        max={3}
+        step={0.1}
+        marks={[
+          {value: 0, label: '0%'},
+          {value: 1, label: '100%'},
+          {value: 2, label: '400%'},
+          {value: 3, label: '700%'},
+        ]}
+        valueLabelFormat={(value) => `${(value * 100).toFixed(0)}%`}
+        valueLabelDisplay="auto"
+        value={gain}
+        onChange={(_event: Event, newValue: number | number[]) => {
+          if (typeof newValue === 'number') {
+            const scaledGain = getGainScaledValue(newValue);
+            // We want the actual gain node to use the scaled value
+            bufferedSpeechPlayer.setGain(scaledGain);
+            // But we want react state to keep track of the non-scaled value
+            setGain(newValue);
+          } else {
+            console.error(
+              `[volume slider] Unexpected non-number value: ${newValue}`,
+            );
+          }
+        }}
+      />
+      <VolumeUp color="primary" />
+    </Stack>
+  );
+
+  const xrDialogComponent = (
+    <XRDialog
+      animateTextDisplay={
+        animateTextDisplay &&
+        translationSentencesAnimatedIndex == translationSentencesBaseTotalLength
+      }
+      bufferedSpeechPlayer={bufferedSpeechPlayer}
+      translationSentences={translationSentences}
+      roomState={roomState}
+      roomID={roomID}
+      startStreaming={startStreaming}
+      stopStreaming={stopStreaming}
+      debugParam={debugParam}
+      onARHidden={() => {
+        setAnimateTextDisplay(urlParams.animateTextDisplay);
+      }}
+      onARVisible={() => setAnimateTextDisplay(false)}
+    />
+  );
+
+  return (
+    <div className="app-wrapper-sra">
+      <Box
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore Not sure why it's complaining about complexity here
+        sx={{width: '100%', maxWidth: '660px', minWidth: '320px'}}>
+        <div className="main-container-sra">
+          <div className="top-section-sra horizontal-padding-sra">
+            <div className="header-container-sra">
+              <img
+                src={seamlessLogoUrl}
+                className="header-icon-sra"
+                alt="Seamless Translation Logo"
+                height={24}
+                width={24}
+              />
+
+              <div>
+                <Typography variant="h1" sx={{color: '#65676B'}}>
+                  Seamless Translation
+                </Typography>
+              </div>
+            </div>
+            <div className="header-container-sra">
+              <div>
+                <Typography variant="body2" sx={{color: '#65676B'}}>
+                  Welcome! Join a room as speaker or listener (or both), and share the 
+                  room code to invite listeners.
+                  <br/>
+                  SeamlessStreaming model is a research model and is not released
+                  for production deployment. The streaming quality is closely 
+                  related to proper VAD segmentation. It works best if you pause 
+                  every couple of sentences, or you may wish adjust the VAD threshold
+                  in the model config.
+                </Typography>
+              </div>
+            </div>
+            <Stack spacing="22px" direction="column">
+              <Box>
+                <RoomConfig
+                  roomState={roomState}
+                  serverState={serverState}
+                  streamingStatus={streamingStatus}
+                  onJoinRoomOrUpdateRoles={() => {
+                    // If the user has switched from speaker to listener we need to tell the
+                    // player to play eagerly, since currently the listener doesn't have any stop/start controls
+                    bufferedSpeechPlayer.start();
+                  }}
+                />
+
+                {isListener && !isSpeaker && (
+                  <Box
+                    sx={{
+                      paddingX: 6,
+                      paddingBottom: 2,
+                      marginY: 2,
+                      display: 'flex',
+                      flexDirection: 'column',
+                      alignItems: 'center',
+                    }}>
+                    {volumeSliderNode}
+                  </Box>
+                )}
+              </Box>
+
+              {isSpeaker && (
+                <>
+                  <Divider />
+
+                  <Stack spacing="12px" direction="column">
+                    <FormLabel id="output-modes-radio-group-label">
+                      Model
+                    </FormLabel>
+                    <FormControl
+                      disabled={
+                        streamFixedConfigOptionsDisabled ||
+                        agentsCapabilities.length === 0
+                      }
+                      fullWidth
+                      sx={{minWidth: '14em'}}>
+                      <InputLabel id="model-selector-input-label">
+                        Model
+                      </InputLabel>
+                      <Select
+                        labelId="model-selector-input-label"
+                        label="Model"
+                        onChange={(e: SelectChangeEvent) => {
+                          const newAgent =
+                            agentsCapabilities.find(
+                              (agent) => e.target.value === agent.name,
+                            ) ?? null;
+                          if (newAgent == null) {
+                            console.error(
+                              'Unable to find agent with name',
+                              e.target.value,
+                            );
+                          }
+                          setAgentAndUpdateParams(newAgent);
+                        }}
+                        value={model ?? ''}>
+                        {agentsCapabilities.map((agent) => (
+                          <MenuItem value={agent.name} key={agent.name}>
+                            {agent.name}
+                          </MenuItem>
+                        ))}
+                      </Select>
+                    </FormControl>
+
+                  </Stack>
+
+                  <Stack spacing={0.5}>
+                    <FormLabel id="output-modes-radio-group-label">
+                      Output
+                    </FormLabel>
+
+                    <Box sx={{paddingTop: 2, paddingBottom: 1}}>
+                      <FormControl fullWidth sx={{minWidth: '14em'}}>
+                        <InputLabel id="target-selector-input-label">
+                          Target Language
+                        </InputLabel>
+                        <Select
+                          labelId="target-selector-input-label"
+                          label="Target Language"
+                          onChange={(e: SelectChangeEvent) => {
+                            setTargetLang(e.target.value);
+                            onSetDynamicConfig({
+                              targetLanguage: e.target.value,
+                            });
+                          }}
+                          value={targetLang ?? ''}>
+                          {currentAgent?.targetLangs.map((langCode) => (
+                            <MenuItem value={langCode} key={langCode}>
+                              {getLanguageFromThreeLetterCode(langCode) != null
+                                ? `${getLanguageFromThreeLetterCode(
+                                    langCode,
+                                  )} (${langCode})`
+                                : langCode}
+                            </MenuItem>
+                          ))}
+                        </Select>
+                      </FormControl>
+                    </Box>
+
+                    <Grid container>
+                      <Grid item xs={12} sm={4}>
+                        <FormControl
+                          disabled={streamFixedConfigOptionsDisabled}>
+                          <RadioGroup
+                            aria-labelledby="output-modes-radio-group-label"
+                            value={outputMode}
+                            onChange={(e) =>
+                              setOutputMode(
+                                e.target.value as SupportedOutputMode,
+                              )
+                            }
+                            name="output-modes-radio-buttons-group">
+                            {
+                              // TODO: Use supported modalities from agentCapabilities
+                              SUPPORTED_OUTPUT_MODES.map(({value, label}) => (
+                                <FormControlLabel
+                                  key={value}
+                                  value={value}
+                                  control={<Radio />}
+                                  label={label}
+                                />
+                              ))
+                            }
+                          </RadioGroup>
+                        </FormControl>
+                      </Grid>
+
+                      <Grid item xs={12} sm={8}>
+                        <Stack
+                          direction="column"
+                          spacing={1}
+                          alignItems="flex-start"
+                          sx={{flexGrow: 1}}>
+                          {isListener && (
+                            <Box
+                              sx={{
+                                flexGrow: 1,
+                                paddingX: 1.5,
+                                paddingY: 1.5,
+                                width: '100%',
+                              }}>
+                              {volumeSliderNode}
+                            </Box>
+                          )}
+                        </Stack>
+                      </Grid>
+                    </Grid>
+                  </Stack>
+
+                  <Typography variant="body2" sx={{color: '#65676B'}}>
+                    Note: we don't recommend echo cancellation, as it may distort 
+                    the input audio (dropping words/sentences) if there is output 
+                    audio playing. Instead, you should use headphones if you'd like 
+                    to listen to the output audio while speaking.
+                  </Typography>
+
+                  <Stack
+                    direction="row"
+                    spacing={2}
+                    justifyContent="space-between">
+                    <Box sx={{flex: 1}}>
+                      <FormControl disabled={streamFixedConfigOptionsDisabled}>
+                        <FormLabel id="input-source-radio-group-label">
+                          Input Source
+                        </FormLabel>
+                        <RadioGroup
+                          aria-labelledby="input-source-radio-group-label"
+                          value={inputSource}
+                          onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
+                            setInputSource(
+                              e.target.value as SupportedInputSource,
+                            )
+                          }
+                          name="input-source-radio-buttons-group">
+                          {SUPPORTED_INPUT_SOURCES.map(({label, value}) => (
+                            <FormControlLabel
+                              key={value}
+                              value={value}
+                              control={<Radio />}
+                              label={label}
+                            />
+                          ))}
+                        </RadioGroup>
+                      </FormControl>
+                    </Box>
+                    <Box sx={{flex: 1}}>
+                      <FormControl disabled={streamFixedConfigOptionsDisabled}>
+                        <FormLabel>Options</FormLabel>
+                        <FormControlLabel
+                          control={
+                            <Checkbox
+                              checked={
+                                enableNoiseSuppression ??
+                                AUDIO_STREAM_DEFAULTS[inputSource]
+                                  .noiseSuppression
+                              }
+                              onChange={(
+                                event: React.ChangeEvent<HTMLInputElement>,
+                              ) =>
+                                setEnableNoiseSuppression(event.target.checked)
+                              }
+                            />
+                          }
+                          label="Noise Suppression (Browser)"
+                        />
+                                                <FormControlLabel
+                          control={
+                            <Checkbox
+                              checked={
+                                enableEchoCancellation ??
+                                AUDIO_STREAM_DEFAULTS[inputSource]
+                                  .echoCancellation
+                              }
+                              onChange={(
+                                event: React.ChangeEvent<HTMLInputElement>,
+                              ) =>
+                                setEnableEchoCancellation(event.target.checked)
+                              }
+                            />
+                          }
+                          label="Echo Cancellation (Browser)"
+                        />
+                        <FormControlLabel
+                          control={
+                            <Checkbox
+                              checked={serverDebugFlag}
+                              onChange={(
+                                event: React.ChangeEvent<HTMLInputElement>,
+                              ) => setServerDebugFlag(event.target.checked)}
+                            />
+                          }
+                          label="Server Debug Flag"
+                        />
+                      </FormControl>
+                    </Box>
+                  </Stack>
+
+                  <Stack direction="row" spacing={2}>
+                    {streamingStatus === 'stopped' ? (
+                      <Button
+                        variant="contained"
+                        onClick={startStreaming}
+                        disabled={
+                          roomID == null ||
+                          // Prevent users from starting streaming if there is a server lock with an active session
+                          (serverState?.serverLock?.isActive === true &&
+                            serverState.serverLock.clientID !== clientID)
+                        }>
+                        {buttonLabelMap[streamingStatus]}
+                      </Button>
+                    ) : (
+                      <Button
+                        variant="contained"
+                        color={
+                          streamingStatus === 'running' ? 'error' : 'primary'
+                        }
+                        disabled={
+                          streamingStatus === 'starting' || roomID == null
+                        }
+                        onClick={stopStreaming}>
+                        {buttonLabelMap[streamingStatus]}
+                      </Button>
+                    )}
+
+                    <Box>
+                      <Button
+                        variant="contained"
+                        aria-label={muted ? 'Unmute' : 'Mute'}
+                        color={muted ? 'info' : 'primary'}
+                        onClick={() => setMuted((prev) => !prev)}
+                        sx={{
+                          borderRadius: 100,
+                          paddingX: 0,
+                          minWidth: '36px',
+                        }}>
+                        {muted ? <MicOff /> : <Mic />}
+                      </Button>
+                    </Box>
+
+                    {roomID == null ? null : (
+                      <Box
+                        sx={{
+                          flexGrow: 1,
+                          display: 'flex',
+                          justifyContent: 'flex-end',
+                        }}>
+                        {xrDialogComponent}
+                      </Box>
+                    )}
+                  </Stack>
+
+                  {serverExceptions.length > 0 && (
+                    <div>
+                      <Alert severity="error">
+                        {`The server encountered an exception. See the browser console for details. You may need to refresh the page to continue using the app.`}
+                      </Alert>
+                    </div>
+                  )}
+
+                  {serverState != null &&
+                    serverState.totalActiveTranscoders >=
+                      TOTAL_ACTIVE_TRANSCODER_WARNING_THRESHOLD && (
+                      <div>
+                        <Alert severity="warning">
+                          {`The server currently has ${serverState?.totalActiveTranscoders} active streaming sessions. Performance may be degraded.`}
+                        </Alert>
+                      </div>
+                    )}
+
+                  {serverState?.serverLock != null &&
+                    serverState.serverLock.clientID !== clientID && (
+                      <div>
+                        <Alert severity="warning">
+                          {`The server is currently locked by "${serverState.serverLock.name}". Priority will be given to that client when they are streaming, and your streaming session may be halted abruptly.`}
+                        </Alert>
+                      </div>
+                    )}
+                </>
+              )}
+            </Stack>
+
+            {isListener && !isSpeaker && (
+              <Box sx={{marginBottom: 1, marginTop: 2}}>
+                {xrDialogComponent}
+              </Box>
+            )}
+          </div>
+
+          {debugParam && roomID != null && <DebugSection />}
+
+          <div className="translation-text-container-sra horizontal-padding-sra">
+            <Stack
+              direction="row"
+              spacing={2}
+              sx={{mb: '16px', alignItems: 'center'}}>
+              <Typography variant="h1" sx={{fontWeight: 700, flexGrow: 1}}>
+                Transcript
+              </Typography>
+              {isSpeaker && (
+                <Button
+                  variant="text"
+                  size="small"
+                  onClick={onClearTranscriptForAll}>
+                  Clear Transcript for All
+                </Button>
+              )}
+            </Stack>
+            <Stack direction="row">
+              <div className="translation-text-sra">
+                {translationSentencesWithEmptyStartingString.map(
+                  (sentence, index, arr) => {
+                    const isLast = index === arr.length - 1;
+                    const maybeRef = isLast
+                      ? {ref: lastTranslationResultRef}
+                      : {};
+                    return (
+                      <div className="text-chunk-sra" key={index} {...maybeRef}>
+                        <Typography variant="body1">
+                          {sentence}
+                          {animateTextDisplay && isLast && (
+                            <Blink
+                              intervalMs={CURSOR_BLINK_INTERVAL_MS}
+                              shouldBlink={
+                                (roomState?.activeTranscoders ?? 0) > 0
+                              }>
+                              <Typography
+                                component="span"
+                                variant="body1"
+                                sx={{
+                                  display: 'inline-block',
+                                  transform: 'scaleY(1.25) translateY(-1px)',
+                                }}>
+                                {'|'}
+                              </Typography>
+                            </Blink>
+                          )}
+                        </Typography>
+                      </div>
+                    );
+                  },
+                )}
+              </div>
+            </Stack>
+          </div>
+        </div>
+      </Box>
+    </div>
+  );
+}

streaming-react-app/src/URLParams.ts

@@ -0,0 +1,50 @@
+import { getBooleanParamFlag, getStringParamFlag } from './getParamFlag';
+import { URLParamsObject } from './types/URLParamsTypes';
+
+/**
+ * These are the URL parameters you can provide to the app to change its behavior.
+ *
+ * Boolean flags can be set by just providing the flag name (`?autoJoin`), or by
+ * explicitly setting it to 1 (true) or 0 (false): `?autoJoin=1` or `?autoJoin=0`
+ *
+ * String flags require an explicit value: `?roomID=ABCD`
+ *
+ * Examples:
+ *
+ * - `http://localhost:5173/?roomID=BBCD&autoJoin&debug`
+ * - `http://localhost:5173/?serverURL=localhost:8000`
+
+ * @returns
+ */
+
+export function getURLParams(): URLParamsObject {
+  return {
+    // animate the translation text when it arrives, typing it out one letter at a time
+    animateTextDisplay: getBooleanParamFlag('animateTextDisplay', true), // default to true;
+
+    // automatically join the room when the app loads. requires roomID to be set via url param as well
+    autoJoin: getBooleanParamFlag('autoJoin', false),
+
+    // automatically check the server debug flag as true
+    debug: getBooleanParamFlag('debug', false),
+
+    // Enable UI on the client that allows locking out other users of the server when it's being used for high profile demos
+    // NOTE: There is an escape hatch for disabling a server lock by setting the name field to remove_server_lock
+    enableServerLock: getBooleanParamFlag('enableServerLock', false),
+
+    // Pre-populate the Room Code field with the provided roomID. Can be used in conjunction with autoJoin to jump straight into the room
+    roomID: getStringParamFlag('roomID'),
+
+    // Use an alternate server URL as the streaming server (useful for pointing to dev servers: http://localhost:5173/?serverURL=localhost:8000)
+    serverURL: getStringParamFlag('serverURL'),
+
+    // Skip the popup dialog that displays within VR, which is mostly redundant with the web based dialog
+    skipARIntro: getBooleanParamFlag('skipARIntro', true), // default to true
+
+    // Shows the translation text in AR in front of an opaque panel covering all the text area
+    // single_block = original single text block with background
+    // lines = each line is a separate block and animates
+    // lines_with_background = adds a panel behind lines
+    ARTranscriptionType: getStringParamFlag('ARTranscriptionType') || 'lines',
+  };
+}

streaming-react-app/src/createBufferedSpeechPlayer.ts

@@ -0,0 +1,173 @@
+import debug from './debug';
+
+type AddAudioToBufferFunction = (
+  samples: Array<number>,
+  sampleRate: number,
+) => void;
+
+export type BufferedSpeechPlayer = {
+  addAudioToBuffer: AddAudioToBufferFunction;
+  setGain: (gain: number) => void;
+  start: () => void;
+  stop: () => void;
+};
+
+type Options = {
+  onEnded?: () => void;
+  onStarted?: () => void;
+};
+
+export default function createBufferedSpeechPlayer({
+  onStarted,
+  onEnded,
+}: Options): BufferedSpeechPlayer {
+  const audioContext = new AudioContext();
+  const gainNode = audioContext.createGain();
+  gainNode.connect(audioContext.destination);
+
+  let unplayedAudioBuffers: Array<AudioBuffer> = [];
+
+  let currentPlayingBufferSource: AudioBufferSourceNode | null = null;
+
+  let isPlaying = false;
+
+  // This means that the player starts in the 'stopped' state, and you need to call player.start() for it to start playing
+  let shouldPlayWhenAudioAvailable = false;
+
+  const setGain = (gain: number) => {
+    gainNode.gain.setValueAtTime(gain, audioContext.currentTime);
+  };
+
+  const start = () => {
+    shouldPlayWhenAudioAvailable = true;
+    debug()?.start();
+    playNextBufferIfNotAlreadyPlaying();
+  };
+
+  // Stop will stop the audio and clear the buffers
+  const stop = () => {
+    shouldPlayWhenAudioAvailable = false;
+
+    // Stop the current buffers
+    currentPlayingBufferSource?.stop();
+    currentPlayingBufferSource = null;
+
+    unplayedAudioBuffers = [];
+
+    onEnded != null && onEnded();
+    isPlaying = false;
+    return;
+  };
+
+  const playNextBufferIfNotAlreadyPlaying = () => {
+    if (!isPlaying) {
+      playNextBuffer();
+    }
+  };
+
+  const playNextBuffer = () => {
+    if (shouldPlayWhenAudioAvailable === false) {
+      console.debug(
+        '[BufferedSpeechPlayer][playNextBuffer] Not playing any more audio because shouldPlayWhenAudioAvailable is false.',
+      );
+      // NOTE: we do not need to set isPlaying = false or call onEnded because that will be handled in the stop() function
+      return;
+    }
+    if (unplayedAudioBuffers.length === 0) {
+      console.debug(
+        '[BufferedSpeechPlayer][playNextBuffer] No buffers to play.',
+      );
+      if (isPlaying) {
+        isPlaying = false;
+        onEnded != null && onEnded();
+      }
+      return;
+    }
+
+    // If isPlaying is false, then we are starting playback fresh rather than continuing it, and should call onStarted
+    if (isPlaying === false) {
+      isPlaying = true;
+      onStarted != null && onStarted();
+    }
+
+    const source = audioContext.createBufferSource();
+
+    // Get the first unplayed buffer from the array, and remove it from the array
+    const buffer = unplayedAudioBuffers.shift() ?? null;
+    source.buffer = buffer;
+    console.debug(
+      `[BufferedSpeechPlayer] Playing buffer with ${source.buffer?.length} samples`,
+    );
+
+    source.connect(gainNode);
+
+    const startTime = new Date().getTime();
+    source.start();
+    currentPlayingBufferSource = source;
+    // This is probably not necessary, but it doesn't hurt
+    isPlaying = true;
+
+    // TODO: consider changing this to a while loop to avoid deep recursion
+    const onThisBufferPlaybackEnded = () => {
+      console.debug(
+        `[BufferedSpeechPlayer] Buffer with ${source.buffer?.length} samples ended.`,
+      );
+      source.removeEventListener('ended', onThisBufferPlaybackEnded);
+      const endTime = new Date().getTime();
+      debug()?.playedAudio(startTime, endTime, buffer);
+      currentPlayingBufferSource = null;
+
+      // We don't set isPlaying = false here because we are attempting to continue playing. It will get set to false if there are no more buffers to play
+      playNextBuffer();
+    };
+
+    source.addEventListener('ended', onThisBufferPlaybackEnded);
+  };
+
+  const addAudioToBuffer: AddAudioToBufferFunction = (samples, sampleRate) => {
+    const incomingArrayBufferChunk = audioContext.createBuffer(
+      // 1 channel
+      1,
+      samples.length,
+      sampleRate,
+    );
+
+    incomingArrayBufferChunk.copyToChannel(
+      new Float32Array(samples),
+      // first channel
+      0,
+    );
+
+    console.debug(
+      `[addAudioToBufferAndPlay] Adding buffer with ${incomingArrayBufferChunk.length} samples to queue.`,
+    );
+
+    unplayedAudioBuffers.push(incomingArrayBufferChunk);
+    debug()?.receivedAudio(
+      incomingArrayBufferChunk.length / incomingArrayBufferChunk.sampleRate,
+    );
+    const audioBuffersTableInfo = unplayedAudioBuffers.map((buffer, i) => {
+      return {
+        index: i,
+        duration: buffer.length / buffer.sampleRate,
+        samples: buffer.length,
+      };
+    });
+    const totalUnplayedDuration = unplayedAudioBuffers.reduce((acc, buffer) => {
+      return acc + buffer.length / buffer.sampleRate;
+    }, 0);
+
+    console.debug(
+      `[addAudioToBufferAndPlay] Current state of incoming audio buffers (${totalUnplayedDuration.toFixed(
+        1,
+      )}s unplayed):`,
+    );
+    console.table(audioBuffersTableInfo);
+
+    if (shouldPlayWhenAudioAvailable) {
+      playNextBufferIfNotAlreadyPlaying();
+    }
+  };
+
+  return {addAudioToBuffer, setGain, stop, start};
+}

streaming-react-app/src/cursorBlinkInterval.ts

@@ -0,0 +1 @@
+export const CURSOR_BLINK_INTERVAL_MS = 500;

streaming-react-app/src/debug.ts

@@ -0,0 +1,257 @@
+import {TYPING_ANIMATION_DELAY_MS} from './StreamingInterface';
+import {getURLParams} from './URLParams';
+import audioBuffertoWav from 'audiobuffer-to-wav';
+import './StreamingInterface.css';
+
+type StartEndTime = {
+  start: number;
+  end: number;
+};
+
+type StartEndTimeWithAudio = StartEndTime & {
+  float32Audio: Float32Array;
+};
+
+type Text = {
+  time: number;
+  chars: number;
+};
+
+type DebugTimings = {
+  receivedAudio: StartEndTime[];
+  playedAudio: StartEndTimeWithAudio[];
+  receivedText: Text[];
+  renderedText: StartEndTime[];
+  sentAudio: StartEndTimeWithAudio[];
+  startRenderTextTime: number | null;
+  startRecordingTime: number | null;
+  receivedAudioSampleRate: number | null;
+};
+
+function getInitialTimings(): DebugTimings {
+  return {
+    receivedAudio: [],
+    playedAudio: [],
+    receivedText: [],
+    renderedText: [],
+    sentAudio: [],
+    startRenderTextTime: null,
+    startRecordingTime: null,
+    receivedAudioSampleRate: null,
+  };
+}
+
+function downloadAudioBuffer(audioBuffer: AudioBuffer, fileName: string): void {
+  const wav = audioBuffertoWav(audioBuffer);
+  const wavBlob = new Blob([new DataView(wav)], {
+    type: 'audio/wav',
+  });
+  const url = URL.createObjectURL(wavBlob);
+  const anchor = document.createElement('a');
+  anchor.href = url;
+  anchor.target = '_blank';
+  anchor.download = fileName;
+  anchor.click();
+}
+
+// Uncomment for debugging without download
+// function playAudioBuffer(audioBuffer: AudioBuffer): void {
+//   const audioContext = new AudioContext();
+//   const source = audioContext.createBufferSource();
+
+//   source.buffer = audioBuffer;
+//   source.connect(audioContext.destination);
+//   source.start();
+// }
+
+// Accumulate timings and audio / text translation samples for debugging and exporting
+class DebugTimingsManager {
+  timings: DebugTimings = getInitialTimings();
+
+  start(): void {
+    this.timings = getInitialTimings();
+    this.timings.startRecordingTime = new Date().getTime();
+  }
+
+  sentAudio(event: AudioProcessingEvent): void {
+    const end = new Date().getTime();
+    const start = end - event.inputBuffer.duration * 1000;
+    // Copy or else buffer seems to be re-used
+    const float32Audio = new Float32Array(event.inputBuffer.getChannelData(0));
+    this.timings.sentAudio.push({
+      start,
+      end,
+      float32Audio,
+    });
+  }
+
+  receivedText(text: string): void {
+    this.timings.receivedText.push({
+      time: new Date().getTime(),
+      chars: text.length,
+    });
+  }
+
+  startRenderText(): void {
+    if (this.timings.startRenderTextTime == null) {
+      this.timings.startRenderTextTime = new Date().getTime();
+    }
+  }
+
+  endRenderText(): void {
+    if (this.timings.startRenderTextTime == null) {
+      console.warn(
+        'Wrong timings of start / end rendering text. startRenderText is null',
+      );
+      return;
+    }
+
+    this.timings.renderedText.push({
+      start: this.timings.startRenderTextTime as number,
+      end: new Date().getTime(),
+    });
+    this.timings.startRenderTextTime = null;
+  }
+
+  receivedAudio(duration: number): void {
+    const start = new Date().getTime();
+    this.timings.receivedAudio.push({
+      start,
+      end: start + duration * 1000,
+    });
+  }
+
+  playedAudio(start: number, end: number, buffer: AudioBuffer | null): void {
+    if (buffer != null) {
+      if (this.timings.receivedAudioSampleRate == null) {
+        this.timings.receivedAudioSampleRate = buffer.sampleRate;
+      }
+      if (this.timings.receivedAudioSampleRate != buffer.sampleRate) {
+        console.error(
+          'Sample rates of received audio are unequal, will fail to reconstruct debug audio',
+          this.timings.receivedAudioSampleRate,
+          buffer.sampleRate,
+        );
+      }
+    }
+    this.timings.playedAudio.push({
+      start,
+      end,
+      float32Audio:
+        buffer == null
+          ? new Float32Array()
+          : new Float32Array(buffer.getChannelData(0)),
+    });
+  }
+
+  getChartData() {
+    const columns = [
+      {type: 'string', id: 'Series'},
+      {type: 'date', id: 'Start'},
+      {type: 'date', id: 'End'},
+    ];
+    return [
+      columns,
+      ...this.timings.sentAudio.map((sentAudio) => [
+        'Sent Audio',
+        new Date(sentAudio.start),
+        new Date(sentAudio.end),
+      ]),
+      ...this.timings.receivedAudio.map((receivedAudio) => [
+        'Received Audio',
+        new Date(receivedAudio.start),
+        new Date(receivedAudio.end),
+      ]),
+      ...this.timings.playedAudio.map((playedAudio) => [
+        'Played Audio',
+        new Date(playedAudio.start),
+        new Date(playedAudio.end),
+      ]),
+      // Best estimate duration by multiplying length with animation duration for each letter
+      ...this.timings.receivedText.map((receivedText) => [
+        'Received Text',
+        new Date(receivedText.time),
+        new Date(
+          receivedText.time + receivedText.chars * TYPING_ANIMATION_DELAY_MS,
+        ),
+      ]),
+      ...this.timings.renderedText.map((renderedText) => [
+        'Rendered Text',
+        new Date(renderedText.start),
+        new Date(renderedText.end),
+      ]),
+    ];
+  }
+
+  downloadInputAudio() {
+    const audioContext = new AudioContext();
+    const totalLength = this.timings.sentAudio.reduce((acc, cur) => {
+      return acc + cur?.float32Audio?.length ?? 0;
+    }, 0);
+    if (totalLength === 0) {
+      return;
+    }
+
+    const incomingArrayBuffer = audioContext.createBuffer(
+      1, // 1 channel
+      totalLength,
+      audioContext.sampleRate,
+    );
+
+    const buffer = incomingArrayBuffer.getChannelData(0);
+    let i = 0;
+    this.timings.sentAudio.forEach((sentAudio) => {
+      sentAudio.float32Audio.forEach((bytes) => {
+        buffer[i++] = bytes;
+      });
+    });
+
+    // Play for debugging
+    // playAudioBuffer(incomingArrayBuffer);
+    downloadAudioBuffer(incomingArrayBuffer, `input_audio.wav`);
+  }
+
+  downloadOutputAudio() {
+    const playedAudio = this.timings.playedAudio;
+    const sampleRate = this.timings.receivedAudioSampleRate;
+    if (
+      playedAudio.length === 0 ||
+      this.timings.startRecordingTime == null ||
+      sampleRate == null
+    ) {
+      return null;
+    }
+
+    let previousEndTime = this.timings.startRecordingTime;
+    const audioArray: number[] = [];
+    playedAudio.forEach((audio) => {
+      const delta = (audio.start - previousEndTime) / 1000;
+      for (let i = 0; i < delta * sampleRate; i++) {
+        audioArray.push(0.0);
+      }
+      audio.float32Audio.forEach((bytes) => audioArray.push(bytes));
+      previousEndTime = audio.end;
+    });
+    const audioContext = new AudioContext();
+    const incomingArrayBuffer = audioContext.createBuffer(
+      1, // 1 channel
+      audioArray.length,
+      sampleRate,
+    );
+
+    incomingArrayBuffer.copyToChannel(
+      new Float32Array(audioArray),
+      0, // first channel
+    );
+
+    // Play for debugging
+    // playAudioBuffer(incomingArrayBuffer);
+    downloadAudioBuffer(incomingArrayBuffer, 'output_audio.wav');
+  }
+}
+
+const debugSingleton = new DebugTimingsManager();
+export default function debug(): DebugTimingsManager | null {
+  const debugParam = getURLParams().debug;
+  return debugParam ? debugSingleton : null;
+}

streaming-react-app/src/float32To16BitPCM.ts

@@ -0,0 +1,16 @@
+export default function float32To16BitPCM(
+  float32Arr: Float32Array,
+): Int16Array {
+  const pcm16bit = new Int16Array(float32Arr.length);
+  for (let i = 0; i < float32Arr.length; ++i) {
+    // force number in [-1,1]
+    const s = Math.max(-1, Math.min(1, float32Arr[i]));
+
+    /**
+     * convert 32 bit float to 16 bit int pcm audio
+     * 0x8000 = minimum int16 value, 0x7fff = maximum int16 value
+     */
+    pcm16bit[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+  }
+  return pcm16bit;
+}

streaming-react-app/src/generateNewRoomID.ts

@@ -0,0 +1,56 @@
+import {random} from 'lodash';
+
+// const USABLE_CHARACTERS = 'BCDFGHJKMPQRTVWXY2346789';
+const USABLE_CHARACTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+const ID_LENGTH = 4;
+
+export function isValidRoomID(id: string | null | undefined): boolean {
+  if (id == null) {
+    return false;
+  }
+  if (id.length !== ID_LENGTH) {
+    return false;
+  }
+  return isValidPartialRoomID(id);
+}
+
+export function isValidPartialRoomID(roomID: string): boolean {
+  return (
+    roomID.length <= ID_LENGTH &&
+    roomID.split('').every((char) => USABLE_CHARACTERS.includes(char))
+  );
+}
+
+export default function generateNewRoomID(): string {
+  return Array.from(
+    {length: ID_LENGTH},
+    () => USABLE_CHARACTERS[random(USABLE_CHARACTERS.length - 1)],
+  ).join('');
+}
+
+export function getSequentialRoomIDForTestingGenerator(): () => string {
+  let counter = 0;
+
+  return function generateNextRoomID(): string {
+    const counterInBase: string = Number(counter)
+      .toString(USABLE_CHARACTERS.length)
+      .padStart(ID_LENGTH, '0');
+
+    if (counterInBase.length > ID_LENGTH) {
+      throw new Error(
+        'Ran out of unique room IDs from the sequential generator',
+      );
+    }
+
+    const result = counterInBase
+      .split('')
+      .map(
+        (digit) => USABLE_CHARACTERS[parseInt(digit, USABLE_CHARACTERS.length)],
+      )
+      .join('');
+
+    counter++;
+
+    return result;
+  };
+}

streaming-react-app/src/getParamFlag.ts

@@ -0,0 +1,39 @@
+import type {URLParamNames} from './types/URLParamsTypes';
+
+export function getBooleanParamFlag(
+  flag: URLParamNames,
+  defaultValue?: boolean,
+): boolean {
+  const paramFlagValue = getBooleanParamFlagWithoutDefault(flag);
+
+  if (paramFlagValue == null) {
+    // The default value for paramFlags is false, unless they explicitly provide a
+    // defaultValue via the config
+    return defaultValue ?? false;
+  }
+
+  return paramFlagValue;
+}
+
+export function getBooleanParamFlagWithoutDefault(
+  flag: URLParamNames,
+): boolean | null {
+  const urlParams = new URLSearchParams(window.location.search);
+
+  if (urlParams.get(flag) == null) {
+    return null;
+  }
+
+  return urlParams.get(flag) !== '0';
+}
+
+export function getStringParamFlag(
+  flag: URLParamNames,
+  defaultValue?: string,
+): string | null {
+  const urlParams = new URLSearchParams(window.location.search);
+
+  const param = urlParams.get(flag);
+
+  return param ?? defaultValue ?? null;
+}

streaming-react-app/src/getTranslationSentencesFromReceivedData.ts

@@ -0,0 +1,22 @@
+import {ServerTextData, TranslationSentences} from './types/StreamingTypes';
+
+export default function getTranslationSentencesFromReceivedData(
+  receivedData: Array<ServerTextData>,
+): TranslationSentences {
+  return receivedData
+    .reduce(
+      (acc, data) => {
+        const newAcc = [
+          ...acc.slice(0, -1),
+          acc[acc.length - 1].trim() + ' ' + data.payload,
+        ];
+        if (data.eos) {
+          newAcc.push('');
+        }
+
+        return newAcc;
+      },
+      [''],
+    )
+    .filter((s) => s.trim().length !== 0);
+}

streaming-react-app/src/isScrolledToDocumentBottom.ts

@@ -0,0 +1,11 @@
+export default function isScrolledToDocumentBottom(
+  bufferPx: number = 0,
+): boolean {
+  if (
+    window.innerHeight + window.scrollY >=
+    document.body.offsetHeight - bufferPx
+  ) {
+    return true;
+  }
+  return false;
+}

streaming-react-app/src/languageLookup.ts

@@ -0,0 +1,117 @@
+const LANG3_FULL = {
+  eng: 'english', 
+  arb: 'arabic', 
+  asm: 'assamese', 
+  bel: 'belarusian', 
+  bul: 'bulgarian', 
+  ben: 'bengali', 
+  cat: 'catalan', 
+  ces: 'czech', 
+  cym: 'welsh', 
+  dan: 'danish', 
+  deu: 'german', 
+  ell: 'greek', 
+  spa: 'spanish', 
+  est: 'estonian', 
+  pes: 'persian', 
+  fin: 'finnish', 
+  fra: 'french', 
+  hin: 'hindi', 
+  hun: 'hungarian', 
+  ind: 'indonesian', 
+  ita: 'italian', 
+  jpn: 'japanese', 
+  kat: 'georgian', 
+  lit: 'lithuanian', 
+  lvs: 'latvian', 
+  khk: 'mongolian', 
+  mar: 'marathi', 
+  mlt: 'maltese', 
+  nld: 'dutch', 
+  pan: 'punjabi', 
+  pol: 'polish', 
+  por: 'portuguese', 
+  ron: 'romanian', 
+  rus: 'russian', 
+  slk: 'slovak', 
+  slv: 'slovenian', 
+  swe: 'swedish', 
+  swh: 'swahili', 
+  tam: 'tamil', 
+  tha: 'thai', 
+  tur: 'turkish', 
+  ukr: 'ukrainian', 
+  urd: 'urdu', 
+  uzn: 'uzbek', 
+  vie: 'vietnamese', 
+  cmn: 'chinese', 
+  afr: 'afrikaans', 
+  isl: 'icelandic', 
+  ltz: 'luxembourgish', 
+  nob: 'norwegian', 
+  glg: 'galician', 
+  bos: 'bosnian', 
+  hrv: 'croatian', 
+  mkd: 'macedonian', 
+  srp: 'serbian', 
+  hye: 'armenian', 
+  azj: 'azerbaijani', 
+  kaz: 'kazakh', 
+  kor: 'korean', 
+  guj: 'gujarati', 
+  kan: 'kannada', 
+  npi: 'nepali', 
+  snd: 'sindhi', 
+  tel: 'telugu', 
+  jav: 'javanese', 
+  zlm: 'malay', 
+  mal: 'malayalam', 
+  tgl: 'tagalog', 
+  mya: 'myanmar', 
+  khm: 'khmer', 
+  lao: 'lao', 
+  heb: 'hebrew', 
+  pbt: 'pashto', 
+  tgk: 'tajik', 
+  amh: 'amharic', 
+  lin: 'lingala', 
+  som: 'somali', 
+  yor: 'yoruba', 
+  sna: 'shona', 
+  mri: 'maori', 
+  hau: 'hausa', 
+  oci: 'occitan', 
+  bak: 'bashkir', 
+  bre: 'breton', 
+  yid: 'yiddish', 
+  hat: 'haitian creole', 
+  mlg: 'malagasy', 
+  sin: 'sinhala', 
+  sqi: 'albanian', 
+  sun: 'sundanese', 
+  eus: 'basque', 
+  nno: 'nynorsk', 
+  tat: 'tatar', 
+  bod: 'tibetan', 
+  fao: 'faroese', 
+  haw: 'hawaiian', 
+  lat: 'latin', 
+  san: 'sanskrit', 
+  tuk: 'turkmen'
+};
+
+export function getLanguageFromThreeLetterCode(code: string): string | null {
+  try {
+    const name = LANG3_FULL[code] ?? null;
+    if (name == null) {
+      return null;
+    }
+    const capitalizedWords = name
+      .split(' ')
+      .map((word: string) => word[0].toUpperCase() + word.slice(1));
+    return capitalizedWords.join(' ');
+  } catch (e) {
+    console.warn(`Unable to get language name for code ${code}: ${e}`);
+  }
+  return null;
+}

streaming-react-app/src/main.tsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App.tsx';
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+);

streaming-react-app/src/react-xr/ARButton.tsx

@@ -0,0 +1,89 @@
+import * as THREE from 'three';
+import {Button} from '@mui/material';
+import {useCallback, useEffect, useState} from 'react';
+import {BufferedSpeechPlayer} from '../createBufferedSpeechPlayer';
+
+type Props = {
+  bufferedSpeechPlayer: BufferedSpeechPlayer;
+  renderer: THREE.WebGLRenderer | null;
+  onARVisible?: () => void;
+  onARHidden?: () => void;
+};
+
+export default function ARButton({
+  bufferedSpeechPlayer,
+  renderer,
+  onARVisible,
+  onARHidden,
+}: Props) {
+  const [session, setSession] = useState<XRSession | null>(null);
+  const [supported, setSupported] = useState<boolean>(true);
+
+  useEffect(() => {
+    if (!navigator.xr) {
+      setSupported(false);
+      return;
+    }
+    navigator.xr.isSessionSupported('immersive-ar').then((supported) => {
+      setSupported(supported);
+    });
+  }, []);
+
+  const resetBuffers = useCallback(
+    (event: XRSessionEvent) => {
+      const session = event.target;
+      if (!(session instanceof XRSession)) {
+        return;
+      }
+      switch (session.visibilityState) {
+        case 'visible':
+          console.log('Restarting speech player, device is visible');
+          bufferedSpeechPlayer.stop();
+          bufferedSpeechPlayer.start();
+          onARVisible?.();
+          break;
+        case 'hidden':
+          console.log('Stopping speech player, device is hidden');
+          bufferedSpeechPlayer.stop();
+          bufferedSpeechPlayer.start();
+          onARHidden?.();
+          break;
+      }
+    },
+    [bufferedSpeechPlayer],
+  );
+
+  async function onSessionStarted(session: XRSession) {
+    setSession(session);
+
+    session.onvisibilitychange = resetBuffers;
+    session.onend = onSessionEnded;
+
+    await renderer.xr.setSession(session);
+  }
+
+  function onSessionEnded() {
+    setSession(null);
+  }
+
+  const onClick = () => {
+    if (session === null) {
+      navigator.xr!.requestSession('immersive-ar').then(onSessionStarted);
+    } else {
+      session.end();
+    }
+  };
+  return (
+    <Button
+      variant="contained"
+      onClick={onClick}
+      disabled={!supported || renderer == null}
+      sx={{mt: 1}}>
+      {supported
+        ? renderer != null
+          ? 'Enter AR'
+          : 'Initializing AR...'
+        : 'AR Not Supported'}
+    </Button>
+  );
+}

streaming-react-app/src/react-xr/Button.tsx

@@ -0,0 +1,117 @@
+import {useRef, useEffect} from 'react';
+import * as THREE from 'three';
+import {extend} from '@react-three/fiber';
+import ThreeMeshUI from 'three-mesh-ui';
+import ThreeMeshUIText, {ThreeMeshUITextType} from './ThreeMeshUIText';
+import {Interactive} from '@react-three/xr';
+
+/**
+ * Using `?url` at the end of this import tells vite this is a static asset, and
+ * provides us a URL to the hashed version of the file when the project is built.
+ * See: https://vitejs.dev/guide/assets.html#explicit-url-imports
+ */
+import robotoFontFamilyJson from '../assets/RobotoMono-Regular-msdf.json?url';
+import robotoFontTexture from '../assets/RobotoMono-Regular.png';
+
+extend(ThreeMeshUI);
+
+/**
+ * Button component that renders as a three-mesh-ui block
+ */
+export default function Button({
+  onClick,
+  content,
+  width,
+  height,
+  fontSize,
+  borderRadius,
+  padding,
+}) {
+  const button = useRef<JSX.IntrinsicElements['block']>();
+  const textRef = useRef<ThreeMeshUITextType>();
+
+  useEffect(() => {
+    if (textRef.current != null) {
+      textRef.current.set({content});
+    }
+  }, [textRef, content]);
+
+  useEffect(() => {
+    if (!button.current) {
+      return;
+    }
+    button.current.setupState({
+      state: 'hovered',
+      attributes: {
+        offset: 0.002,
+        backgroundColor: new THREE.Color(0x607b8f),
+        fontColor: new THREE.Color(0xffffff),
+      },
+    });
+    button.current.setupState({
+      state: 'idle',
+      attributes: {
+        offset: 0.001,
+        backgroundColor: new THREE.Color(0x465a69),
+        fontColor: new THREE.Color(0xffffff),
+      },
+    });
+    button.current.setupState({
+      state: 'selected',
+      attributes: {
+        offset: 0.005,
+        backgroundColor: new THREE.Color(0x000000),
+        fontColor: new THREE.Color(0xffffff),
+      },
+    });
+    button.current.setState('idle');
+  }, []);
+
+  const args = [
+    {
+      width,
+      height,
+      fontSize,
+      padding,
+      justifyContent: 'end',
+      textAlign: 'center',
+      alignItems: 'center',
+      borderRadius,
+      fontFamily: robotoFontFamilyJson,
+      fontTexture: robotoFontTexture,
+      backgroundOpacity: 1,
+      backgroundColor: new THREE.Color(0x779092),
+      fontColor: new THREE.Color(0x000000),
+    },
+  ];
+
+  return (
+    <Interactive
+      // These are for XR mode
+      onSelect={() => {
+        onClick();
+      }}
+      onHover={() => button.current.setState('hovered')}
+      onBlur={() => button.current.setState('idle')}
+      onSelectStart={() => button.current.setState('selected')}
+      onSelectEnd={() => button.current.setState('idle')}>
+      <block
+        // These are for non-XR modes
+        onPointerEnter={() => button.current.setState('hovered')}
+        onPointerLeave={() => button.current.setState('idle')}
+        onPointerDown={() => button.current.setState('selected')}
+        onPointerUp={() => {
+          button.current.setState('hovered');
+          onClick();
+        }}>
+        <block args={args} ref={button}>
+          <ThreeMeshUIText
+            ref={textRef}
+            fontColor={new THREE.Color(0xffffff)}
+            content={content}
+          />
+        </block>
+      </block>
+    </Interactive>
+  );
+}

streaming-react-app/src/react-xr/Colors.ts

@@ -0,0 +1,6 @@
+import * as THREE from 'three';
+
+export const WHITE = new THREE.Color('#FFFFFF');
+export const BLACK = new THREE.Color('#000000');
+export const RED = new THREE.Color('red');
+export const BLUE = new THREE.Color('blue');

streaming-react-app/src/react-xr/MovementController.tsx

@@ -0,0 +1,64 @@
+import {useRef} from 'react';
+import {useFrame} from '@react-three/fiber';
+import {useController, useXR} from '@react-three/xr';
+import * as THREE from 'three';
+
+const USE_HORIZONTAL = true;
+const USE_VERTICAL = true;
+const USE_ROTATION = true;
+const HORIZONTAL_AXIS = 2;
+const VERTICAL_AXIS = 3;
+const ROTATION_AXIS = 2;
+const SENSITIVITY = 0.05;
+const DEADZONE = 0.05;
+
+/**
+ * Component to add into the ThreeJS canvas that reads controller (Quest) inputs to change camera position
+ */
+export default function MovementController() {
+  const xr = useXR();
+  const controller = useController('right');
+  const forward = useRef(new THREE.Vector3());
+  const horizontal = useRef(new THREE.Vector3());
+
+  useFrame(() => {
+    const player = xr.player;
+    const camera = xr.player.children[0];
+    const cameraMatrix = camera.matrixWorld.elements;
+    forward.current
+      .set(-cameraMatrix[8], -cameraMatrix[9], -cameraMatrix[10])
+      .normalize();
+
+    const axes = controller?.inputSource?.gamepad?.axes ?? [0, 0, 0, 0];
+
+    if (USE_HORIZONTAL) {
+      horizontal.current.copy(forward.current);
+      horizontal.current.cross(camera.up).normalize();
+
+      player.position.add(
+        horizontal.current.multiplyScalar(
+          (Math.abs(axes[HORIZONTAL_AXIS]) > DEADZONE
+            ? axes[HORIZONTAL_AXIS]
+            : 0) * SENSITIVITY,
+        ),
+      );
+    }
+
+    if (USE_VERTICAL) {
+      player.position.add(
+        forward.current.multiplyScalar(
+          (Math.abs(axes[VERTICAL_AXIS]) > DEADZONE ? axes[VERTICAL_AXIS] : 0) *
+            SENSITIVITY,
+        ),
+      );
+    }
+
+    if (USE_ROTATION) {
+      player.rotation.y -=
+        (Math.abs(axes[ROTATION_AXIS]) > DEADZONE ? axes[ROTATION_AXIS] : 0) *
+        SENSITIVITY;
+    }
+  });
+
+  return <></>;
+}

streaming-react-app/src/react-xr/Playground.tsx

@@ -0,0 +1,133 @@
+/**
+ * EXPERIMENTAL components to play around with but not officially use in the demo while
+ * we develop.
+ */
+import {useEffect, useState} from 'react';
+import {Object3DNode, extend} from '@react-three/fiber';
+import ThreeMeshUI from 'three-mesh-ui';
+
+import {} from '@react-three/xr';
+import {Sparkles, Shadow} from '@react-three/drei';
+
+// import FontImage from './assets/Roboto-msdf.png';
+import {FontLoader} from 'three/examples/jsm/loaders/FontLoader.js';
+import {TextGeometry} from 'three/examples/jsm/geometries/TextGeometry.js';
+import ThreeMeshUIText from './ThreeMeshUIText';
+import {ContactShadows, BakeShadows} from '@react-three/drei';
+
+extend({TextGeometry});
+extend(ThreeMeshUI);
+
+declare module '@react-three/fiber' {
+  interface ThreeElements {
+    textGeometry: Object3DNode<TextGeometry, typeof TextGeometry>;
+  }
+}
+
+// This is for textGeometry.. not using three-mesh-ui to display text
+export function TitleMesh() {
+  const font = new FontLoader().parse();
+  console.log('font', font);
+  const [text, setText] = useState('Text');
+
+  useEffect(() => {
+    setTimeout(() => {
+      setText(text + ' more ');
+      console.log('adding more tex..', text);
+    }, 1000);
+  }, [text]);
+
+  return (
+    <mesh>
+      <textGeometry args={[text, {font, size: 5, height: 1}]} />
+      <meshPhysicalMaterial attach={'material'} color={'white'} />
+    </mesh>
+  );
+}
+
+export function Sphere({
+  size = 1,
+  amount = 50,
+  color = 'white',
+  emissive,
+  ...props
+}) {
+  return (
+    <mesh {...props}>
+      <sphereGeometry args={[size, 64, 64]} />
+      <meshPhysicalMaterial
+        roughness={0}
+        color={color}
+        emissive={emissive || color}
+        envMapIntensity={0.2}
+      />
+      <Sparkles count={amount} scale={size * 2} size={6} speed={0.4} />
+      <Shadow
+        rotation={[-Math.PI / 2, 0, 0]}
+        scale={size}
+        position={[0, -size, 0]}
+        color={emissive}
+        opacity={0.5}
+      />
+    </mesh>
+  );
+}
+
+export function Title({accentColor}) {
+  return (
+    <block
+      args={[
+        {
+          width: 1,
+          height: 0.25,
+          backgroundOpacity: 0,
+          justifyContent: 'center',
+        },
+      ]}>
+      <ThreeMeshUIText content={'Hello '} />
+      <ThreeMeshUIText content={'world!'} args={[{fontColor: accentColor}]} />
+    </block>
+  );
+}
+
+export function RandomComponents() {
+  return (
+    <>
+      <color args={['#eee']} attach={'background'} />
+      <Sphere
+        color="white"
+        amount={50}
+        emissive="green"
+        glow="lightgreen"
+        position={[1, 1, -1]}
+      />
+      <Sphere
+        color="white"
+        amount={30}
+        emissive="purple"
+        glow="#ff90f0"
+        size={0.5}
+        position={[-1.5, 0.5, -2]}
+      />
+      <Sphere
+        color="lightpink"
+        amount={20}
+        emissive="orange"
+        glow="#ff9f50"
+        size={0.25}
+        position={[-1, 0.25, 1]}
+      />
+      <ContactShadows
+        renderOrder={2}
+        color="black"
+        resolution={1024}
+        frames={1}
+        scale={10}
+        blur={1.5}
+        opacity={0.65}
+        far={0.5}
+      />
+      <BakeShadows />
+    </>
+  );
+}