Initial OSS demo commit

---------

Co-authored-by: Justin Haaheim <[email protected]>
Co-authored-by: Mark Duppenthaler <[email protected]>

seamless_server/app_pubsub.py

@@ -0,0 +1,790 @@
+from operator import itemgetter
+import os
+from typing import Any, Optional, Tuple, Dict, TypedDict
+from urllib import parse
+from uuid import uuid4
+import colorlog
+import io
+import logging
+from pprint import pformat
+import socketio
+import sys
+import time
+import random
+import string
+
+from src.room import Room, Member
+from src.simuleval_agent_directory import NoAvailableAgentException
+from src.simuleval_agent_directory import SimulevalAgentDirectory
+from src.simuleval_transcoder import SimulevalTranscoder
+from src.transcoder_helpers import get_transcoder_output_events
+
+###############################################
+# Constants
+###############################################
+
+DEBUG = True
+
+ALL_ROOM_ID = "ALL"
+
+ROOM_ID_USABLE_CHARACTERS = string.ascii_uppercase
+ROOM_ID_LENGTH = 4
+
+ROOM_LISTENERS_SUFFIX = "_listeners"
+ROOM_SPEAKERS_SUFFIX = "_speakers"
+
+ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME = "remove_server_lock"
+
+###############################################
+# Configure logger
+###############################################
+
+logger = logging.getLogger("socketio_server_pubsub")
+logger.propagate = False
+
+handler = colorlog.StreamHandler(stream=sys.stdout)
+
+formatter = colorlog.ColoredFormatter(
+    "%(log_color)s[%(asctime)s][%(levelname)s][%(module)s]:%(reset)s %(message)s",
+    reset=True,
+    log_colors={
+        "DEBUG": "cyan",
+        "INFO": "green",
+        "WARNING": "yellow",
+        "ERROR": "red",
+        "CRITICAL": "red,bg_white",
+    },
+)
+
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+logger.setLevel(logging.WARNING)
+
+print("")
+print("")
+print("=" * 20 + " ⭐️ Starting Server... ⭐️ " + "=" * 20)
+
+###############################################
+# Configure socketio server
+###############################################
+
+CLIENT_BUILD_PATH = "../streaming-react-app/dist/"
+static_files = {
+    "/": CLIENT_BUILD_PATH,
+    "/assets/seamless-db6a2555.svg": {
+        "filename": CLIENT_BUILD_PATH + "assets/seamless-db6a2555.svg",
+        "content_type": "image/svg+xml",
+    },
+}
+
+# sio is the main socket.io entrypoint
+sio = socketio.AsyncServer(
+    async_mode="asgi",
+    cors_allowed_origins="*",
+    logger=logger,
+    # engineio_logger=logger,
+)
+# sio.logger.setLevel(logging.DEBUG)
+app = socketio.ASGIApp(sio, static_files=static_files)
+
+# rooms is indexed by room_id
+rooms: Dict[str, Room] = {}
+
+
+class MemberDirectoryObject(TypedDict):
+    room: Room
+    member_object: Member
+
+
+# member_directory is indexed by client_id
+# NOTE: client_id is really "client session id", meaning that it is unique to a single browser session.
+# If a user opens a new tab, they will have a different client_id and can join another room, join
+# the same room with different roles, etc.
+# NOTE: For a long-running production server we would want to clean up members after a certain timeout
+# but for this limited application we can just keep them around
+member_directory: Dict[str, MemberDirectoryObject] = {}
+
+
+class ServerLock(TypedDict):
+    name: str
+    client_id: str
+    member_object: Member
+
+
+server_lock: Optional[ServerLock] = None
+
+server_id = str(uuid4())
+
+# Specify specific models to load (some environments have issues loading multiple models)
+# See AgentWithInfo with JSON format details.
+models_override = os.environ.get("MODELS_OVERRIDE")
+
+available_agents = SimulevalAgentDirectory()
+logger.info("Building and adding agents...")
+if models_override is not None:
+    logger.info(f"MODELS_OVERRIDE supplied from env vars: {models_override}")
+available_agents.build_and_add_agents(models_override)
+
+agents_capabilities_for_json = available_agents.get_agents_capabilities_list_for_json()
+
+
+###############################################
+# Helpers
+###############################################
+
+
+def catch_and_log_exceptions_for_sio_event_handlers(func):
+    # wrapper should have the same signature as the original function
+    async def catch_exception_wrapper(*args, **kwargs):
+        try:
+            return await func(*args, **kwargs)
+        except Exception as e:
+            message = f"[app_pubsub] Caught exception in '{func.__name__}' event handler:\n\n{e}"
+            logger.exception(message, stack_info=True)
+
+            try:
+                exception_data = {
+                    "message": message,
+                    "timeEpochMs": int(time.time() * 1000),
+                }
+
+                try:
+                    # Let's try to add as much useful metadata as possible to the server_exception event
+                    sid = args[0]
+                    if isinstance(sid, str) and len(sid) > 0:
+                        session_data = await get_session_data(sid)
+                        if session_data:
+                            client_id = session_data.get("client_id")
+                            member = session_data.get("member_object")
+                            room = session_data.get("room_object")
+
+                            exception_data["room"] = str(room)
+                            exception_data["member"] = str(member)
+                            exception_data["clientID"] = str(client_id)
+                except Exception as inner_e:
+                    # We expect there will be times when clientID or other values aren't present, so just log this as a warning
+                    logger.warn(
+                        f"[app_pubsub] Caught exception while trying add additional_data to server_exception:\n\n{inner_e}"
+                    )
+
+                # For now let's emit this to all clients. We ultimatley may want to emit it just to the room it's happening in.
+                await sio.emit("server_exception", exception_data)
+            except Exception as inner_e:
+                logger.exception(
+                    f"[app_pubsub] Caught exception while trying to emit server_exception event:\n{inner_e}"
+                )
+
+            # Re-raise the exception so it's handled normally by the server
+            raise e
+
+    # Set the name of the wrapper to the name of the original function so that the socketio server can associate it with the right event
+    catch_exception_wrapper.__name__ = func.__name__
+    return catch_exception_wrapper
+
+
+async def emit_room_state_update(room):
+    await sio.emit(
+        "room_state_update",
+        room.to_json(),
+        room=room.room_id,
+    )
+
+
+async def emit_server_state_update():
+    room_statuses = {
+        room_id: room.get_room_status_dict() for room_id, room in rooms.items()
+    }
+    total_active_connections = sum(
+        [room_status["activeConnections"] for room_status in room_statuses.values()]
+    )
+    total_active_transcoders = sum(
+        [room_status["activeTranscoders"] for room_status in room_statuses.values()]
+    )
+    logger.info(
+        f"[Server Status]: {total_active_connections} active connections (in rooms); {total_active_transcoders} active transcoders"
+    )
+    logger.info(f"[Server Status]: server_lock={server_lock}")
+    server_lock_object_for_js = (
+        {
+            "name": server_lock.get("name"),
+            "clientID": server_lock.get("client_id"),
+            "isActive": server_lock.get("member_object")
+            and server_lock.get("member_object").transcoder is not None,
+        }
+        if server_lock
+        else None
+    )
+    await sio.emit(
+        "server_state_update",
+        {
+            "statusByRoom": room_statuses,
+            "totalActiveConnections": total_active_connections,
+            "totalActiveTranscoders": total_active_transcoders,
+            "agentsCapabilities": agents_capabilities_for_json,
+            "serverLock": server_lock_object_for_js,
+        },
+        room=ALL_ROOM_ID,
+    )
+
+
+async def get_session_data(sid):
+    session = await sio.get_session(sid)
+    # It seems like if the session has not been set that get_session may return None, so let's provide a fallback empty dictionary here
+    return session or {}
+
+
+async def set_session_data(sid, client_id, room_id, room_object, member_object):
+    await sio.save_session(
+        sid,
+        {
+            "client_id": client_id,
+            "room_id": room_id,
+            "room_object": room_object,
+            "member_object": member_object,
+        },
+    )
+
+
+def get_random_room_id():
+    return "".join(random.choices(ROOM_ID_USABLE_CHARACTERS, k=ROOM_ID_LENGTH))
+
+
+def get_random_unused_room_id():
+    room_id = get_random_room_id()
+    while room_id in rooms:
+        room_id = get_random_room_id()
+    return room_id
+
+
+###############################################
+# Socket.io Basic Event Handlers
+###############################################
+
+
+@sio.on("connect")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def connect(sid, environ):
+    logger.info(f"📥 [event: connected] sid={sid}")
+
+    # TODO: Sanitize/validate query param input
+    query_params = dict(parse.parse_qsl(environ["QUERY_STRING"]))
+    client_id = query_params.get("clientID")
+
+    logger.debug(f"query_params:\n{pformat(query_params)}")
+
+    if client_id is None:
+        logger.info("No clientID provided. Disconnecting...")
+        await sio.disconnect(sid)
+        return
+
+    # On reconnect we need to rejoin rooms and reset session data
+    if member_directory.get(client_id):
+        room = member_directory[client_id].get("room")
+        room_id = room.room_id
+        # Note: We could also get this from room.members[client_id]
+        member = member_directory[client_id].get("member_object")
+
+        member.connection_status = "connected"
+        member.session_id = sid
+
+        logger.info(
+            f"[event: connect] {member} reconnected. Attempting to re-add them to socketio rooms and reset session data."
+        )
+
+        if room is None or member is None:
+            logger.error(
+                f"[event: connect] {client_id} is reconnecting, but room or member is None. This should not happen."
+            )
+            await sio.disconnect(sid)
+            return
+
+        sio.enter_room(sid, room_id)
+        sio.enter_room(sid, ALL_ROOM_ID)
+
+        if client_id in room.listeners:
+            sio.enter_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        if client_id in room.speakers:
+            sio.enter_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+
+        # Save the room_id to the socketio client session
+        await set_session_data(
+            sid,
+            client_id=client_id,
+            room_id=room.room_id,
+            room_object=room,
+            member_object=member,
+        )
+        await emit_room_state_update(room)
+    else:
+        # Save the client id to the socketio client session
+        await set_session_data(
+            sid, client_id=client_id, room_id=None, room_object=None, member_object=None
+        )
+
+    await sio.emit("server_id", server_id, to=sid)
+    await emit_server_state_update()
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def disconnect(sid):
+    global server_lock
+    session_data = await get_session_data(sid)
+    # logger.info("session_data", session_data)
+
+    client_id = None
+    member = None
+    room = None
+
+    if session_data:
+        client_id = session_data.get("client_id")
+        member = session_data.get("member_object")
+        room = session_data.get("room_object")
+
+    logger.info(
+        f"[event: disconnect][{room or 'NOT_IN_ROOM'}] member: {member or 'NO_MEMBER_OBJECT'} disconnected"
+    )
+
+    # Release the lock if this is the client that holds the current server lock
+    if server_lock and server_lock.get("client_id") == client_id:
+        server_lock = None
+
+    if member:
+        member.connection_status = "disconnected"
+
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+            member.requested_output_type = None
+
+        if room:
+            logger.info(
+                f"[event: disconnect] {member} disconnected from room {room.room_id}"
+            )
+            await emit_room_state_update(room)
+        else:
+            logger.info(
+                f"[event: disconnect] {member} disconnected, but no room object present. This should not happen."
+            )
+    else:
+        logger.info(
+            f"[event: disconnect] client_id {client_id or 'NO_CLIENT_ID'} with sid {sid} in rooms {str(sio.rooms(sid))} disconnected"
+        )
+
+    await emit_server_state_update()
+
+
+@sio.on("*")
+async def catch_all(event, sid, data):
+    logger.info(f"[unhandled event: {event}] sid={sid} data={data}")
+
+
+###############################################
+# Socket.io Streaming Event handlers
+###############################################
+
+
+@sio.on("join_room")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def join_room(sid, client_id, room_id_from_client, config_dict):
+    global server_lock
+
+    args = {
+        "sid": sid,
+        "client_id": client_id,
+        "room_id": room_id_from_client,
+        "config_dict": config_dict,
+    }
+    logger.info(f"[event: join_room] {args}")
+    session_data = await get_session_data(sid)
+    logger.info(f"session_data: {session_data}")
+
+    room_id = room_id_from_client
+    if room_id is None:
+        room_id = get_random_unused_room_id()
+        logger.info(
+            f"No room_id provided. Generating a random, unused room_id: {room_id}"
+        )
+
+    # Create the room if it doesn't already exist
+    if room_id not in rooms:
+        rooms[room_id] = Room(room_id)
+
+    room = rooms[room_id]
+
+    member = None
+
+    name = "[NO_NAME]"
+
+    # If the client is reconnecting use their existing member object. Otherwise create a new one.
+    if client_id in room.members:
+        member = room.members[client_id]
+        logger.info(f"{member} is rejoining room {room_id}.")
+    else:
+        member_number = len(room.members) + 1
+        name = f"Member {member_number}"
+        member = Member(
+            client_id=client_id,
+            session_id=sid,
+            name=name,
+        )
+        logger.info(f"Created a new Member object: {member}")
+        logger.info(f"Adding {member} to room {room_id}")
+        room.members[client_id] = member
+
+    # Also add them to the member directory
+    member_directory[client_id] = {"room": room, "member_object": member}
+
+    # Join the socketio room, which enables broadcasting to all members of the room
+    sio.enter_room(sid, room_id)
+    # Join the room for all clients
+    sio.enter_room(sid, ALL_ROOM_ID)
+
+    if "listener" in config_dict["roles"]:
+        sio.enter_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        if client_id not in room.listeners:
+            room.listeners.append(client_id)
+    else:
+        sio.leave_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        room.listeners = [
+            listener_id for listener_id in room.listeners if listener_id != client_id
+        ]
+
+    if "speaker" in config_dict["roles"]:
+        sio.enter_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+        if client_id not in room.speakers:
+            room.speakers.append(client_id)
+    else:
+        sio.leave_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+        # If the person is no longer a speaker they should no longer be able to lock the server
+        if server_lock and server_lock.get("client_id") == client_id:
+            logger.info(
+                f"🔓 Server is now unlocked from client {server_lock.get('client_id')} with name/info: {server_lock.get('name')}"
+            )
+            server_lock = None
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+        room.speakers = [
+            speaker_id for speaker_id in room.speakers if speaker_id != client_id
+        ]
+
+    # If we currently own the server lock and are updating roles and we no longer have server lock specified, release it
+    if (
+        server_lock is not None
+        and server_lock["client_id"] == client_id
+        and config_dict.get("lockServerName") is None
+    ):
+        logger.info(f"[join_room] Releasing server lock: {pformat(server_lock)}")
+        server_lock = None
+
+    # Only speakers should be able to lock the server
+    if config_dict.get("lockServerName") is not None and "speaker" in config_dict.get(
+        "roles", {}
+    ):
+        # If something goes wrong and the server gets stuck in a locked state the client can
+        # force the server to remove the lock by passing the special name ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
+        if (
+            server_lock is not None
+            and config_dict.get("lockServerName")
+            == ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
+        ):
+            server_lock = None
+            logger.info(
+                f"🔓 Server lock has been reset by {client_id} using the escape hatch name {ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME}"
+            )
+
+        # If the server is not locked, set a lock. If it's already locked to this client, update the lock object
+        elif server_lock is None or server_lock.get("client_id") == client_id:
+            # TODO: Add some sort of timeout as a backstop in case someone leaves the browser tab open after locking the server
+            server_lock = {
+                "name": config_dict.get("lockServerName"),
+                "client_id": client_id,
+                "member_object": member,
+            }
+            logger.info(
+                f"🔒 Server is now locked to client {server_lock.get('client_id')} with name/info: {server_lock.get('name')}\nThis client will have priority over all others until they disconnect."
+            )
+        # If the server is already locked to someone else, don't allow this client to lock it
+        elif server_lock is not None and server_lock.get("client_id") != client_id:
+            logger.warn(
+                f"⚠️  Server is already locked to client {server_lock.get('client_id')}. Ignoring request to lock to client {client_id}."
+            )
+            # TODO: Maybe throw an error here?
+
+    # Save the room_id to the socketio client session
+    await set_session_data(
+        sid,
+        client_id=client_id,
+        room_id=room_id,
+        room_object=room,
+        member_object=member,
+    )
+
+    await emit_room_state_update(room)
+    await emit_server_state_update()
+
+    return {"roomsJoined": sio.rooms(sid), "roomID": room_id}
+
+
+# TODO: Add code to prevent more than one speaker from connecting/streaming at a time
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def configure_stream(sid, config):
+    session_data = await get_session_data(sid)
+    client_id, member, room = itemgetter("client_id", "member_object", "room_object")(
+        session_data
+    )
+
+    logger.debug(
+        f"[event: configure_stream][{room}] Received stream config from {member}\n{pformat(config)}"
+    )
+
+    if member is None or room is None:
+        logger.error(
+            f"Received stream config from {member}, but member or room is None. This should not happen."
+        )
+        return {"status": "error", "message": "member_or_room_is_none"}
+
+    # If there is a server lock WITH an active transcoder session, prevent other users from configuring and starting a stream
+    # If the server lock client does NOT have an active transcoder session allow this to proceed, knowing that
+    # this stream will be interrupted if the server lock client starts streaming
+    if (
+        server_lock is not None
+        and server_lock.get("client_id") != client_id
+        and server_lock.get("member_object")
+        and server_lock.get("member_object").transcoder is not None
+    ):
+        logger.warn(
+            f"Server is locked to client {server_lock.get('client_id')}. Ignoring request to configure stream from client {client_id}."
+        )
+        return {"status": "error", "message": "server_locked"}
+
+    debug = config.get("debug")
+    async_processing = config.get("async_processing")
+
+    # Currently s2s, s2t or s2s&t
+    model_type = config.get("model_type")
+    member.requested_output_type = model_type
+
+    model_name = config.get("model_name")
+
+    try:
+        agent = available_agents.get_agent_or_throw(model_name)
+    except NoAvailableAgentException as e:
+        logger.warn(f"Error while getting agent: {e}")
+        # await sio.emit("error", str(e), to=sid)
+        await sio.disconnect(sid)
+        return {"status": "error", "message": str(e)}
+
+    if member.transcoder:
+        logger.warn(
+            "Member already has a transcoder configured. Closing it, and overwriting with a new transcoder..."
+        )
+        member.transcoder.close = True
+
+    t0 = time.time()
+    try:
+        member.transcoder = SimulevalTranscoder(
+            agent,
+            config["rate"],
+            debug=debug,
+            buffer_limit=int(config["buffer_limit"]),
+        )
+    except Exception as e:
+        logger.warn(f"Got exception while initializing agents: {e}")
+        # await sio.emit("error", str(e), to=sid)
+        await sio.disconnect(sid)
+        return {"status": "error", "message": str(e)}
+
+    t1 = time.time()
+    logger.debug(f"Booting up VAD and transcoder took {t1-t0} sec")
+
+    # TODO: if async_processing is false, then we need to run transcoder.process_pipeline_once() whenever we receive audio, or at some other sensible interval
+    if async_processing:
+        member.transcoder.start()
+
+    # We need to emit a room state update here since room state now includes # of active transcoders
+    await emit_room_state_update(room)
+    await emit_server_state_update()
+
+    return {"status": "ok", "message": "server_ready"}
+    # await sio.emit("server_ready", None, to=sid)
+
+
+# The config here is a partial config, meaning it may not contain all the config values -- only the ones the user
+# wants to change
+@sio.on("set_dynamic_config")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def set_dynamic_config(
+    sid,
+    # partial_config's type is defined in StreamingTypes.ts
+    partial_config,
+):
+    session_data = await get_session_data(sid)
+
+    # client_id = None
+    member = None
+    # room = None
+
+    if session_data:
+        # client_id = session_data.get("client_id")
+        member = session_data.get("member_object")
+        # room = session_data.get("room_object")
+
+    if member:
+        new_dynamic_config = {
+            **(member.transcoder_dynamic_config or {}),
+            **partial_config,
+        }
+        logger.info(
+            f"[set_dynamic_config] Setting new dynamic config:\n\n{pformat(new_dynamic_config)}\n"
+        )
+        member.transcoder_dynamic_config = new_dynamic_config
+
+    return {"status": "ok", "message": "dynamic_config_set"}
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def incoming_audio(sid, blob):
+    # logger.info(f"[event: incoming_audio] {sid}")
+
+    session_data = await get_session_data(sid)
+
+    client_id = None
+    member = None
+    room = None
+
+    if session_data:
+        client_id = session_data.get("client_id")
+        member = session_data.get("member_object")
+        room = session_data.get("room_object")
+
+    logger.debug(f"[event: incoming_audio] from member {member}")
+
+    # If the server is locked by someone else, kill our transcoder and ignore incoming audio
+    # If the server lock client does NOT have an active transcoder session allow this incoming audio pipeline to proceed,
+    # knowing that this stream will be interrupted if the server lock client starts streaming
+    if (
+        server_lock is not None
+        and server_lock.get("client_id") != client_id
+        and server_lock.get("member_object")
+        and server_lock.get("member_object").transcoder is not None
+    ):
+        # TODO: Send an event to the client to let them know their streaming session has been killed
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+            # Update both room state and server state given that the number of active transcoders has changed
+            if room:
+                await emit_room_state_update(room)
+            await emit_server_state_update()
+        logger.warn(
+            f"[incoming_audio] Server is locked to client {server_lock.get('client_id')}. Ignoring incoming audio from client {client_id}."
+        )
+        return
+
+    if member is None or room is None:
+        logger.error(
+            f"[incoming_audio] Received incoming_audio from {member}, but member or room is None. This should not happen."
+        )
+        return
+
+    # NOTE: bytes and bytearray are very similar, but bytes is immutable, and is what is returned by socketio
+    if not isinstance(blob, bytes):
+        logger.error(
+            f"[incoming_audio] Received audio from {member}, but it was not of type `bytes`. type(blob) = {type(blob)}"
+        )
+        return
+
+    if member.transcoder is None:
+        logger.error(
+            f"[incoming_audio] Received audio from {member}, but no transcoder configured to process it (member.transcoder is None). This should not happen."
+        )
+        return
+
+    member.transcoder.process_incoming_bytes(
+        blob, dynamic_config=member.transcoder_dynamic_config
+    )
+
+    # TODO: What we have below is NOT a good way to do this, because instead of sending output when its ready we're
+    # sending it when new input comes in, which means it's just sitting around. This is a temporary hack until
+    # we figure out a better architecture for awaiting transcoder output and sending it to the client.
+    events = get_transcoder_output_events(member.transcoder)
+    logger.debug(f"[incoming_audio] transcoder output events: {len(events)}")
+
+    if len(events) == 0:
+        logger.debug("[incoming_audio] No transcoder output to send")
+    else:
+        for e in events:
+            if e["event"] == "translation_speech" and member.requested_output_type in [
+                "s2s",
+                "s2s&t",
+            ]:
+                logger.debug("[incoming_audio] Sending translation_speech event")
+                await sio.emit(
+                    "translation_speech", e, room=f"{room.room_id}_listeners"
+                )
+            elif e["event"] == "translation_text" and member.requested_output_type in [
+                "s2t",
+                "s2s&t",
+            ]:
+                logger.debug("[incoming_audio] Sending translation_text event")
+                await sio.emit("translation_text", e, room=f"{room.room_id}_listeners")
+            else:
+                logger.error(f"[incoming_audio] Unexpected event type: {e['event']}")
+
+    return
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def stop_stream(sid):
+    session_data = await get_session_data(sid)
+    client_id, member, room = itemgetter("client_id", "member_object", "room_object")(
+        session_data
+    )
+
+    logger.debug(f"[event: stop_stream][{room}] Attempting to stop stream for {member}")
+
+    if member is None or room is None:
+        message = f"Received stop_stream from {member}, but member or room is None. This should not happen."
+        logger.error(message)
+        return {"status": "error", "message": message}
+
+    # In order to stop the stream and end the transcoder thread, set close to True and unset it for the member
+    if member.transcoder:
+        member.transcoder.close = True
+        member.transcoder = None
+    else:
+        message = f"Received stop_stream from {member}, but member.transcoder is None. This should not happen."
+        logger.warn(message)
+
+    # We need to emit a room state update here since room state now includes # of active transcoders
+    await emit_room_state_update(room)
+    # Emit a server state update now that we've changed the number of active transcoders
+    await emit_server_state_update()
+
+    return {"status": "ok", "message": "Stream stopped"}
+
+
+@sio.on("clear_transcript_for_all")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def clear_transcript_for_all(sid):
+    session_data = await get_session_data(sid)
+
+    room = session_data.get("room_object")
+
+    if room:
+        await sio.emit("clear_transcript", room=f"{room.room_id}")
+    else:
+        logger.error("[clear_transcript] room is None. This should not happen.")
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def set_name(sid, name):
+    logger.info(f"[Event: set_name] name={name}")
+    await sio.save_session(sid, {"name": name})

seamless_server/src/connection_tracker.py

@@ -0,0 +1,64 @@
+from logging import Logger
+import time
+
+
+class StreamingConnectionInfo:
+    def __init__(self, address, active_connections, latest_message_received_timestamp):
+        self.address = address
+        self.active_connections = active_connections
+        self.latest_message_received_timestamp = latest_message_received_timestamp
+
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self):
+        return str(
+            {
+                "address": self.address,
+                "active_connections": self.active_connections,
+                "latest_message_received_timestamp": self.latest_message_received_timestamp,
+            }
+        )
+
+
+class ConnectionTracker:
+    def __init__(self, logger: Logger):
+        self.connections = dict()
+        self.logger = logger
+
+    def __str__(self):
+        return str(self.connections)
+
+    def add_connection(self, address):
+        if address not in self.connections:
+            self.connections[address] = StreamingConnectionInfo(address, 1, time.time())
+        else:
+            self.connections[address].active_connections += 1
+            self.connections[address].latest_message_received_timestamp = time.time()
+
+    def log_recent_message(self, address):
+        if address in self.connections:
+            self.connections[address].latest_message_received_timestamp = time.time()
+        else:
+            self.logger.warning(
+                f"Address {address} not found in connection tracker when attempting to log recent message"
+            )
+
+    def remove_connection(self, address):
+        if address in self.connections:
+            self.connections[address].active_connections -= 1
+            if self.connections[address].active_connections < 0:
+                self.logger.warning(
+                    f"Address {address} has negative active connections ({self.connections[address].active_connections})"
+                )
+            if self.connections[address].active_connections <= 0:
+                del self.connections[address]
+        else:
+            self.logger.warning(
+                f"Address {address} not found in connection tracker when attempting to remove it"
+            )
+
+    def get_active_connection_count(self):
+        return sum(
+            [connection.active_connections for connection in self.connections.values()]
+        )

seamless_server/src/room.py

@@ -0,0 +1,64 @@
+# import json
+import uuid
+
+
+class Room:
+    def __init__(self, room_id) -> None:
+        self.room_id = room_id
+        # members is a dict from client_id to Member
+        self.members = {}
+
+        # listeners and speakers are lists of client_id's
+        self.listeners = []
+        self.speakers = []
+
+    def __str__(self) -> str:
+        return f"Room {self.room_id} ({len(self.members)} member{'s' if len(self.members) == 1 else ''})"
+
+    def to_json(self):
+        varsResult = vars(self)
+        # Remember: result is just a shallow copy, so result.members === self.members
+        # Because of that, we need to jsonify self.members without writing over result.members,
+        # which we do here via dictionary unpacking (the ** operator)
+        result = {
+            **varsResult,
+            "members": {key: value.to_json() for (key, value) in self.members.items()},
+            "activeTranscoders": self.get_active_transcoders(),
+        }
+
+        return result
+
+    def get_active_connections(self):
+        return len(
+            [m for m in self.members.values() if m.connection_status == "connected"]
+        )
+
+    def get_active_transcoders(self):
+        return len([m for m in self.members.values() if m.transcoder is not None])
+
+    def get_room_status_dict(self):
+        return {
+            "activeConnections": self.get_active_connections(),
+            "activeTranscoders": self.get_active_transcoders(),
+        }
+
+
+class Member:
+    def __init__(self, client_id, session_id, name) -> None:
+        self.client_id = client_id
+        self.session_id = session_id
+        self.name = name
+        self.connection_status = "connected"
+        self.transcoder = None
+        self.requested_output_type = None
+        self.transcoder_dynamic_config = None
+
+    def __str__(self) -> str:
+        return f"{self.name} (id: {self.client_id[:4]}...) ({self.connection_status})"
+
+    def to_json(self):
+        self_vars = vars(self)
+        return {
+            **self_vars,
+            "transcoder": self.transcoder is not None,
+        }

seamless_server/src/simuleval_agent_directory.py

@@ -0,0 +1,163 @@
+# Creates a directory in which to look up available agents
+
+from typing import List
+from src.simuleval_transcoder import SimulevalTranscoder
+import json
+import logging
+
+logger = logging.getLogger("socketio_server_pubsub")
+
+# fmt: off
+M4T_P0_LANGS = [
+    "eng",
+    "arb", "ben", "cmn", "deu", "fil",
+    "fra", "hin", "ind", "ita", "jpn",
+    "kor", "nld", "por", "rus", "spa",
+    "swh", "tha", "tur", "urd", "vie",
+]
+# fmt: on
+
+
+class NoAvailableAgentException(Exception):
+    pass
+
+
+class AgentWithInfo:
+    def __init__(
+        self,
+        agent,
+        name: str,
+        modalities: List[str],
+        source_langs: List[str],
+        target_langs: List[str],
+        # Supported dynamic params are defined in StreamingTypes.ts
+        dynamic_params: List[str] = [],
+        description="",
+    ):
+        self.agent = agent
+        self.name = name
+        self.description = description
+        self.modalities = modalities
+        self.source_langs = source_langs
+        self.target_langs = target_langs
+        self.dynamic_params = dynamic_params
+
+    def get_capabilities_for_json(self):
+        return {
+            "name": self.name,
+            "description": self.description,
+            "modalities": self.modalities,
+            "sourceLangs": self.source_langs,
+            "targetLangs": self.target_langs,
+            "dynamicParams": self.dynamic_params,
+        }
+
+    @classmethod
+    def load_from_json(cls, config: str):
+        """
+        Takes in JSON array of models to load in, e.g.
+        [{"name": "s2s_m4t_emma-unity2_multidomain_v0.1", "description": "M4T model that supports simultaneous S2S and S2T", "modalities": ["s2t", "s2s"], "sourceLangs": ["arb", "ben", "hin", "ind", "ita", "jpn", "por", "rus", "spa", "swh", "tha", "tur", "urd", "vie"], "targetLangs": ["en"]},
+        {"name": "s2s_m4t_expr-emma_v0.1", "description": "ES-EN expressive model that supports S2S and S2T", "modalities": ["s2t", "s2s"], "sourceLangs": ["arb", "ben", "hin", "ind", "ita", "jpn", "por", "rus", "spa", "swh", "tha", "tur", "urd", "vie"], "targetLangs": ["en"]}]
+        """
+        configs = json.loads(config)
+        agents = []
+        for config in configs:
+            agent = SimulevalTranscoder.build_agent(config["name"])
+            agents.append(
+                AgentWithInfo(
+                    agent=agent,
+                    name=config["name"],
+                    modalities=config["modalities"],
+                    source_langs=config["sourceLangs"],
+                    target_langs=config["targetLangs"],
+                )
+            )
+        return agents
+
+
+class SimulevalAgentDirectory:
+    # Available models. These are the directories where the models can be found, and also serve as an ID for the model.
+    # s2t:
+    s2t_es_en_agent = "s2t_es-en_tt-waitk_multidomain"
+    s2t_en_es_agent = "s2t_en-es_tt-waitk_multidomain"
+    s2t_es_en_emma_agent = "s2t_es-en_emma_multidomain_v0.3"
+    s2t_en_es_emma_agent = "s2t_en-es_emma_multidomain_v0.3"
+    # s2s:
+    s2s_es_en_agent = "s2s_es-en_tt-waitk-unity2_multidomain"
+    s2s_es_en_emma_agent = "s2s_es-en_emma-unity2_multidomain_v0.2"
+    s2s_m4t_expr_emma_agent = "s2s_m4t_expr-emma_v0.3"
+    s2s_m4t_emma_agent = "s2s_m4t_emma-unity2_multidomain_v0.4"
+
+    def __init__(self):
+        self.agents = []
+        self.did_build_and_add_agents = False
+
+    def add_agent(self, agent: AgentWithInfo):
+        self.agents.append(agent)
+
+    def build_agent_if_available(self, model_id, config_name=None):
+        agent = None
+        try:
+            if config_name is not None:
+                agent = SimulevalTranscoder.build_agent(
+                    model_id,
+                    config_name=config_name,
+                )
+            else:
+                agent = SimulevalTranscoder.build_agent(
+                    model_id,
+                )
+        except Exception as e:
+            logger.warning("Failed to build agent %s: %s" % (model_id, e))
+            raise e
+
+        return agent
+
+    def build_and_add_agents(self, models_override=None):
+        if self.did_build_and_add_agents:
+            return
+
+        if models_override is not None:
+            agent_infos = AgentWithInfo.load_from_json(models_override)
+            for agent_info in agent_infos:
+                self.add_agent(agent_info)
+        else:
+            s2s_m4t_expr_agent = self.build_agent_if_available(
+                SimulevalAgentDirectory.s2s_m4t_emma_agent,
+                config_name="vad_s2st_sc_24khz_main.yaml",
+            )
+
+            if s2s_m4t_expr_agent:
+                self.add_agent(
+                    AgentWithInfo(
+                        agent=s2s_m4t_expr_agent,
+                        name=SimulevalAgentDirectory.s2s_m4t_emma_agent,
+                        modalities=["s2t", "s2s"],
+                        source_langs=M4T_P0_LANGS,
+                        target_langs=["eng", "spa", "fra", "deu", "ita", "cmn"],
+                        dynamic_params=["expressive"],
+                        description="ES-EN expressive model that supports S2S and S2T",
+                    )
+                )
+
+        if len(self.agents) == 0:
+            logger.error(
+                "No agents were loaded. This likely means you are missing the actual model files specified in simuleval_agent_directory."
+            )
+
+        self.did_build_and_add_agents = True
+
+    def get_agent(self, name):
+        for agent in self.agents:
+            if agent.name == name:
+                return agent.agent
+        return None
+
+    def get_agent_or_throw(self, name):
+        agent = self.get_agent(name)
+        if agent is None:
+            raise NoAvailableAgentException("No agent found with name= %s" % (name))
+        return agent
+
+    def get_agents_capabilities_list_for_json(self):
+        return [agent.get_capabilities_for_json() for agent in self.agents]

seamless_server/src/simuleval_transcoder.py

@@ -0,0 +1,423 @@
+from simuleval.utils.agent import build_system_from_dir
+from typing import Any, List, Optional, Tuple, Union
+import numpy as np
+import soundfile
+import io
+import asyncio
+from simuleval.agents.pipeline import TreeAgentPipeline
+from simuleval.agents.states import AgentStates
+from simuleval.data.segments import Segment, EmptySegment, SpeechSegment
+import threading
+import math
+import logging
+import sys
+from pathlib import Path
+import time
+from g2p_en import G2p
+import torch
+import traceback
+import time
+import random
+import colorlog
+
+from .speech_and_text_output import SpeechAndTextOutput
+
+MODEL_SAMPLE_RATE = 16_000
+
+logger = logging.getLogger(__name__)
+# logger.propagate = False
+handler = colorlog.StreamHandler(stream=sys.stdout)
+formatter = colorlog.ColoredFormatter(
+    "%(log_color)s[%(asctime)s][%(levelname)s][%(module)s]:%(reset)s %(message)s",
+    reset=True,
+    log_colors={
+        "DEBUG": "cyan",
+        "INFO": "green",
+        "WARNING": "yellow",
+        "ERROR": "red",
+        "CRITICAL": "red,bg_white",
+    },
+)
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+logger.setLevel(logging.WARNING)
+
+
+class OutputSegments:
+    def __init__(self, segments: Union[List[Segment], Segment]):
+        if isinstance(segments, Segment):
+            segments = [segments]
+        self.segments: List[Segment] = [s for s in segments]
+
+    @property
+    def is_empty(self):
+        return all(segment.is_empty for segment in self.segments)
+
+    @property
+    def finished(self):
+        return all(segment.finished for segment in self.segments)
+
+    def compute_length(self, g2p):
+        lengths = []
+        for segment in self.segments:
+            if segment.data_type == "text":
+                lengths.append(len([x for x in g2p(segment.content) if x != " "]))
+            elif segment.data_type == "speech":
+                lengths.append(len(segment.content) / MODEL_SAMPLE_RATE)
+            elif isinstance(segment, EmptySegment):
+                continue
+            else:
+                logger.warning(
+                    f"Unexpected data_type: {segment.data_type} not in 'speech', 'text'"
+                )
+        return max(lengths)
+
+    @classmethod
+    def join_output_buffer(
+        cls, buffer: List[List[Segment]], output: SpeechAndTextOutput
+    ):
+        num_segments = len(buffer[0])
+        for i in range(num_segments):
+            segment_list = [
+                buffer[j][i]
+                for j in range(len(buffer))
+                if buffer[j][i].data_type is not None
+            ]
+            if len(segment_list) == 0:
+                continue
+            if len(set(segment.data_type for segment in segment_list)) != 1:
+                logger.warning(
+                    f"Data type mismatch at {i}: {set(segment.data_type for segment in segment_list)}"
+                )
+                continue
+            data_type = segment_list[0].data_type
+            if data_type == "text":
+                if output.text is not None:
+                    logger.warning("Multiple text outputs, overwriting!")
+                output.text = " ".join([segment.content for segment in segment_list])
+            elif data_type == "speech":
+                if output.speech_samples is not None:
+                    logger.warning("Multiple speech outputs, overwriting!")
+                speech_out = []
+                for segment in segment_list:
+                    speech_out += segment.content
+                output.speech_samples = speech_out
+                output.speech_sample_rate = segment.sample_rate
+            elif isinstance(segment_list[0], EmptySegment):
+                continue
+            else:
+                logger.warning(
+                    f"Invalid output buffer data type: {data_type}, expected 'speech' or 'text"
+                )
+
+        return output
+
+    def __repr__(self) -> str:
+        repr_str = str(self.segments)
+        return f"{self.__class__.__name__}(\n\t{repr_str}\n)"
+
+
+class SimulevalTranscoder:
+    def __init__(self, agent, sample_rate, debug, buffer_limit):
+        self.agent = agent
+        self.input_queue = asyncio.Queue()
+        self.output_queue = asyncio.Queue()
+        self.states = self.agent.build_states()
+        if debug:
+            self.get_states_root().debug = True
+        self.incoming_sample_rate = sample_rate
+        self.close = False
+        self.g2p = G2p()
+
+        # buffer all outgoing translations within this amount of time
+        self.output_buffer_idle_ms = 5000
+        self.output_buffer_size_limit = (
+            buffer_limit  # phonemes for text, seconds for speech
+        )
+        self.output_buffer_cur_size = 0
+        self.output_buffer: List[List[Segment]] = []
+        self.speech_output_sample_rate = None
+
+        self.last_output_ts = time.time() * 1000
+        self.timeout_ms = (
+            30000  # close the transcoder thread after this amount of silence
+        )
+        self.first_input_ts = None
+        self.first_output_ts = None
+        self.debug = debug
+        self.debug_ts = f"{time.time()}_{random.randint(1000, 9999)}"
+        if self.debug:
+            debug_folder = Path(__file__).resolve().parent.parent / "debug"
+            self.test_incoming_wav = soundfile.SoundFile(
+                debug_folder / f"{self.debug_ts}_test_incoming.wav",
+                mode="w+",
+                format="WAV",
+                subtype="PCM_16",
+                samplerate=self.incoming_sample_rate,
+                channels=1,
+            )
+            self.get_states_root().test_input_segments_wav = soundfile.SoundFile(
+                debug_folder / f"{self.debug_ts}_test_input_segments.wav",
+                mode="w+",
+                format="WAV",
+                samplerate=MODEL_SAMPLE_RATE,
+                channels=1,
+            )
+
+    def get_states_root(self) -> AgentStates:
+        if isinstance(self.agent, TreeAgentPipeline):
+            # self.states is a dict
+            return self.states[self.agent.source_module]
+        else:
+            # self.states is a list
+            return self.states[0]
+
+    def reset_states(self):
+        if isinstance(self.agent, TreeAgentPipeline):
+            states_iter = self.states.values()
+        else:
+            states_iter = self.states
+        for state in states_iter:
+            state.reset()
+
+    def debug_log(self, *args):
+        if self.debug:
+            logger.info(*args)
+
+    @classmethod
+    def build_agent(cls, model_path, config_name="vad_s2st_main.yaml"):
+        logger.info(f"Building simuleval agent: {model_path}, {config_name}")
+        agent = build_system_from_dir(
+            Path(__file__).resolve().parent.parent / f"models/{model_path}",
+            config_name=config_name,
+        )
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        agent.to(device, fp16=True)
+        logger.info(
+            f"Successfully built simuleval agent {model_path} on device {device}"
+        )
+
+        return agent
+
+    def process_incoming_bytes(self, incoming_bytes, dynamic_config):
+        # TODO: We probably want to do some validation on dynamic_config to ensure it has what we needs
+        segment, sr = self._preprocess_wav(incoming_bytes)
+        segment = SpeechSegment(
+            content=segment,
+            sample_rate=sr,
+            tgt_lang=dynamic_config.get("targetLanguage"),
+            config=dynamic_config,
+        )
+        # # segment is array([0, 0, 0, ..., 0, 0, 0], dtype=int16)
+        self.input_queue.put_nowait(segment)
+
+    def get_input_segment(self):
+        if self.input_queue.empty():
+            return None
+        chunk = self.input_queue.get_nowait()
+        self.input_queue.task_done()
+        return chunk
+
+    def convert_waveform(
+        self,
+        waveform: Union[np.ndarray, torch.Tensor],
+        sample_rate: int,
+        normalize_volume: bool = False,
+        to_mono: bool = False,
+        to_sample_rate: Optional[int] = None,
+    ) -> Tuple[Union[np.ndarray, torch.Tensor], int]:
+        """convert a waveform:
+        - to a target sample rate
+        - from multi-channel to mono channel
+        - volume normalization
+
+        Args:
+            waveform (numpy.ndarray or torch.Tensor): 2D original waveform
+                (channels x length)
+            sample_rate (int): original sample rate
+            normalize_volume (bool): perform volume normalization
+            to_mono (bool): convert to mono channel if having multiple channels
+            to_sample_rate (Optional[int]): target sample rate
+        Returns:
+            waveform (numpy.ndarray): converted 2D waveform (channels x length)
+            sample_rate (float): target sample rate
+        """
+        try:
+            import torchaudio.sox_effects as ta_sox
+        except ImportError:
+            raise ImportError("Please install torchaudio: pip install torchaudio")
+
+        effects = []
+        if normalize_volume:
+            effects.append(["gain", "-n"])
+        if to_sample_rate is not None and to_sample_rate != sample_rate:
+            effects.append(["rate", f"{to_sample_rate}"])
+        if to_mono and waveform.shape[0] > 1:
+            effects.append(["channels", "1"])
+        if len(effects) > 0:
+            is_np_input = isinstance(waveform, np.ndarray)
+            _waveform = torch.from_numpy(waveform) if is_np_input else waveform
+            converted, converted_sample_rate = ta_sox.apply_effects_tensor(
+                _waveform, sample_rate, effects
+            )
+            if is_np_input:
+                converted = converted.numpy()
+            return converted, converted_sample_rate
+        return waveform, sample_rate
+
+    def _preprocess_wav(self, data: Any) -> Tuple[np.ndarray, int]:
+        segment, sample_rate = soundfile.read(
+            io.BytesIO(data),
+            dtype="float32",
+            always_2d=True,
+            frames=-1,
+            start=0,
+            format="RAW",
+            subtype="PCM_16",
+            samplerate=self.incoming_sample_rate,
+            channels=1,
+        )
+        if self.debug:
+            self.test_incoming_wav.seek(0, soundfile.SEEK_END)
+            self.test_incoming_wav.write(segment)
+
+        segment = segment.T
+        segment, new_sample_rate = self.convert_waveform(
+            segment,
+            sample_rate,
+            normalize_volume=False,
+            to_mono=True,
+            to_sample_rate=MODEL_SAMPLE_RATE,
+        )
+
+        assert MODEL_SAMPLE_RATE == new_sample_rate
+        segment = segment.squeeze(axis=0)
+        return segment, new_sample_rate
+
+    def process_pipeline_impl(self, input_segment):
+        try:
+            with torch.no_grad():
+                output_segment = OutputSegments(
+                    self.agent.pushpop(input_segment, self.states)
+                )
+            if (
+                self.get_states_root().first_input_ts is not None
+                and self.first_input_ts is None
+            ):
+                # TODO: this is hacky
+                self.first_input_ts = self.get_states_root().first_input_ts
+
+            if not output_segment.is_empty:
+                self.output_queue.put_nowait(output_segment)
+
+            if output_segment.finished:
+                self.debug_log("OUTPUT SEGMENT IS FINISHED. Resetting states.")
+
+                self.reset_states()
+
+                if self.debug:
+                    # when we rebuild states, this value is reset to whatever
+                    # is in the system dir config, which defaults debug=False.
+                    self.get_states_root().debug = True
+        except Exception as e:
+            logger.error(f"Got exception while processing pipeline: {e}")
+            traceback.print_exc()
+        return input_segment
+
+    def process_pipeline_loop(self):
+        if self.close:
+            return  # closes the thread
+
+        self.debug_log("processing_pipeline")
+        while not self.close:
+            input_segment = self.get_input_segment()
+            if input_segment is None:
+                if self.get_states_root().is_fresh_state:  # TODO: this is hacky
+                    time.sleep(0.3)
+                else:
+                    time.sleep(0.03)
+                continue
+            self.process_pipeline_impl(input_segment)
+        self.debug_log("finished processing_pipeline")
+
+    def process_pipeline_once(self):
+        if self.close:
+            return
+
+        self.debug_log("processing pipeline once")
+        input_segment = self.get_input_segment()
+        if input_segment is None:
+            return
+        self.process_pipeline_impl(input_segment)
+        self.debug_log("finished processing_pipeline_once")
+
+    def get_output_segment(self):
+        if self.output_queue.empty():
+            return None
+
+        output_chunk = self.output_queue.get_nowait()
+        self.output_queue.task_done()
+        return output_chunk
+
+    def start(self):
+        self.debug_log("starting transcoder in a thread")
+        threading.Thread(target=self.process_pipeline_loop).start()
+
+    def first_translation_time(self):
+        return round((self.first_output_ts - self.first_input_ts) / 1000, 2)
+
+    def get_buffered_output(self) -> SpeechAndTextOutput:
+        now = time.time() * 1000
+        self.debug_log(f"get_buffered_output queue size: {self.output_queue.qsize()}")
+        while not self.output_queue.empty():
+            tmp_out = self.get_output_segment()
+            if tmp_out and tmp_out.compute_length(self.g2p) > 0:
+                if len(self.output_buffer) == 0:
+                    self.last_output_ts = now
+                self._populate_output_buffer(tmp_out)
+                self._increment_output_buffer_size(tmp_out)
+
+                if tmp_out.finished:
+                    self.debug_log("tmp_out.finished")
+                    res = self._gather_output_buffer_data(final=True)
+                    self.debug_log(f"gathered output data: {res}")
+                    self.output_buffer = []
+                    self.increment_output_buffer_size = 0
+                    self.last_output_ts = now
+                    self.first_output_ts = now
+                    return res
+            else:
+                self.debug_log("tmp_out.compute_length is not > 0")
+
+        if len(self.output_buffer) > 0 and (
+            now - self.last_output_ts >= self.output_buffer_idle_ms
+            or self.output_buffer_cur_size >= self.output_buffer_size_limit
+        ):
+            self.debug_log(
+                "[get_buffered_output] output_buffer is not empty. getting res to return."
+            )
+            self.last_output_ts = now
+            res = self._gather_output_buffer_data(final=False)
+            self.debug_log(f"gathered output data: {res}")
+            self.output_buffer = []
+            self.output_buffer_phoneme_count = 0
+            self.first_output_ts = now
+            return res
+        else:
+            self.debug_log("[get_buffered_output] output_buffer is empty...")
+            return None
+
+    def _gather_output_buffer_data(self, final):
+        output = SpeechAndTextOutput()
+        output.final = final
+        output = OutputSegments.join_output_buffer(self.output_buffer, output)
+        return output
+
+    def _increment_output_buffer_size(self, segment: OutputSegments):
+        self.output_buffer_cur_size += segment.compute_length(self.g2p)
+
+    def _populate_output_buffer(self, segment: OutputSegments):
+        self.output_buffer.append(segment.segments)
+
+    def _compute_phoneme_count(self, string: str) -> int:
+        return len([x for x in self.g2p(string) if x != " "])

seamless_server/src/speech_and_text_output.py

@@ -0,0 +1,15 @@
+# Provides a container to return both speech and text output from our model at the same time
+
+
+class SpeechAndTextOutput:
+    def __init__(
+        self,
+        text: str = None,
+        speech_samples: list = None,
+        speech_sample_rate: float = None,
+        final: bool = False,
+    ):
+        self.text = text
+        self.speech_samples = speech_samples
+        self.speech_sample_rate = speech_sample_rate
+        self.final = final

seamless_server/src/timing.ipynb

@@ -0,0 +1,169 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Text(1, 0, 'nonincremental'), Text(2, 0, 'incremental')]"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAktElEQVR4nO3de3BU5f3H8c9mkUAwiYLkggSTGjBQIuEmJDSStLQMRYediEWUQalYFbAKKG3QSsefJVZJgRlBRMpoRQQNMXa2XqAoGGALQsAxCsotBCUJXmrCNdHd/f3hZGVrbptkeXY379fMGd1znnP2G/RkPzz7PM+xuN1utwAAAAwJM10AAADo2AgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIzqZLqAlnC5XDpx4oQiIyNlsVhMlwMAAFrA7Xbr1KlT6tWrl8LCGu//CIowcuLECSUkJJguAwAAtMLx48fVu3fvRo8HRRiJjIyU9P0PExUVZbgaAADQEjU1NUpISPB8jjcmKMJI/VczUVFRhBEAAIJMc0MsGMAKAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMCooFj1Dx+F0OlVcXKyKigrFx8crMzNTVqvVdFkAAD+iZwQBo7CwUMnJycrOztatt96q7OxsJScnq7Cw0HRpAAA/IowgIBQWFmrixIlKTU2Vw+HQqVOn5HA4lJqaqokTJxJIACCEWdxut9t0Ec2pqalRdHS0qqureTZNCHI6nUpOTlZqaqqKioq8HjPtcrlks9lUWlqqgwcP8pUNAASRln5+0zMC44qLi1VWVqb58+d7BRFJCgsLU25uro4ePari4mJDFQIA/IkwAuMqKiokSQMHDmzweP3++nYAgNBCGIFx8fHxkqTS0tIGj9fvr28HAAgthBEYl5mZqcTERC1cuFAul8vrmMvlUl5enpKSkpSZmWmoQgCAPxFGYJzValV+fr7sdrtsNpvXbBqbzSa73a5FixYxeBUAQhSLniEg5OTkqKCgQHPnzlVGRoZnf1JSkgoKCpSTk2OwOgCAPzG1FwGFFVgBIHS09PObnhEEFKvVqqysLNNlAAAuIsaMAAAAowgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMIowAgAAjCKMAAAAowgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMIowAgAAjPIpjOTl5Wn48OGKjIxUTEyMbDabPvnkk2bPe/XVV5WSkqIuXbooNTVVb7zxRqsLBgAAocWnMLJ161bNnDlT//nPf7Rp0yZ9++23+tWvfqUzZ840es6OHTs0efJk3Xnnndq7d69sNptsNptKS0vbXDwAAAh+Frfb7W7tyV988YViYmK0detWXX/99Q22mTRpks6cOSO73e7ZN3LkSKWlpWnFihUtep+amhpFR0erurpaUVFRrS0XAABcRC39/G7TmJHq6mpJUvfu3Rtt43A4NGbMGK99Y8eOlcPhaMtbAwCAENGptSe6XC498MADGjVqlAYOHNhou8rKSsXGxnrti42NVWVlZaPn1NbWqra21vO6pqamtWUCAIAA1+qekZkzZ6q0tFTr1q1rz3okfT9QNjo62rMlJCS0+3sAAIDA0KowMmvWLNntdr377rvq3bt3k23j4uJUVVXlta+qqkpxcXGNnpObm6vq6mrPdvz48daUCQAAgoBPYcTtdmvWrFl67bXX9M477ygpKanZc9LT07V582avfZs2bVJ6enqj54SHhysqKsprAwAAocmnMSMzZ87U2rVr9frrrysyMtIz7iM6Olpdu3aVJE2dOlVXXnml8vLyJEn333+/Ro8erfz8fI0fP17r1q3T7t27tXLlynb+UQAAQDDyqWfkmWeeUXV1tbKyshQfH+/Z1q9f72lTXl6uiooKz+uMjAytXbtWK1eu1KBBg1RQUKCioqImB70CAICOo03rjFwsrDMCAEDwuSjrjAAAALQVYQQAABhFGAEAAEYRRgAAgFGEEQAAYBRhBAAAGEUYAQAARhFGAACAUYQRAABgFGEEAAAYRRgBAABGEUYAAIBRhBEAAGAUYQQAABhFGAEAAEYRRgAAgFGEEQAAYBRhBAAAGEUYAQAARhFGAACAUYQRAABgFGEEAAAYRRgBAABGEUYAAIBRhBEAAGAUYQQAABhFGAEAAEYRRgAAgFGEEQAAYBRhBAAAGEUYAQAARhFGAACAUYQRAABgFGEEAAAYRRgBAABGEUYAAIBRhBEAAGAUYQQAABhFGAEAAEYRRgAAgFGEEQAAYBRhBAAAGEUYAQAARhFGAACAUYQRAABgFGEEAAAYRRgBAABGEUYAAIBRhBEAAGAUYQQAABhFGAEAAEYRRgAAgFGEEQAAYBRhBAAAGEUYAQAARhFGAACAUYQRAABgFGEEAAAYRRgBAABGEUYAAIBRhBEAAGAUYQQAABjVyXQBwIWcTqeKi4tVUVGh+Ph4ZWZmymq1mi4LAOBH9IwgYBQWFio5OVnZ2dm69dZblZ2dreTkZBUWFpouDQDgR4QRBITCwkJNnDhRqampcjgcOnXqlBwOh1JTUzVx4kQCCQCEMIvb7XabLqI5NTU1io6OVnV1taKiokyXg3bmdDqVnJys1NRUFRUVKSzsh4zscrlks9lUWlqqgwcP8pUNAASRln5+0zMC44qLi1VWVqb58+d7BRFJCgsLU25uro4ePari4mJDFQIA/MnnMPLee+/pxhtvVK9evWSxWFRUVNRk+y1btshisfxoq6ysbG3NCDEVFRWSpIEDBzZ4vH5/fTsAQGjxOYycOXNGgwYN0rJly3w675NPPlFFRYVni4mJ8fWtEaLi4+MlSaWlpQ0er99f3w4AEFp8nto7btw4jRs3zuc3iomJ0WWXXebzeQh9mZmZSkxM1MKFCxscM5KXl6ekpCRlZmYarBIA4C8XbcxIWlqa4uPj9ctf/lLbt29vsm1tba1qamq8NoQuq9Wq/Px82e122Ww2r9k0NptNdrtdixYtYvAqAIQov4eR+Ph4rVixQhs2bNCGDRuUkJCgrKwslZSUNHpOXl6eoqOjPVtCQoK/y4RhOTk5Kigo0IcffqiMjAxFRUUpIyNDpaWlKigoUE5OjukSAQB+0qapvRaLRa+99ppsNptP540ePVp9+vTRiy++2ODx2tpa1dbWel7X1NQoISGBqb0dACuwAkDoaOnUXiPLwV933XXatm1bo8fDw8MVHh5+EStCoLBarcrKyjJdBgDgIjKyzsi+ffuYGQEAACS1omfk9OnTOnTokOf10aNHtW/fPnXv3l19+vRRbm6uPv/8c/3jH/+QJC1ZskRJSUn66U9/qvPnz2vVqlV65513tHHjxvb7KQAAQNDyOYzs3r1b2dnZntdz5syRJN1+++16/vnnVVFRofLycs/xuro6zZ07V59//rkiIiJ07bXX6t///rfXNQAAQMfFs2kAAIBf8GwaAAAQFAgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMIowAgAAjCKMAAAAowgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMIowAgAAjCKMAAAAowgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMIowAgAAjCKMAAAAowgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMIowAgAAjCKMAAAAowgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMIowAgAAjCKMAAAAowgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAon8PIe++9pxtvvFG9evWSxWJRUVFRs+ds2bJFQ4YMUXh4uJKTk/X888+3olQAABCKfA4jZ86c0aBBg7Rs2bIWtT969KjGjx+v7Oxs7du3Tw888ICmT5+ut99+2+diAQBA6Onk6wnjxo3TuHHjWtx+xYoVSkpKUn5+viSpf//+2rZtmxYvXqyxY8f6+vYAACDE+H3MiMPh0JgxY7z2jR07Vg6Ho9FzamtrVVNT47UBAIDQ5PcwUllZqdjYWK99sbGxqqmp0blz5xo8Jy8vT9HR0Z4tISHB32UCAABDAnI2TW5urqqrqz3b8ePHTZcEAAD8xOcxI76Ki4tTVVWV176qqipFRUWpa9euDZ4THh6u8PBwf5cGAAACgN97RtLT07V582avfZs2bVJ6erq/3xoAAAQBn8PI6dOntW/fPu3bt0/S91N39+3bp/Lycknff8UydepUT/t77rlHR44c0bx583TgwAEtX75cr7zyimbPnt0+PwEAAAhqPoeR3bt3a/DgwRo8eLAkac6cORo8eLAeffRRSVJFRYUnmEhSUlKS/vWvf2nTpk0aNGiQ8vPztWrVKqb1AgAASZLF7Xa7TRfRnJqaGkVHR6u6ulpRUVGmywEAAC3Q0s9vvw9gBQBAkpxOp4qLi1VRUaH4+HhlZmbKarWaLgsBICCn9gIAQkthYaGSk5OVnZ2tW2+9VdnZ2UpOTlZhYaHp0hAACCMAAL8qLCzUxIkTlZqaKofDoVOnTsnhcCg1NVUTJ04kkIAxIwAA/3E6nUpOTlZqaqqKiooUFvbD34FdLpdsNptKS0t18OBBvrIJQS39/KZnBADgN8XFxSorK9P8+fO9gogkhYWFKTc3V0ePHlVxcbGhChEICCMAAL+pqKiQJA0cOLDB4/X769uhYyKMAAD8Jj4+XpJUWlra4PH6/fXt0DERRgAAfpOZmanExEQtXLhQLpfL65jL5VJeXp6SkpKUmZlpqEIEAsIIAMBvrFar8vPzZbfbZbPZvGbT2Gw22e12LVq0iMGrHRyLngEA/ConJ0cFBQWaO3euMjIyPPuTkpJUUFCgnJwcg9UhEDC1FwBwUbACa8fDcvAISvyyAkKX1WpVVlaW6TIQgBgzgoDBctEA0DERRhAQWC4aADouxozAOJaLBoDQxHLwCBosFw0AHRthBMaxXDQAdGyEERjHctEA0LERRmAcy0UDQMdGGIFxLBcNAB0bi54hILBcNAB0XEztRUBhBVYACB1M7QUAAEGBMIKAwXLwANAxEUYQEFgOHgA6LsaMwLgLl4PfsGGDtm/f7hkzMmrUKN10000sBw8AQYgxIwga9cvBZ2RkqF+/fl5f0/Tr10/p6eksBw8AIYwwAuPql3mfP39+g1/TPPzww17tAAChhXVGYFxMTIwkadSoUV5P7R05cqSKioo0evRobdu2zdMOABBa6BlBwAuCYU0AgDYgjMC4kydPSpK2bdvW4HLw27dv92oHIDg5nU5t2bJFL7/8srZs2SKn02m6JAQIwgiMq38ab15enj788ENlZGQoKipKGRkZKi0t1cKFC73aAQg+rCOEphBGYFz9U3t37NihTz/9VO+++67Wrl2rd999V5988okcDgdP7QWCGOsIoTmsM4KAUP/L6oYbblBubq4GDhyo0tJS5eXlyW6387A8IEhduI7QhQPUJcnlcslms7GOUAhjnREElfqn9jb0NQ1BBAhe9esIzZ8/3yuISFJYWJhyc3NZRwhM7UXgyMnJ0YQJE3hqLxBC6tcHGjhwYIPH6/ezjlDHRs8IAMBv6geel5aWNni8fj8D1Ds2wggCBqPtgdBTP0B94cKFcrlcXsdcLpfy8vIYoA7CCAIDo+2B0GS1WpWfny+73d7gOkJ2u12LFi3i69gOjtk0MI7R9kDoKyws1Ny5c1VWVubZl5SUpEWLFjFAPYS19PObMALjtmzZouzsbDkcDo0cOfJHxx0OhzIyMvTuu+8qKyvr4hcIoF04nU4GqHcwLf38ZjYNjLtwtH1Dv6wYbQ+EBqvVyl8o0CDCCIyrH0X/9NNP69lnn/Xqxk1MTNTvfvc7r3YAgNDCAFYYl5mZqZ49e3pWXr1wgNvAgQM1f/58xcTEMNoeAEIUYQQBwWKxeP7d7XZ7NgBA6COMwLji4mKdPHlSeXl5Ki0t9VoO/qOPPtLChQt18uRJlosGgBBFGIFx9QNTZ82apUOHDnk9tffgwYOaNWuWVzsAQGghjMA4losGgI6NdUZgXP2iZ1dccYW++OILHTt2zHPsqquuUs+ePfXVV1+x6BkABJmWfn7TMwLjrFarbr75Zu3evVvnz5/XypUrdeLECa1cuVLnz5/X7t27NXHiRIIIAIQoekZg3IU9I19++eWPlovu0aMHPSMAEIRYgRVBo7i4WGVlZXr55Zc1fPjwH63AumvXLmVkZKi4uJjVGwEgBBFGYNyFy8E3tFw0y8EDQGhjzAiMYzYNAHRshBEYl5mZqcTERC1cuFAul8vrmMvlUl5enpKSklgOHgBCFGEExlmtVuXn58tut8tms3k9m8Zms8lut2vRokUMXgWAEMWYEQSEnJwcFRQUaO7cucrIyPDsT0pKUkFBgXJycgxWBwDwJ6b2IqA4nc4fzaahRwQAghNTexGUGppNAwAIbYwZAQAARhFGAACAUYQRAABgFGNGcNGcPXtWBw4caLbduXPnVFZWpsTERHXt2rXJtikpKYqIiGivEgEABhBGcNEcOHBAQ4cObddr7tmzR0OGDGnXawIALi7CCC6alJQU7dmzp9l2+/fv15QpU7RmzRr179+/2WsCAIIbYQQXTUREhE+9GP3796fXAwA6gFYNYF22bJkSExPVpUsXjRgxQrt27Wq07fPPPy+LxeK1denSpdUFAwCA0OJzGFm/fr3mzJmjBQsWqKSkRIMGDdLYsWN18uTJRs+JiopSRUWFZzt27FibigYAAKHD5zDyt7/9TXfddZemTZumAQMGaMWKFYqIiNDq1asbPcdisSguLs6zxcbGtqloAEDwqaur05IlS3TfffdpyZIlqqurM10SAoRPYaSurk579uzRmDFjfrhAWJjGjBkjh8PR6HmnT5/WVVddpYSEBE2YMEEfffRRk+9TW1urmpoarw0AELzmzZunbt26afbs2Xr66ac1e/ZsdevWTfPmzTNdGgKAT2Hkyy+/lNPp/FHPRmxsrCorKxs855prrtHq1av1+uuva82aNXK5XMrIyNBnn33W6Pvk5eUpOjrasyUkJPhSJgAggMybN09PPfWUevTooeeee04VFRV67rnn1KNHDz311FMEEvj21N4TJ07oyiuv1I4dO5Senu7ZP2/ePG3dulU7d+5s9hrffvut+vfvr8mTJ+v//u//GmxTW1ur2tpaz+uamholJCTw1N4OoqSkREOHDmUNESAE1NXVqVu3burRo4c+++wzder0wyTO7777Tr1799ZXX32lM2fOqHPnzgYrhT+09Km9PvWMXHHFFbJaraqqqvLaX1VVpbi4uBZd45JLLtHgwYN16NChRtuEh4crKirKawMABJ/ly5fru+++0+OPP+4VRCSpU6dOeuyxx/Tdd99p+fLlhipEIPApjHTu3FlDhw7V5s2bPftcLpc2b97s1VPSFKfTqQ8//FDx8fG+VQoACDqHDx+WJN1www0NHq/fX98OHZPPs2nmzJmj5557Ti+88IL279+ve++9V2fOnNG0adMkSVOnTlVubq6n/WOPPaaNGzfqyJEjKikp0ZQpU3Ts2DFNnz69/X4KAEBAuvrqqyVJdru9wdk0drvdqx06Jp9XYJ00aZK++OILPfroo6qsrFRaWpreeustz6DW8vJyhYX9kHH++9//6q677lJlZaUuv/xyDR06VDt27NCAAQPa76cAAASkGTNm6KGHHtL999+ve+65R06n03PswQcfVHh4uDp16qQZM2YYrBKm+TSA1ZSWDoBBaGAAKxBarrvuOr3//vuyWCy67bbbNHfuXOXn5+ull16S2+3W8OHDm1zJG8GrpZ/fPJsGAOA3dXV12rt3ryIiInT+/HmtWbNGa9askSRZrVaFh4dr7969qqurYzZNB9aqZ9MAANAS9bNpli5dqnPnzmnx4sWaNWuWFi9erLNnz2rx4sXMpgE9IwAA/7lwNk3nzp31wAMPeB1nNg0kekYAAH504WyahjCbBhIDWBGAGMAKhA5WYO3Y/LICKwAAvujcubNmz56tqqoq9e7dWytXrtSJEye0cuVK9e7dW1VVVZo9ezZBpINjzAgAwK+efPJJSdLixYt19913e/Z36tRJDz30kOc4Oi7CCADA75588kk9/vjjWr58uQ4fPqyrr75aM2bMoEcEkggjAICLpKHZNIDEmBEAAGAYPSMAgDY7e/asDhw40Gy7c+fOqaysTImJieratWuTbVNSUhQREdFeJSKAEUYAAG124MABDR06tF2vyfT+joMwAgBos5SUFO3Zs6fZdvv379eUKVO0Zs0a9e/fv9lromMgjAAA2iwiIsKnXoz+/fvT6wEPBrACAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCim9qLdHDx4UKdOnWrzdfbv3+/1z7aIjIxU375923wdAID/EEbQLg4ePKh+/fq16zWnTJnSLtf59NNPCSQAEMAII2gX9T0iLVlVsTm+PLuiKfUrPbZHbw0AwH8II2hX7bWq4qhRo9qhGgBAMGAAKwAAMIowAgAAjCKMAAAAowgjAADAKMIIAAAwijACAACMYmovAKBZrLAMfyKMAACaxArL8DfCCACgSaywDH8jjAAAWoQVluEvhBG0C8t35zU4Lkxdv/lUOhEY46K7fvOpBseFyfLdedOlAACaQBhBu+hyulwld18qvXe39J7par7XX1LJ3Zdq/+lySRmmywEANIIwgnZx/tI+GvLsab300kvqn5JiuhxJ0v4DB3Tbbbfp77/uY7oUAEATCCNoF+5OXbS30qVzl/WTeqWZLkeSdK7Spb2VLrk7dTFdCgCgCYHx5T4AAOiwCCMAAMAovqYBADSJ2XLwN8IIAKBJzJaDvxFGAABNYrYc/I0wAgBoErPl4G+B8eUfAADosOgZQbs4e/asJKmkpKTN12rPB2kBAAIfYQTt4sCBA5Kku+66y3AlPxYZGWm6BABAEwgjaBc2m02SlJKSooiIiDZdq/7R4O3xuPLIyEj17du3TdcAAPgXYQTt4oorrtD06dPb9Zrt9bhyAG3D17DwN8IIAKBJfA0LfyOMAACaxNew8DfCCACgSXwNC39jnREAAGAUYQQAABhFGAEAAEYRRgAAgFGEEQAAYBRhBAAAGEUYAQAARhFGAACAUYQRAABgFCuw4qI5e/as5xkXTal/AFZLHoTVHstTA2g77m+0hcXtdrtNF9GcmpoaRUdHq7q6WlFRUabLQSuVlJRo6NCh7XrNPXv2sKQ0EAC4v9GQln5+0zOCiyYlJUV79uxptp0vjxhPSUlpr/IAtAH3N9qCnhEAAOAXLf38ZgArAAAwqlVhZNmyZUpMTFSXLl00YsQI7dq1q8n2r776qlJSUtSlSxelpqbqjTfeaFWxAAAg9PgcRtavX685c+ZowYIFKikp0aBBgzR27FidPHmywfY7duzQ5MmTdeedd2rv3r2y2Wyy2WwqLS1tc/EAACD4+TxmZMSIERo+fLiefvppSZLL5VJCQoLuu+8+/fGPf/xR+0mTJunMmTOy2+2efSNHjlRaWppWrFjRovdkzAgAAMHHL2NG6urqtGfPHo0ZM+aHC4SFacyYMXI4HA2e43A4vNpL0tixYxttL0m1tbWqqanx2gAAQGjyKYx8+eWXcjqdio2N9dofGxurysrKBs+prKz0qb0k5eXlKTo62rMlJCT4UiYAAAgiATmbJjc3V9XV1Z7t+PHjpksCAAB+4tOiZ1dccYWsVquqqqq89ldVVSkuLq7Bc+Li4nxqL0nh4eEKDw/3pTQAABCkfOoZ6dy5s4YOHarNmzd79rlcLm3evFnp6ekNnpOenu7VXpI2bdrUaHsAANCx+Lwc/Jw5c3T77bdr2LBhuu6667RkyRKdOXNG06ZNkyRNnTpVV155pfLy8iRJ999/v0aPHq38/HyNHz9e69at0+7du7Vy5cr2/UkAAEBQ8jmMTJo0SV988YUeffRRVVZWKi0tTW+99ZZnkGp5ebnCwn7ocMnIyNDatWv1yCOPaP78+erbt6+Kioo0cODA9vspAABA0OLZNAAAwC94Ng0AAAgKPn9NY0J95w2LnwEAEDzqP7eb+xImKMLIqVOnJInFzwAACEKnTp1SdHR0o8eDYsyIy+XSiRMnFBkZKYvFYroc+FlNTY0SEhJ0/PhxxggBIYb7u2Nxu906deqUevXq5TW55X8FRc9IWFiYevfubboMXGRRUVH8sgJCFPd3x9FUj0g9BrACAACjCCMAAMAowggCTnh4uBYsWMDziYAQxP2NhgTFAFYAABC66BkBAABGEUYAAIBRhBEAAGAUYQTNysrK0gMPPGC6jKB0xx13yGazmS4DHQj3a+twr5oVFIuewazCwkJdcsklpsswKisrS2lpaVqyZInpUoAmdfT7lXs1OBFG0Kzu3bv79fp1dXXq3LmzX98D6Cj8eb9yr8Jf+JomyGVlZen3v/+95s2bp+7duysuLk5//vOfPcfLy8s1YcIEXXrppYqKitJvfvMbVVVVeY7/+c9/Vlpaml588UUlJiYqOjpat9xyi+fhhPXvcWG3b2JiohYuXKjf/va3ioyMVJ8+fbRy5Uqvuj777DNNnjxZ3bt3V7du3TRs2DDt3LnT6z1XrVqlpKQkdenSRZL0zTffaPr06erZs6eioqL085//XB988MGPal29erX69OmjSy+9VDNmzJDT6dSTTz6puLg4xcTE6C9/+YtXLS29bmN/BnfccYe2bt2qpUuXymKxyGKxqKysTE6nU3feeaeSkpLUtWtXXXPNNVq6dGkr/0sC7ePC+5V7lXs1WBBGQsALL7ygbt26aefOnXryySf12GOPadOmTXK5XJowYYK+/vprbd26VZs2bdKRI0c0adIkr/MPHz6soqIi2e122e12bd26VU888UST75mfn69hw4Zp7969mjFjhu6991598sknkqTTp09r9OjR+vzzz/XPf/5TH3zwgebNmyeXy+U5/9ChQ9qwYYMKCwu1b98+SdLNN9+skydP6s0339SePXs0ZMgQ/eIXv9DXX3/tVeubb76pt956Sy+//LL+/ve/a/z48frss8+0detW/fWvf9Ujjzzi+WXqy3Ub+zNYunSp0tPTddddd6miokIVFRVKSEiQy+VS79699eqrr+rjjz/Wo48+qvnz5+uVV15p3X9IwA+4V7lXg4IbQW306NHun/3sZ177hg8f7v7DH/7g3rhxo9tqtbrLy8s9xz766CO3JPeuXbvcbrfbvWDBAndERIS7pqbG0+ahhx5yjxgxwus97r//fs/rq666yj1lyhTPa5fL5Y6JiXE/88wzbrfb7X722WfdkZGR7q+++qrBmhcsWOC+5JJL3CdPnvTsKy4udkdFRbnPnz/v1fbqq692P/vss43WOnbsWHdiYqLb6XR69l1zzTXuvLy8Nl23uT+DxsycOdN90003eV7ffvvt7gkTJjR7HtBeLvx/lXu1cdyrgYUxIyHg2muv9XodHx+vkydPav/+/UpISFBCQoLn2IABA3TZZZdp//79Gj58uKTvu3IjIyN/dH5L39NisSguLs5zzr59+zR48OAmv7u+6qqr1LNnT8/rDz74QKdPn1aPHj282p07d06HDx/2vP7fWmNjY2W1Wr0eTR0bG+uppbXXbcmfgSQtW7ZMq1evVnl5uc6dO6e6ujqlpaU1ex5wsXCvfo97NbARRkLA/46ct1gsXt2s/ji/qXO6du3a7Ht269bN6/Xp06cVHx+vLVu2/KjtZZdd1uT7NlVLW67b3J/BunXr9OCDDyo/P1/p6emKjIzUU0895dXtDJjGvcq9GgwIIyGsf//+On78uI4fP+7pHfn444/1zTffaMCAAX5732uvvVarVq3S119/3eKR/UOGDFFlZaU6deqkxMTEdqulva7buXNnOZ1Or33bt29XRkaGZsyY4dl34d/ggEDHvYpAwQDWEDZmzBilpqbqtttuU0lJiXbt2qWpU6dq9OjRGjZsmN/ed/LkyYqLi5PNZtP27dt15MgRbdiwQQ6Ho8la09PTZbPZtHHjRpWVlWnHjh16+OGHtXv37lbX0l7XTUxM1M6dO1VWVqYvv/xSLpdLffv21e7du/X222/r008/1Z/+9Ce9//77ra4VuNi4VxEoCCMhzGKx6PXXX9fll1+u66+/XmPGjNFPfvITrV+/3q/v27lzZ23cuFExMTH69a9/rdTUVD3xxBOyWq1N1vrGG2/o+uuv17Rp09SvXz/dcsstOnbsmGJjY1tdS3td98EHH5TVatWAAQPUs2dPlZeX6+6771ZOTo4mTZqkESNG6KuvvvL6mxcQ6LhXESgsbrfbbboIAADQcdEzAgAAjCKMAAAAowgjAADAKMIIAAAwijACAACMIowAAACjCCMAAMAowggAADCKMAIAAIwijAAAAKMIIwAAwCjCCAAAMOr/ASXbw3ES3Qk1AAAAAElFTkSuQmCC",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "\n",
+    "nonincremental = [0.144, 0.268, 0.175, 0.292, 0.161, 0.147, 0.238, 0.159, 0.26, 0.161, 0.511, 0.263, 0.149, 0.141, 0.427, 0.06, 0.055, 0.056, 0.145, 0.142, 0.137, 0.302, 0.146, 0.35, 0.256, 0.162, 0.161, 0.33, 0.176, 0.4, 0.294, 0.176, 0.308, 0.206, 0.349, 0.3, 0.326, 0.216, 0.236, 0.852, 0.057, 0.145, 0.386, 0.157, 0.154, 0.351, 0.321, 0.161, 0.158, 0.271, 0.163, 0.619, 0.052, 0.055, 0.164, 0.147, 0.145, 0.214, 0.179, 0.147, 0.398, 0.351, 0.162, 0.158, 0.293, 0.292, 0.169, 0.202, 0.427, 0.205, 0.335, 0.197, 0.211, 0.217, 0.23, 0.225, 0.495, 0.26, 0.346, 0.243, 0.271, 0.259, 0.266, 0.256, 0.288, 0.492, 0.303, 0.294, 0.322, 0.317, 0.299, 1.524, 0.059, 0.056, 0.147, 0.219, 0.161, 0.334, 0.159, 0.175, 0.375, 0.16, 0.177, 0.337, 0.273, 0.184, 0.183, 0.778, 0.228, 0.305, 0.36, 0.159, 0.157, 0.469, 0.057, 0.055, 0.059, 0.059, 0.056, 0.059, 0.063, 0.056, 0.059, 0.056, 0.154, 0.145, 0.208, 0.15, 0.428, 0.281, 0.146, 0.151, 0.324, 0.154, 0.208, 0.141, 0.163, 0.359, 0.282, 0.271, 0.142, 0.529, 0.164, 0.224, 0.139, 0.139, 0.159, 0.344, 0.242, 0.26, 0.155, 0.498, 0.053, 0.152, 0.214, 0.15, 0.148, 0.147, 0.356, 0.252, 0.149, 0.43, 0.057, 0.059, 0.154, 0.212, 0.17, 0.14, 0.144, 0.362, 0.164, 0.216, 0.21, 0.166, 0.155, 0.281, 0.155, 0.314, 0.157, 0.544, 0.253, 0.343, 0.272, 0.186, 0.19, 0.327, 0.212, 0.216, 0.374, 0.241, 0.328, 0.463, 0.424, 0.318, 0.244, 0.245, 0.251, 0.256, 0.546, 0.287, 0.269, 0.316, 0.416, 0.303, 0.407, 0.305, 0.31, 0.311, 0.319, 0.326, 0.343, 0.344, 0.447, 0.685, 0.385, 0.359, 0.369, 0.654, 0.404, 2.228, 0.115, 0.166, 0.216, 0.141, 0.138, 0.268, 0.156, 0.345, 0.314, 0.183, 0.185, 0.193, 0.297, 0.396, 0.203, 0.179, 0.289, 0.187, 0.322, 0.203, 0.204, 0.46, 0.256, 0.244, 0.759]\n",
+    "# incremental = [0.139, 0.262, 0.16, 0.347, 0.177, 0.155, 0.16, 0.24, 0.201, 0.165, 0.489, 0.198, 0.227, 0.153, 0.156, 0.407, 0.056, 0.058, 0.143, 0.143, 0.219, 0.155, 0.328, 0.224, 0.335, 0.163, 0.247, 0.277, 0.197, 0.348, 0.337, 0.193, 0.195, 0.209, 0.293, 0.336, 0.226, 0.224, 0.206, 0.841, 0.066, 0.165, 0.261, 0.213, 0.141, 0.33, 0.186, 0.152, 0.159, 0.334, 0.175, 0.16, 0.583, 0.055, 0.155, 0.206, 0.141, 0.34, 0.152, 0.144, 0.214, 0.274, 0.307, 0.162, 0.235, 0.277, 0.172, 0.206, 0.189, 0.337, 0.46, 0.223, 0.209, 0.209, 0.214, 0.216, 0.371, 0.224, 0.375, 0.338, 0.266, 0.244, 0.269, 0.274, 0.273, 0.271, 0.283, 0.283, 0.305, 0.434, 0.33, 1.08, 0.065, 0.059, 0.134, 0.15, 0.145, 0.206, 0.354, 0.165, 0.255, 0.159, 0.286, 0.275, 0.16, 0.175, 0.181, 0.533, 0.201, 0.18, 0.185, 0.373, 0.16, 0.162, 0.455, 0.06, 0.057, 0.053, 0.057, 0.061, 0.063, 0.047, 0.059, 0.056, 0.057, 0.158, 0.154, 0.141, 0.383, 0.151, 0.142, 0.141, 0.359, 0.143, 0.225, 0.155, 0.151, 0.268, 0.334, 0.152, 0.206, 0.161, 0.401, 0.235, 0.275, 0.187, 0.151, 0.152, 0.287, 0.244, 0.272, 0.159, 0.559, 0.185, 0.156, 0.152, 0.146, 0.143, 0.263, 0.161, 0.316, 0.163, 0.438, 0.058, 0.167, 0.241, 0.161, 0.162, 0.151, 0.371, 0.062, 0.156, 0.215, 0.149, 0.145, 0.293, 0.165, 0.328, 0.36, 0.238, 0.387, 0.189, 0.437, 0.207, 0.228, 0.233, 0.365, 0.358, 0.381, 0.226, 0.225, 0.219, 0.448, 0.402, 0.246, 0.322, 0.289, 0.272, 0.394, 0.291, 0.365, 0.298, 0.409, 0.319, 0.323, 0.431, 0.339, 0.326, 0.311, 0.319, 0.325, 0.333, 0.367, 0.364, 0.594, 0.382, 0.595, 0.411, 0.414, 0.419, 1.668, 0.153, 0.23, 0.263, 0.147, 0.281, 0.156, 0.172, 0.354, 0.166, 0.199, 0.244, 0.174, 0.182, 0.294, 0.25, 0.184, 0.182, 0.357, 0.329, 0.226, 0.417, 0.239, 0.214, 0.579]\n",
+    "# incremental = [0.27, 0.147, 0.285, 0.146, 0.232, 0.168, 0.226, 0.267, 0.183, 0.151, 0.481, 0.174, 0.218, 0.142, 0.385, 0.061, 0.056, 0.155, 0.162, 0.219, 0.152, 0.279, 0.217, 0.425, 0.235, 0.162, 0.161, 0.3, 0.176, 0.174, 0.349, 0.305, 0.212, 0.221, 0.364, 0.223, 0.408, 0.467, 0.243, 0.707, 0.18, 0.143, 0.212, 0.181, 0.322, 0.265, 0.151, 0.157, 0.22, 0.291, 0.166, 0.55, 0.052, 0.154, 0.208, 0.147, 0.369, 0.163, 0.238, 0.151, 0.325, 0.289, 0.17, 0.234, 0.178, 0.269, 0.196, 0.216, 0.392, 0.358, 0.21, 0.22, 0.216, 0.302, 0.223, 0.256, 0.533, 0.273, 0.322, 0.25, 0.248, 0.314, 0.3, 0.284, 0.439, 0.288, 0.299, 0.298, 0.299, 0.304, 1.138, 0.18, 0.056, 0.054, 0.151, 0.135, 0.219, 0.281, 0.254, 0.168, 0.155, 0.301, 0.301, 0.276, 0.173, 0.161, 0.523, 0.242, 0.217, 0.142, 0.261, 0.147, 0.155, 0.47, 0.067, 0.063, 0.059, 0.223, 0.06, 0.057, 0.058, 0.062, 0.06, 0.058, 0.157, 0.148, 0.147, 0.484, 0.188, 0.145, 0.147, 0.344, 0.054, 0.14, 0.217, 0.143, 0.298, 0.151, 0.226, 0.155, 0.157, 0.43, 0.217, 0.214, 0.152, 0.153, 0.233, 0.285, 0.224, 0.329, 0.187, 0.436, 0.287, 0.143, 0.263, 0.171, 0.172, 0.36, 0.269, 0.676, 0.22, 0.411, 0.056, 0.161, 0.136, 0.255, 0.152, 0.156, 0.147, 0.402, 0.181, 0.208, 0.156, 0.149, 0.3, 0.158, 0.263, 0.366, 0.161, 0.41, 0.42, 0.459, 0.18, 0.177, 0.188, 0.305, 0.228, 0.354, 0.234, 0.248, 0.222, 0.437, 0.399, 0.294, 0.375, 0.266, 0.257, 0.478, 0.315, 0.388, 0.301, 0.362, 0.315, 0.315, 0.439, 0.311, 0.325, 0.344, 0.375, 0.367, 0.349, 0.333, 0.366, 0.556, 0.393, 0.599, 0.393, 0.766, 0.403, 1.396, 0.423, 0.232, 0.258, 0.145, 0.283, 0.156, 0.155, 0.27, 0.278, 0.172, 0.16, 0.167, 0.238, 0.322, 0.188, 0.181, 0.198, 0.275, 0.31, 0.208, 0.468, 0.223, 0.261, 0.62, ]\n",
+    "\n",
+    "# incremental orig [vad_update + states]\n",
+    "# incremental = [0.123, 0.194, 0.206, 0.163, 0.145, 0.15, 0.142, 0.226, 0.136, 0.139, 0.131, 0.274, 0.139, 0.172, 0.125, 0.325, 0.053, 0.056, 0.054, 0.123, 0.124, 0.126, 0.255, 0.129, 0.258, 0.19, 0.139, 0.141, 0.138, 0.155, 0.263, 0.159, 0.166, 0.163, 0.174, 0.196, 0.198, 0.193, 0.208, 0.192, 0.56, 0.068, 0.33, 0.163, 0.124, 0.195, 0.136, 0.125, 0.124, 0.13, 0.146, 0.143, 0.423, 0.063, 0.125, 0.139, 0.12, 0.157, 0.127, 0.2, 0.138, 0.316, 0.132, 0.202, 0.203, 0.218, 0.142, 0.153, 0.149, 0.137, 0.147, 0.159, 0.152, 0.201, 0.168, 0.209, 0.171, 0.201, 0.223, 0.248, 0.131, 0.199, 0.137, 0.135, 0.176, 0.201, 0.135, 0.193, 0.131, 0.177, 0.145, 0.145, 0.3, 0.056, 0.058, 0.176, 0.162, 0.121, 0.245, 0.144, 0.125, 0.203, 0.143, 0.144, 0.153, 0.195, 0.146, 0.35, 0.124, 0.154, 0.124, 0.265, 0.131, 0.284, 0.236, 0.068, 0.061, 0.054, 0.056, 0.067, 0.056, 0.05, 0.049, 0.061, 0.054, 0.136, 0.172, 0.119, 0.312, 0.123, 0.122, 0.123, 0.26, 0.059, 0.123, 0.149, 0.118, 0.119, 0.184, 0.192, 0.133, 0.133, 0.207, 0.118, 0.165, 0.12, 0.113, 0.12, 0.226, 0.136, 0.17, 0.131, 0.282, 0.054, 0.128, 0.174, 0.133, 0.156, 0.229, 0.218, 0.15, 0.128, 0.226, 0.054, 0.117, 0.126, 0.182, 0.128, 0.127, 0.146, 0.209, 0.138, 0.16, 0.129, 0.123, 0.21, 0.132, 0.144, 0.128, 0.15, 0.377, 0.114, 0.123, 0.205, 0.123, 0.125, 0.266, 0.175, 0.298, 0.131, 0.183, 0.143, 0.157, 0.168, 0.174, 0.17, 0.168, 0.187, 0.186, 0.182, 0.186, 0.211, 0.205, 0.218, 0.226, 0.223, 0.211, 0.235, 0.227, 0.23, 0.255, 0.265, 0.274, 0.271, 0.298, 0.265, 0.614, 0.297, 0.32, 0.302, 1.153, 0.09, 0.16, 0.123, 0.17, 0.242, 0.13, 0.135, 0.21, 0.125, 0.172, 0.179, 0.139, 0.15, 0.153, 0.157, 0.145, 0.15, 0.166, 0.17, 0.167, 0.365, 0.181, 0.185, 0.217, 0.442]\n",
+    "\n",
+    "# incremental 1\n",
+    "# incremental = [0.166, 0.141, 0.319, 0.247, 0.272, 0.15, 0.149, 0.188, 0.143, 0.146, 0.283, 0.141, 0.166, 0.122, 0.246, 0.054, 0.055, 0.13, 0.122, 0.172, 0.131, 0.224, 0.173, 0.19, 0.143, 0.217, 0.15, 0.155, 0.152, 0.169, 0.161, 0.176, 0.175, 0.305, 0.253, 0.131, 0.18, 0.136, 0.137, 0.318, 0.064, 0.123, 0.165, 0.124, 0.19, 0.199, 0.132, 0.141, 0.133, 0.2, 0.14, 0.327, 0.054, 0.136, 0.17, 0.125, 0.228, 0.146, 0.13, 0.135, 0.178, 0.21, 0.227, 0.128, 0.125, 0.167, 0.136, 0.177, 0.265, 0.196, 0.158, 0.144, 0.149, 0.144, 0.207, 0.162, 0.17, 0.167, 0.237, 0.196, 0.18, 0.185, 0.192, 0.198, 0.187, 0.205, 0.205, 0.226, 0.226, 0.213, 0.609, 0.062, 0.06, 0.053, 0.126, 0.122, 0.176, 0.191, 0.13, 0.147, 0.178, 0.136, 0.206, 0.144, 0.143, 0.142, 0.386, 0.128, 0.166, 0.136, 0.228, 0.131, 0.132, 0.22, 0.056, 0.063, 0.07, 0.07, 0.064, 0.079, 0.065, 0.059, 0.053, 0.062, 0.128, 0.129, 0.125, 0.241, 0.153, 0.135, 0.124, 0.227, 0.055, 0.132, 0.173, 0.126, 0.191, 0.13, 0.175, 0.162, 0.135, 0.185, 0.175, 0.169, 0.132, 0.128, 0.134, 0.214, 0.14, 0.179, 0.134, 0.277, 0.049, 0.137, 0.164, 0.124, 0.128, 0.229, 0.137, 0.189, 0.124, 0.223, 0.052, 0.123, 0.168, 0.163, 0.127, 0.126, 0.129, 0.217, 0.123, 0.168, 0.123, 0.132, 0.216, 0.149, 0.193, 0.268, 0.139, 0.138, 0.146, 0.185, 0.169, 0.155, 0.171, 0.186, 0.277, 0.199, 0.184, 0.204, 0.241, 0.348, 0.244, 0.226, 0.223, 0.219, 0.24, 0.253, 0.312, 0.248, 0.29, 0.285, 0.273, 0.27, 0.271, 0.273, 0.282, 0.306, 0.315, 0.321, 0.302, 0.327, 0.296, 0.311, 0.324, 0.333, 0.354, 0.564, 0.346, 0.798, 0.281, 0.431, 0.424, 0.3, 0.359, 0.393, 0.415, 0.203, 0.137, 0.138, 0.137, 0.146, 0.139, 0.21, 0.165, 0.165, 0.164, 0.183, 0.215, 0.191, 0.185, 0.268, 0.183, 0.281]\n",
+    "# incremental = [0.14, 0.133, 0.207, 0.146, 0.179, 0.137, 0.132, 0.139, 0.203, 0.194, 0.156, 0.282, 0.172, 0.118, 0.13, 0.386, 0.06, 0.062, 0.062, 0.121, 0.119, 0.125, 0.238, 0.128, 0.166, 0.225, 0.165, 0.166, 0.15, 0.166, 0.149, 0.223, 0.158, 0.179, 0.186, 0.182, 0.187, 0.186, 0.198, 0.197, 0.399, 0.066, 0.132, 0.196, 0.175, 0.123, 0.12, 0.167, 0.127, 0.174, 0.191, 0.127, 0.127, 0.301, 0.056, 0.128, 0.18, 0.179, 0.177, 0.135, 0.157, 0.228, 0.215, 0.215, 0.132, 0.128, 0.173, 0.172, 0.129, 0.137, 0.236, 0.178, 0.144, 0.152, 0.147, 0.158, 0.179, 0.284, 0.177, 0.181, 0.192, 0.181, 0.241, 0.233, 0.231, 0.218, 0.223, 0.221, 0.213, 0.244, 0.244, 0.232, 0.772, 0.058, 0.065, 0.134, 0.175, 0.125, 0.205, 0.132, 0.132, 0.199, 0.197, 0.12, 0.128, 0.168, 0.132, 0.131, 0.288, 0.174, 0.124, 0.195, 0.134, 0.134, 0.265, 0.048, 0.059, 0.073, 0.056, 0.061, 0.059, 0.057, 0.055, 0.057, 0.061, 0.132, 0.138, 0.135, 0.249, 0.132, 0.162, 0.128, 0.121, 0.282, 0.135, 0.161, 0.131, 0.131, 0.189, 0.175, 0.132, 0.135, 0.138, 0.24, 0.169, 0.131, 0.132, 0.143, 0.228, 0.136, 0.141, 0.141, 0.296, 0.052, 0.124, 0.164, 0.128, 0.123, 0.214, 0.199, 0.143, 0.139, 0.265, 0.058, 0.056, 0.131, 0.161, 0.13, 0.127, 0.123, 0.228, 0.135, 0.236, 0.407, 0.25, 0.409, 0.504, 0.336, 0.596, 0.134, 0.143, 0.147, 0.275, 0.315, 0.159, 0.172, 0.182, 0.182, 0.187, 0.191, 0.187, 0.2, 0.242, 0.276, 0.142, 0.123, 0.127, 0.166, 0.175, 0.176, 0.188, 0.211, 0.285, 0.236, 0.297, 0.147, 0.159, 0.154, 0.171, 0.164, 0.228, 0.183, 0.299, 0.395, 0.126, 0.122, 0.171, 0.17, 0.133, 0.152, 0.386, 0.053, 0.134, 0.17, 0.136, 0.131, 0.213, 0.126, 0.195, 0.133, 0.128, 0.13, 0.177, 0.136, 0.14, 0.19, 0.194, 0.162, 0.166, 0.169, 0.227, 0.185, 0.28, 0.188, 0.188, 0.259]\n",
+    "# incremental = [1.174, 0.16, 0.2, 0.223, 0.19, 0.147, 0.153, 0.193, 0.137, 0.177, 0.145, 0.258, 0.178, 0.124, 0.124, 0.241, 0.057, 0.056, 0.058, 0.153, 0.147, 0.122, 0.226, 0.126, 0.219, 0.131, 0.139, 0.179, 0.137, 0.142, 0.148, 0.222, 0.151, 0.158, 0.173, 0.186, 0.187, 0.218, 0.176, 0.201, 0.35, 0.051, 0.247, 0.205, 0.134, 0.155, 0.192, 0.132, 0.132, 0.184, 0.14, 0.188, 0.339, 0.049, 0.216, 0.133, 0.118, 0.145, 0.165, 0.126, 0.137, 0.296, 0.205, 0.13, 0.128, 0.236, 0.219, 0.162, 0.135, 0.139, 0.244, 0.191, 0.162, 0.153, 0.156, 0.162, 0.168, 0.212, 0.189, 0.208, 0.177, 0.192, 0.188, 0.199, 0.204, 0.195, 0.224, 0.25, 0.226, 0.256, 0.294, 0.34, 0.207, 0.064, 0.063, 0.125, 0.176, 0.13, 0.208, 0.142, 0.131, 0.189, 0.186, 0.124, 0.122, 0.182, 0.122, 0.29, 0.135, 0.171, 0.13, 0.205, 0.121, 0.132, 0.278, 0.06, 0.056, 0.06, 0.056, 0.053, 0.052, 0.053, 0.055, 0.053, 0.062, 0.13, 0.128, 0.126, 0.239, 0.126, 0.129, 0.136, 0.126, 0.238, 0.133, 0.166, 0.127, 0.144, 0.199, 0.183, 0.14, 0.137, 0.28, 0.132, 0.174, 0.138, 0.123, 0.132, 0.217, 0.172, 0.137, 0.138, 0.295, 0.064, 0.118, 0.166, 0.127, 0.127, 0.175, 0.228, 0.138, 0.125, 0.244, 0.054, 0.059, 0.137, 0.163, 0.121, 0.127, 0.122, 0.234, 0.124, 0.155, 0.183, 0.124, 0.188, 0.166, 0.137, 0.19, 0.306, 0.145, 0.157, 0.301, 0.162, 0.157, 0.163, 0.176, 0.175, 0.177, 0.19, 0.193, 0.203, 0.201, 0.206, 0.214, 0.216, 0.217, 0.237, 0.228, 0.24, 0.234, 0.248, 0.258, 0.268, 0.264, 0.402, 0.29, 0.262, 0.278, 0.282, 0.291, 0.289, 0.316, 0.303, 0.31, 0.306, 0.3, 0.325, 0.327, 0.329, 0.649, 0.06, 0.143, 0.171, 0.127, 0.125, 0.184, 0.125, 0.283, 0.133, 0.131, 0.204, 0.135, 0.142, 0.201, 0.143, 0.16, 0.152, 0.167, 0.219, 0.169, 0.18, 0.287, 0.2, 0.189, 0.277]\n",
+    "# synthesize speech\n",
+    "# incremental = [0.177, 0.133, 0.192, 0.133, 0.201, 0.133, 0.141, 0.137, 0.175, 0.22, 0.153, 0.331, 0.184, 0.18, 0.147, 0.138, 0.393, 0.067, 0.156, 0.16, 0.136, 0.122, 0.224, 0.205, 0.182, 0.135, 0.271, 0.147, 0.146, 0.178, 0.158, 0.151, 0.164, 0.173, 0.162, 0.186, 0.346, 0.199, 0.12, 0.131, 0.32, 0.064, 0.063, 0.142, 0.165, 0.129, 0.213, 0.124, 0.133, 0.138, 0.284, 0.14, 0.145, 0.374, 0.054, 0.137, 0.164, 0.118, 0.199, 0.133, 0.131, 0.28, 0.163, 0.254, 0.19, 0.119, 0.162, 0.128, 0.132, 0.269, 0.158, 0.235, 0.141, 0.141, 0.242, 0.178, 0.178, 0.151, 0.19, 0.178, 0.236, 0.176, 0.24, 0.206, 0.201, 0.207, 0.213, 0.218, 0.22, 0.257, 0.308, 0.304, 0.076, 0.056, 0.063, 0.069, 0.165, 0.198, 0.151, 0.278, 0.135, 0.21, 0.147, 0.14, 0.229, 0.151, 0.159, 0.151, 0.417, 0.167, 0.124, 0.172, 0.307, 0.144, 0.244, 0.062, 0.054, 0.063, 0.053, 0.06, 0.049, 0.053, 0.075, 0.061, 0.061, 0.054, 0.131, 0.124, 0.124, 0.284, 0.175, 0.138, 0.128, 0.253, 0.13, 0.161, 0.119, 0.151, 0.271, 0.145, 0.217, 0.14, 0.139, 0.269, 0.199, 0.176, 0.135, 0.13, 0.14, 0.257, 0.163, 0.224, 0.144, 0.327, 0.143, 0.183, 0.151, 0.127, 0.136, 0.287, 0.143, 0.13, 0.131, 0.296, 0.052, 0.134, 0.121, 0.169, 0.125, 0.125, 0.272, 0.157, 0.206, 0.199, 0.131, 0.128, 0.176, 0.241, 0.14, 0.244, 0.145, 0.147, 0.352, 0.268, 0.16, 0.163, 0.171, 0.187, 0.202, 0.193, 0.191, 0.23, 0.227, 0.212, 0.285, 0.236, 0.234, 0.217, 0.213, 0.236, 0.238, 0.33, 0.28, 0.268, 0.286, 0.283, 0.286, 0.364, 0.315, 0.29, 0.297, 0.297, 0.297, 0.322, 0.378, 0.311, 0.341, 0.344, 0.334, 0.34, 0.698, 0.069, 0.132, 0.167, 0.186, 0.13, 0.222, 0.136, 0.204, 0.208, 0.131, 0.134, 0.152, 0.145, 0.15, 0.144, 0.26, 0.166, 0.156, 0.162, 0.18, 0.297, 0.185, 0.185, 0.201, 0.328]\n",
+    "# incremental = [0.146, 0.133, 0.186, 0.137, 0.199, 0.145, 0.128, 0.163, 0.218, 0.217, 0.159, 0.315, 0.175, 0.176, 0.125, 0.141, 0.277, 0.064, 0.061, 0.123, 0.116, 0.166, 0.217, 0.199, 0.171, 0.211, 0.134, 0.142, 0.148, 0.276, 0.157, 0.156, 0.157, 0.542, 0.19, 0.186, 0.186, 0.187, 0.186, 0.174, 0.432, 0.051, 0.052, 0.127, 0.162, 0.132, 0.125, 0.238, 0.14, 0.134, 0.172, 0.132, 0.135, 0.368, 0.054, 0.246, 0.163, 0.115, 0.166, 0.131, 0.13, 0.28, 0.217, 0.122, 0.126, 0.18, 0.125, 0.134, 0.131, 0.221, 0.134, 0.226, 0.151, 0.144, 0.163, 0.16, 0.169, 0.27, 0.174, 0.184, 0.19, 0.178, 0.187, 0.206, 0.203, 0.202, 0.206, 0.217, 0.224, 0.241, 0.251, 0.233, 0.665, 0.054, 0.052, 0.061, 0.165, 0.173, 0.124, 0.224, 0.135, 0.207, 0.143, 0.197, 0.146, 0.153, 0.147, 0.137, 0.419, 0.166, 0.128, 0.167, 0.237, 0.141, 0.237, 0.056, 0.054, 0.051, 0.054, 0.063, 0.059, 0.059, 0.049, 0.062, 0.056, 0.057, 0.126, 0.116, 0.432, 0.268, 0.121, 0.119, 0.174, 0.266, 0.131, 0.17, 0.127, 0.141, 0.274, 0.161, 0.213, 0.143, 0.158, 0.273, 0.171, 0.176, 0.144, 0.138, 0.217, 0.182, 0.143, 0.203, 0.141, 0.298, 0.129, 0.174, 0.125, 0.124, 0.125, 0.28, 0.135, 0.143, 0.143, 0.3, 0.055, 0.166, 0.165, 0.161, 0.124, 0.129, 0.255, 0.133, 0.178, 0.255, 0.157, 0.421, 0.132, 0.221, 0.245, 0.148, 0.138, 0.231, 0.263, 0.256, 0.449, 0.174, 0.19, 0.189, 0.188, 0.19, 0.194, 0.207, 0.206, 0.206, 0.278, 0.229, 0.221, 0.207, 0.211, 0.238, 0.24, 0.302, 0.264, 0.297, 0.272, 0.264, 0.264, 0.267, 0.269, 0.277, 0.297, 0.299, 0.303, 0.304, 0.301, 0.311, 0.297, 0.342, 0.322, 0.332, 1.311, 0.091, 0.13, 0.15, 0.197, 0.128, 0.217, 0.127, 0.205, 0.207, 0.125, 0.132, 0.139, 0.141, 0.188, 0.184, 0.222, 0.155, 0.154, 0.167, 0.176, 0.298, 0.179, 0.196, 0.185, 0.306]\n",
+    "\n",
+    "\n",
+    "# incremental 2 [_train]\n",
+    "# incremental = [0.144, 0.217, 0.149, 0.264, 0.15, 0.241, 0.157, 0.152, 0.25, 0.158, 0.454, 0.233, 0.231, 0.152, 0.148, 0.357, 0.064, 0.06, 0.143, 0.142, 0.17, 0.26, 0.225, 0.237, 0.306, 0.157, 0.164, 0.25, 0.179, 0.294, 0.179, 0.178, 0.192, 0.186, 0.206, 0.198, 0.238, 0.216, 0.217, 0.342, 0.054, 0.056, 0.144, 0.215, 0.146, 0.181, 0.143, 0.15, 0.266, 0.149, 0.143, 0.144, 0.514, 0.049, 0.215, 0.142, 0.222, 0.223, 0.154, 0.164, 0.34, 0.154, 0.241, 0.201, 0.173, 0.297, 0.138, 0.225, 0.27, 0.266, 0.155, 0.154, 0.148, 0.233, 0.166, 0.166, 0.283, 0.178, 0.188, 0.193, 0.193, 0.205, 0.214, 0.203, 0.215, 0.293, 0.242, 0.245, 0.256, 0.259, 0.59, 0.06, 0.058, 0.06, 0.06, 0.151, 0.151, 0.227, 0.261, 0.161, 0.239, 0.164, 0.264, 0.257, 0.156, 0.233, 0.161, 0.506, 0.245, 0.151, 0.225, 0.327, 0.16, 0.337, 0.059, 0.056, 0.056, 0.057, 0.067, 0.058, 0.067, 0.053, 0.052, 0.06, 0.065, 0.148, 0.145, 0.139, 0.38, 0.16, 0.148, 0.149, 0.332, 0.148, 0.232, 0.164, 0.149, 0.301, 0.149, 0.156, 0.159, 0.161, 0.329, 0.225, 0.229, 0.155, 0.16, 0.159, 0.308, 0.151, 0.281, 0.165, 0.43, 0.189, 0.348, 0.138, 0.152, 0.147, 0.267, 0.203, 0.27, 0.228, 0.53, 0.052, 0.147, 0.226, 0.147, 0.15, 0.147, 0.338, 0.145, 0.235, 0.279, 0.154, 0.155, 0.154, 0.351, 0.165, 0.155, 0.162, 0.177, 0.173, 0.248, 0.21, 0.191, 0.206, 0.198, 0.206, 0.211, 0.22, 0.291, 0.312, 0.221, 0.684, 0.236, 0.24, 0.258, 0.256, 0.263, 0.27, 0.277, 0.295, 0.301, 0.345, 0.295, 0.3, 0.309, 0.31, 0.317, 0.319, 0.334, 0.341, 0.372, 0.344, 0.364, 0.369, 0.38, 0.376, 0.398, 0.617, 0.064, 0.141, 0.212, 0.147, 0.153, 0.354, 0.16, 0.221, 0.156, 0.153, 0.163, 0.265, 0.144, 0.236, 0.145, 0.146, 0.175, 0.153, 0.221, 0.325, 0.305, 0.159, 0.179, 0.173, 0.375]\n",
+    "\n",
+    "# nonincremental infer\n",
+    "# incremental = [0.122, 0.196, 0.123, 0.314, 0.136, 0.127, 0.133, 0.204, 0.193, 0.135, 0.306, 0.137, 0.126, 0.174, 0.121, 0.303, 0.066, 0.055, 0.131, 0.135, 0.166, 0.13, 0.245, 0.237, 0.191, 0.135, 0.206, 0.211, 0.149, 0.311, 0.257, 0.233, 0.168, 0.171, 0.178, 0.256, 0.186, 0.196, 0.181, 0.585, 0.06, 0.059, 0.139, 0.169, 0.123, 0.241, 0.202, 0.128, 0.141, 0.226, 0.152, 0.128, 0.392, 0.054, 0.246, 0.166, 0.115, 0.248, 0.139, 0.137, 0.185, 0.243, 0.224, 0.142, 0.185, 0.239, 0.156, 0.17, 0.255, 0.192, 0.243, 0.181, 0.19, 0.223, 0.186, 0.195, 0.304, 0.197, 0.23, 0.316, 0.214, 0.248, 0.236, 0.239, 0.236, 0.328, 0.286, 0.281, 0.262, 0.342, 0.289, 0.796, 0.287, 0.062, 0.052, 0.13, 0.119, 0.165, 0.227, 0.134, 0.24, 0.224, 0.133, 0.15, 0.28, 0.139, 0.143, 0.36, 0.121, 0.462, 0.125, 0.309, 0.145, 0.251, 0.256, 0.06, 0.058, 0.058, 0.056, 0.055, 0.048, 0.047, 0.066, 0.064, 0.069, 0.138, 0.127, 0.134, 0.282, 0.119, 0.123, 0.116, 0.268, 0.128, 0.165, 0.173, 0.141, 0.198, 0.209, 0.129, 0.13, 0.133, 0.328, 0.174, 0.173, 0.13, 0.132, 0.142, 0.252, 0.201, 0.137, 0.136, 0.318, 0.054, 0.123, 0.128, 0.123, 0.118, 0.24, 0.128, 0.202, 0.121, 0.278, 0.058, 0.13, 0.18, 0.124, 0.138, 0.126, 0.271, 0.059, 0.139, 0.164, 0.125, 0.187, 0.261, 0.132, 0.307, 0.132, 0.224, 0.398, 0.303, 0.307, 0.154, 0.153, 0.161, 0.204, 0.196, 0.285, 0.19, 0.203, 0.189, 0.196, 0.366, 0.22, 0.31, 0.258, 0.287, 0.322, 0.236, 0.241, 0.537, 0.311, 0.261, 0.261, 0.402, 0.281, 0.277, 0.28, 0.303, 0.345, 0.307, 0.297, 0.319, 0.487, 0.319, 0.339, 0.33, 0.339, 0.341, 1.037, 0.126, 0.237, 0.21, 0.124, 0.269, 0.161, 0.132, 0.223, 0.143, 0.137, 0.181, 0.143, 0.221, 0.179, 0.263, 0.162, 0.169, 0.254, 0.187, 0.184, 0.321, 0.203, 0.203, 0.444]\n",
+    "# incremental = [0.126, 0.202, 0.239, 0.218, 0.152, 0.153, 0.228, 0.137, 0.139, 0.145, 0.349, 0.201, 0.138, 0.165, 0.326, 0.057, 0.056, 0.059, 0.143, 0.131, 0.134, 0.291, 0.15, 0.277, 0.15, 0.197, 0.249, 0.167, 0.26, 0.243, 0.246, 0.156, 0.242, 0.188, 0.26, 0.241, 0.376, 0.209, 0.204, 0.437, 0.054, 0.241, 0.238, 0.139, 0.223, 0.212, 0.193, 0.145, 0.234, 0.212, 0.141, 0.408, 0.045, 0.149, 0.149, 0.134, 0.131, 0.183, 0.145, 0.137, 0.382, 0.305, 0.139, 0.142, 0.249, 0.157, 0.147, 0.265, 0.173, 0.243, 0.192, 0.182, 0.184, 0.189, 0.19, 0.221, 0.303, 0.208, 0.329, 0.226, 0.234, 0.239, 0.247, 0.246, 0.354, 0.39, 0.282, 0.274, 0.274, 0.278, 0.275, 0.823, 0.05, 0.067, 0.141, 0.174, 0.127, 0.274, 0.139, 0.221, 0.143, 0.145, 0.154, 0.298, 0.155, 0.15, 0.407, 0.155, 0.18, 0.132, 0.233, 0.212, 0.143, 0.296, 0.064, 0.089, 0.057, 0.06, 0.054, 0.05, 0.055, 0.056, 0.065, 0.064, 0.169, 0.129, 0.127, 0.311, 0.133, 0.131, 0.134, 0.142, 0.267, 0.132, 0.169, 0.133, 0.13, 0.29, 0.216, 0.147, 0.137, 0.299, 0.14, 0.207, 0.138, 0.141, 0.13, 0.266, 0.286, 0.136, 0.144, 0.339, 0.061, 0.405, 0.182, 0.145, 0.145, 0.182, 0.297, 0.221, 0.148, 0.298, 0.084, 0.07, 0.144, 0.182, 0.14, 0.134, 0.135, 0.288, 0.144, 0.194, 0.216, 0.143, 0.133, 0.135, 0.257, 0.246, 0.25, 0.361, 0.149, 0.422, 0.265, 0.167, 0.171, 0.19, 0.194, 0.182, 0.299, 0.196, 0.342, 0.21, 0.32, 0.222, 0.319, 0.232, 0.235, 0.252, 0.364, 0.249, 0.274, 0.294, 0.315, 0.295, 0.312, 0.285, 0.288, 0.303, 0.285, 0.353, 0.36, 0.314, 0.3, 0.351, 0.317, 0.312, 0.337, 0.561, 0.38, 1.073, 0.057, 0.145, 0.253, 0.136, 0.14, 0.205, 0.156, 0.29, 0.194, 0.136, 0.139, 0.143, 0.22, 0.266, 0.164, 0.159, 0.165, 0.187, 0.26, 0.184, 0.311, 0.236, 0.193, 0.186, 0.464]\n",
+    "\n",
+    "# incremental infer\n",
+    "# incremental = [0.117, 0.179, 0.229, 0.222, 0.143, 0.13, 0.137, 0.192, 0.215, 0.15, 0.257, 0.137, 0.176, 0.139, 0.199, 0.295, 0.066, 0.057, 0.162, 0.129, 0.176, 0.135, 0.266, 0.145, 0.265, 0.184, 0.211, 0.151, 0.221, 0.269, 0.244, 0.237, 0.176, 0.177, 0.181, 0.234, 0.193, 0.185, 0.203, 0.512, 0.057, 0.102, 0.127, 0.19, 0.126, 0.256, 0.227, 0.131, 0.134, 0.134, 0.204, 0.144, 0.35, 0.051, 0.275, 0.166, 0.2, 0.169, 0.131, 0.139, 0.263, 0.228, 0.225, 0.151, 0.178, 0.219, 0.14, 0.176, 0.291, 0.25, 0.213, 0.19, 0.192, 0.199, 0.206, 0.196, 0.28, 0.21, 0.317, 0.265, 0.236, 0.222, 0.272, 0.251, 0.262, 0.361, 0.273, 0.265, 0.272, 0.277, 0.279, 0.732, 0.064, 0.059, 0.066, 0.14, 0.135, 0.248, 0.14, 0.137, 0.198, 0.149, 0.232, 0.145, 0.268, 0.151, 0.151, 0.373, 0.179, 0.176, 0.181, 0.25, 0.131, 0.235, 0.058, 0.058, 0.055, 0.054, 0.059, 0.064, 0.058, 0.061, 0.055, 0.066, 0.057, 0.125, 0.127, 0.134, 0.275, 0.128, 0.126, 0.146, 0.306, 0.129, 0.164, 0.131, 0.128, 0.228, 0.213, 0.133, 0.137, 0.142, 0.319, 0.169, 0.174, 0.134, 0.129, 0.131, 0.302, 0.245, 0.14, 0.144, 0.323, 0.128, 0.164, 0.125, 0.135, 0.132, 0.266, 0.139, 0.191, 0.14, 0.259, 0.053, 0.13, 0.161, 0.122, 0.138, 0.122, 0.263, 0.057, 0.127, 0.158, 0.125, 0.126, 0.191, 0.205, 0.138, 0.129, 0.253, 0.143, 0.151, 0.148, 0.167, 0.165, 0.16, 0.383, 0.174, 0.296, 0.187, 0.334, 0.218, 0.207, 0.364, 0.206, 0.204, 0.226, 0.218, 0.237, 0.235, 0.279, 0.357, 0.273, 0.269, 0.265, 0.274, 0.271, 0.285, 0.293, 0.292, 0.301, 0.308, 0.308, 0.305, 0.302, 0.305, 0.305, 0.336, 0.339, 0.347, 1.08, 0.134, 0.169, 0.19, 0.124, 0.212, 0.178, 0.186, 0.209, 0.134, 0.128, 0.178, 0.137, 0.209, 0.187, 0.225, 0.163, 0.15, 0.161, 0.248, 0.281, 0.221, 0.183, 0.193, 0.429]\n",
+    "# incremental = [0.126, 0.189, 0.13, 0.293, 0.133, 0.149, 0.137, 0.235, 0.154, 0.141, 0.326, 0.132, 0.121, 0.17, 0.132, 0.291, 0.058, 0.061, 0.132, 0.132, 0.168, 0.134, 0.249, 0.235, 0.201, 0.144, 0.21, 0.214, 0.145, 0.258, 0.222, 0.223, 0.176, 0.208, 0.227, 0.259, 0.231, 0.32, 0.215, 0.569, 0.057, 0.057, 0.133, 0.129, 0.16, 0.289, 0.203, 0.134, 0.143, 0.23, 0.185, 0.142, 0.364, 0.054, 0.271, 0.165, 0.128, 0.29, 0.196, 0.138, 0.145, 0.198, 0.213, 0.149, 0.189, 0.213, 0.153, 0.159, 0.251, 0.171, 0.281, 0.184, 0.184, 0.189, 0.203, 0.203, 0.285, 0.194, 0.219, 0.347, 0.227, 0.229, 0.246, 0.232, 0.241, 0.365, 0.291, 0.284, 0.275, 0.274, 0.272, 0.779, 0.073, 0.058, 0.055, 0.143, 0.133, 0.159, 0.24, 0.133, 0.133, 0.238, 0.14, 0.144, 0.281, 0.142, 0.145, 0.368, 0.131, 0.165, 0.124, 0.26, 0.128, 0.128, 0.258, 0.061, 0.06, 0.051, 0.063, 0.069, 0.06, 0.055, 0.055, 0.052, 0.065, 0.13, 0.124, 0.124, 0.283, 0.129, 0.127, 0.131, 0.26, 0.056, 0.126, 0.178, 0.142, 0.234, 0.215, 0.13, 0.127, 0.132, 0.31, 0.183, 0.176, 0.133, 0.158, 0.143, 0.251, 0.205, 0.139, 0.144, 0.318, 0.054, 0.128, 0.17, 0.127, 0.129, 0.254, 0.214, 0.198, 0.156, 0.323, 0.057, 0.128, 0.167, 0.168, 0.136, 0.129, 0.25, 0.068, 0.189, 0.177, 0.128, 0.137, 0.283, 0.141, 0.305, 0.14, 0.226, 0.378, 0.263, 0.449, 0.172, 0.158, 0.168, 0.178, 0.221, 0.326, 0.189, 0.188, 0.193, 0.205, 0.371, 0.227, 0.327, 0.216, 0.231, 0.341, 0.251, 0.252, 0.298, 0.261, 0.281, 0.295, 0.359, 0.283, 0.285, 0.294, 0.305, 0.317, 0.322, 0.322, 0.317, 0.471, 0.315, 0.327, 0.328, 0.343, 0.347, 0.962, 0.136, 0.276, 0.237, 0.134, 0.227, 0.137, 0.134, 0.225, 0.151, 0.144, 0.182, 0.165, 0.215, 0.163, 0.243, 0.161, 0.171, 0.177, 0.251, 0.187, 0.315, 0.224, 0.207, 0.426]\n",
+    "# incremental = [0.157, 0.136, 0.199, 0.132, 0.266, 0.164, 0.14, 0.148, 0.223, 0.24, 0.148, 0.299, 0.195, 0.184, 0.151, 0.14, 0.284, 0.068, 0.069, 0.181, 0.136, 0.129, 0.226, 0.248, 0.164, 0.358, 0.174, 0.196, 0.154, 0.244, 0.244, 0.224, 0.172, 0.256, 0.187, 0.22, 0.186, 0.219, 0.216, 0.2, 0.528, 0.063, 0.058, 0.139, 0.169, 0.128, 0.136, 0.278, 0.148, 0.132, 0.139, 0.155, 0.149, 0.382, 0.063, 0.144, 0.128, 0.177, 0.181, 0.132, 0.145, 0.317, 0.154, 0.234, 0.148, 0.281, 0.152, 0.164, 0.254, 0.178, 0.252, 0.187, 0.193, 0.184, 0.2, 0.199, 0.207, 0.304, 0.227, 0.313, 0.291, 0.24, 0.249, 0.245, 0.264, 0.333, 0.371, 0.272, 0.278, 0.28, 0.302, 0.756, 0.061, 0.068, 0.056, 0.059, 0.135, 0.163, 0.136, 0.215, 0.256, 0.135, 0.151, 0.227, 0.147, 0.281, 0.17, 0.159, 0.358, 0.166, 0.134, 0.197, 0.311, 0.167, 0.259, 0.061, 0.053, 0.072, 0.07, 0.078, 0.073, 0.078, 0.08, 0.076, 0.074, 0.071, 0.192, 0.126, 0.125, 0.281, 0.134, 0.126, 0.136, 0.25, 0.133, 0.174, 0.139, 0.129, 0.222, 0.216, 0.192, 0.144, 0.135, 0.248, 0.183, 0.176, 0.189, 0.144, 0.22, 0.184, 0.2, 0.135, 0.133, 0.315, 0.128, 0.298, 0.126, 0.126, 0.121, 0.34, 0.143, 0.208, 0.146, 0.276, 0.058, 0.132, 0.135, 0.176, 0.125, 0.127, 0.28, 0.131, 0.165, 0.191, 0.136, 0.128, 0.173, 0.234, 0.289, 0.168, 0.281, 0.415, 0.331, 0.362, 0.161, 0.162, 0.189, 0.194, 0.2, 0.292, 0.2, 0.199, 0.328, 0.363, 0.237, 0.352, 0.218, 0.227, 0.235, 0.291, 0.26, 0.321, 0.273, 0.306, 0.273, 0.274, 0.271, 0.369, 0.293, 0.286, 0.304, 0.308, 0.322, 0.299, 0.313, 0.444, 0.329, 0.35, 0.521, 0.365, 0.908, 0.054, 0.183, 0.167, 0.185, 0.129, 0.222, 0.132, 0.238, 0.139, 0.131, 0.135, 0.185, 0.138, 0.221, 0.187, 0.218, 0.153, 0.158, 0.176, 0.246, 0.297, 0.222, 0.202, 0.191, 0.463]\n",
+    "# incremental = [0.329, 0.188, 0.206, 0.228, 0.138, 0.133, 0.138, 0.205, 0.178, 0.143, 0.368, 0.167, 0.167, 0.128, 0.132, 0.283, 0.069, 0.064, 0.13, 0.121, 0.165, 0.174, 0.245, 0.177, 0.253, 0.171, 0.142, 0.208, 0.185, 0.268, 0.23, 0.238, 0.169, 0.167, 0.22, 0.179, 0.19, 0.186, 0.194, 0.512, 0.059, 0.083, 0.123, 0.166, 0.13, 0.247, 0.205, 0.135, 0.144, 0.229, 0.148, 0.139, 0.372, 0.057, 0.127, 0.122, 0.165, 0.208, 0.29, 0.133, 0.215, 0.235, 0.22, 0.146, 0.186, 0.215, 0.161, 0.176, 0.254, 0.168, 0.24, 0.174, 0.181, 0.216, 0.202, 0.201, 0.289, 0.202, 0.294, 0.207, 0.208, 0.214, 0.215, 0.234, 0.245, 0.346, 0.27, 0.271, 0.276, 0.334, 0.272, 0.704, 0.141, 0.052, 0.07, 0.157, 0.13, 0.161, 0.222, 0.137, 0.199, 0.15, 0.217, 0.14, 0.263, 0.149, 0.147, 0.379, 0.162, 0.185, 0.129, 0.237, 0.135, 0.127, 0.228, 0.06, 0.06, 0.064, 0.056, 0.06, 0.057, 0.054, 0.054, 0.061, 0.054, 0.122, 0.144, 0.15, 0.279, 0.13, 0.119, 0.124, 0.254, 0.138, 0.204, 0.181, 0.128, 0.204, 0.183, 0.185, 0.149, 0.148, 0.305, 0.213, 0.174, 0.14, 0.135, 0.143, 0.244, 0.204, 0.134, 0.134, 0.317, 0.06, 0.131, 0.123, 0.128, 0.161, 0.323, 0.128, 0.197, 0.146, 0.268, 0.057, 0.129, 0.169, 0.132, 0.126, 0.165, 0.258, 0.059, 0.174, 0.17, 0.127, 0.128, 0.202, 0.239, 0.243, 0.134, 0.138, 0.136, 0.295, 0.274, 0.155, 0.157, 0.204, 0.304, 0.313, 0.178, 0.205, 0.29, 0.193, 0.198, 0.393, 0.206, 0.307, 0.205, 0.213, 0.309, 0.217, 0.225, 0.266, 0.237, 0.285, 0.32, 0.267, 0.361, 0.28, 0.274, 0.272, 0.286, 0.287, 0.32, 0.308, 0.409, 0.442, 0.561, 0.325, 0.307, 0.315, 0.852, 0.412, 0.161, 0.181, 0.125, 0.216, 0.138, 0.197, 0.191, 0.131, 0.316, 0.173, 0.139, 0.202, 0.151, 0.229, 0.156, 0.157, 0.164, 0.284, 0.282, 0.214, 0.179, 0.185, 0.467]\n",
+    "# incremental = [0.189, 0.126, 0.17, 0.23, 0.136, 0.135, 0.164, 0.234, 0.152, 0.141, 0.145, 0.329, 0.138, 0.206, 0.138, 0.325, 0.06, 0.056, 0.061, 0.136, 0.128, 0.13, 0.257, 0.138, 0.245, 0.148, 0.214, 0.214, 0.144, 0.197, 0.278, 0.178, 0.173, 0.257, 0.18, 0.18, 0.225, 0.338, 0.21, 0.2, 0.39, 0.057, 0.143, 0.17, 0.139, 0.242, 0.204, 0.184, 0.136, 0.209, 0.14, 0.148, 0.404, 0.055, 0.162, 0.144, 0.129, 0.172, 0.152, 0.128, 0.133, 0.33, 0.233, 0.139, 0.138, 0.239, 0.149, 0.158, 0.245, 0.172, 0.175, 0.25, 0.177, 0.191, 0.188, 0.195, 0.188, 0.287, 0.202, 0.301, 0.22, 0.222, 0.229, 0.229, 0.24, 0.275, 0.34, 0.279, 0.267, 0.27, 0.276, 0.291, 0.748, 0.058, 0.059, 0.142, 0.172, 0.152, 0.25, 0.136, 0.214, 0.142, 0.157, 0.138, 0.266, 0.152, 0.148, 0.146, 0.368, 0.172, 0.13, 0.233, 0.14, 0.135, 0.28, 0.058, 0.059, 0.06, 0.097, 0.064, 0.058, 0.058, 0.057, 0.055, 0.059, 0.16, 0.126, 0.128, 0.282, 0.138, 0.125, 0.122, 0.132, 0.322, 0.128, 0.199, 0.431, 0.121, 0.204, 0.189, 0.13, 0.124, 0.264, 0.135, 0.173, 0.147, 0.125, 0.127, 0.244, 0.193, 0.126, 0.132, 0.294, 0.051, 0.133, 0.159, 0.127, 0.124, 0.207, 0.221, 0.195, 0.138, 0.258, 0.048, 0.058, 0.128, 0.164, 0.126, 0.122, 0.135, 0.272, 0.13, 0.166, 0.186, 0.124, 0.173, 0.133, 0.239, 0.269, 0.135, 0.284, 0.293, 0.244, 0.227, 0.162, 0.157, 0.194, 0.185, 0.203, 0.284, 0.202, 0.33, 0.212, 0.314, 0.216, 0.395, 0.242, 0.235, 0.235, 0.339, 0.253, 0.331, 0.287, 0.284, 0.277, 0.313, 0.281, 0.318, 0.3, 0.358, 0.317, 0.31, 0.309, 0.306, 0.462, 0.338, 0.334, 0.354, 0.479, 0.373, 0.911, 0.055, 0.185, 0.161, 0.127, 0.24, 0.125, 0.129, 0.215, 0.179, 0.127, 0.143, 0.135, 0.208, 0.223, 0.162, 0.15, 0.161, 0.18, 0.238, 0.185, 0.309, 0.217, 0.207, 0.19, 0.425]\n",
+    "# incremental = [0.133, 0.192, 0.198, 0.229, 0.133, 0.135, 0.239, 0.146, 0.143, 0.141, 0.325, 0.186, 0.164, 0.146, 0.304, 0.059, 0.058, 0.052, 0.13, 0.14, 0.139, 0.272, 0.167, 0.308, 0.152, 0.144, 0.243, 0.149, 0.211, 0.223, 0.227, 0.163, 0.28, 0.184, 0.178, 0.243, 0.338, 0.199, 0.226, 0.391, 0.06, 0.19, 0.207, 0.129, 0.21, 0.197, 0.169, 0.133, 0.132, 0.231, 0.145, 0.361, 0.055, 0.189, 0.135, 0.125, 0.123, 0.166, 0.133, 0.129, 0.319, 0.225, 0.139, 0.151, 0.19, 0.243, 0.152, 0.256, 0.165, 0.25, 0.191, 0.173, 0.204, 0.21, 0.212, 0.198, 0.296, 0.197, 0.312, 0.234, 0.225, 0.217, 0.259, 0.249, 0.336, 0.348, 0.289, 0.274, 0.308, 0.273, 0.29, 0.742, 0.066, 0.055, 0.128, 0.16, 0.132, 0.238, 0.134, 0.221, 0.142, 0.135, 0.149, 0.268, 0.151, 0.147, 0.152, 0.361, 0.162, 0.124, 0.177, 0.126, 0.127, 0.303, 0.061, 0.061, 0.053, 0.053, 0.057, 0.059, 0.056, 0.063, 0.059, 0.079, 0.187, 0.126, 0.132, 0.279, 0.131, 0.175, 0.123, 0.135, 0.304, 0.127, 0.155, 0.143, 0.126, 0.215, 0.197, 0.177, 0.133, 0.245, 0.125, 0.168, 0.168, 0.122, 0.122, 0.257, 0.223, 0.142, 0.129, 0.311, 0.073, 0.136, 0.203, 0.135, 0.125, 0.752, 0.382, 0.217, 0.13, 0.262, 0.056, 0.057, 0.135, 0.181, 0.128, 0.132, 0.126, 0.249, 0.126, 0.161, 0.204, 0.132, 0.133, 0.133, 0.243, 0.247, 0.213, 0.359, 0.148, 0.433, 0.243, 0.152, 0.176, 0.186, 0.187, 0.178, 0.286, 0.199, 0.201, 0.198, 0.367, 0.32, 0.198, 0.224, 0.218, 0.216, 0.341, 0.233, 0.241, 0.291, 0.311, 0.286, 0.31, 0.28, 0.277, 0.274, 0.288, 0.361, 0.31, 0.303, 0.291, 0.363, 0.425, 0.318, 0.362, 0.463, 0.365, 0.918, 0.062, 0.194, 0.164, 0.129, 0.126, 0.19, 0.134, 0.279, 0.186, 0.139, 0.137, 0.144, 0.228, 0.233, 0.15, 0.155, 0.162, 0.166, 0.272, 0.187, 0.323, 0.183, 0.188, 0.198, 0.423]\n",
+    "# incremental = [0.187, 0.13, 0.26, 0.14, 0.134, 0.251, 0.132, 0.212, 0.15, 0.143, 0.363, 0.141, 0.167, 0.128, 0.285, 0.055, 0.052, 0.136, 0.13, 0.168, 0.129, 0.247, 0.241, 0.194, 0.152, 0.218, 0.209, 0.151, 0.267, 0.175, 0.224, 0.165, 0.174, 0.269, 0.201, 0.243, 0.322, 0.219, 0.199, 0.558, 0.058, 0.139, 0.171, 0.14, 0.336, 0.141, 0.137, 0.148, 0.24, 0.319, 0.149, 0.56, 0.055, 0.137, 0.172, 0.134, 0.269, 0.129, 0.138, 0.131, 0.245, 0.22, 0.137, 0.183, 0.232, 0.16, 0.151, 0.247, 0.171, 0.253, 0.19, 0.182, 0.189, 0.204, 0.186, 0.199, 0.328, 0.232, 0.372, 0.229, 0.214, 0.228, 0.242, 0.234, 0.653, 0.289, 0.324, 0.344, 0.293, 0.279, 0.749, 0.063, 0.056, 0.057, 0.136, 0.124, 0.167, 0.251, 0.139, 0.134, 0.236, 0.134, 0.147, 0.322, 0.162, 0.142, 0.376, 0.14, 0.173, 0.138, 0.266, 0.142, 0.136, 0.278, 0.065, 0.059, 0.059, 0.062, 0.057, 0.055, 0.063, 0.064, 0.072, 0.065, 0.136, 0.124, 0.12, 0.276, 0.132, 0.164, 0.138, 0.256, 0.05, 0.134, 0.167, 0.124, 0.234, 0.152, 0.201, 0.154, 0.137, 0.334, 0.174, 0.183, 0.132, 0.134, 0.167, 0.308, 0.209, 0.143, 0.141, 0.313, 0.056, 0.13, 0.176, 0.131, 0.133, 0.245, 0.214, 0.206, 0.148, 0.25, 0.053, 0.122, 0.118, 0.189, 0.13, 0.128, 0.136, 0.24, 0.135, 0.171, 0.126, 0.146, 0.216, 0.208, 0.29, 0.133, 0.132, 0.286, 0.336, 0.263, 0.157, 0.166, 0.173, 0.184, 0.185, 0.284, 0.185, 0.203, 0.203, 0.198, 0.359, 0.312, 0.339, 0.256, 0.253, 0.284, 0.335, 0.259, 0.334, 0.307, 0.304, 0.281, 0.373, 0.3, 0.289, 0.29, 0.286, 0.327, 0.3, 0.299, 0.32, 0.435, 0.32, 0.464, 0.339, 0.334, 0.348, 1.288, 0.058, 0.201, 0.126, 0.527, 0.284, 0.148, 0.13, 0.21, 0.175, 0.138, 0.135, 0.141, 0.215, 0.238, 0.16, 0.164, 0.161, 0.186, 0.247, 0.192, 0.375, 0.191, 0.2, 0.431]\n",
+    "incremental = [0.153, 0.171, 0.197, 0.138, 0.23, 0.137, 0.141, 0.145, 0.225, 0.146, 0.148, 0.336, 0.138, 0.163, 0.181, 0.133, 0.354, 0.064, 0.053, 0.133, 0.132, 0.166, 0.135, 0.232, 0.234, 0.231, 0.204, 0.214, 0.157, 0.189, 0.236, 0.221, 0.245, 0.229, 0.205, 0.203, 0.248, 0.194, 0.201, 0.197, 0.483, 0.057, 0.052, 0.132, 0.158, 0.13, 0.251, 0.197, 0.135, 0.143, 0.262, 0.144, 0.135, 0.38, 0.054, 0.226, 0.128, 0.166, 0.261, 0.139, 0.132, 0.284, 0.23, 0.234, 0.142, 0.211, 0.217, 0.155, 0.163, 0.285, 0.169, 0.242, 0.201, 0.188, 0.189, 0.186, 0.187, 0.311, 0.214, 0.213, 0.304, 0.23, 0.265, 0.219, 0.234, 0.252, 0.239, 0.339, 0.278, 0.365, 0.271, 0.277, 0.729, 0.051, 0.055, 0.059, 0.132, 0.121, 0.162, 0.239, 0.133, 0.135, 0.214, 0.142, 0.132, 0.258, 0.144, 0.135, 0.386, 0.13, 0.135, 0.133, 0.273, 0.136, 0.133, 0.254, 0.05, 0.049, 0.05, 0.053, 0.093, 0.064, 0.055, 0.053, 0.049, 0.056, 0.142, 0.125, 0.131, 0.273, 0.131, 0.124, 0.121, 0.264, 0.135, 0.172, 0.159, 0.136, 0.205, 0.21, 0.159, 0.159, 0.14, 0.324, 0.161, 0.171, 0.127, 0.139, 0.154, 0.293, 0.22, 0.149, 0.188, 0.331, 0.051, 0.133, 0.132, 0.122, 0.152, 0.25, 0.191, 0.277, 0.134, 0.259, 0.051, 0.129, 0.168, 0.169, 0.134, 0.121, 0.253, 0.051, 0.147, 0.163, 0.137, 0.13, 0.264, 0.286, 0.179, 0.142, 0.224, 0.367, 0.156, 0.444, 0.173, 0.178, 0.187, 0.186, 0.189, 0.277, 0.184, 0.189, 0.189, 0.198, 0.366, 0.214, 0.306, 0.221, 0.221, 0.335, 0.269, 0.253, 0.299, 0.285, 0.28, 0.271, 0.353, 0.285, 0.291, 0.28, 0.307, 0.297, 0.31, 0.297, 0.305, 0.48, 0.323, 0.33, 0.329, 0.33, 0.333, 0.922, 0.131, 0.168, 0.188, 0.135, 0.21, 0.143, 0.13, 0.224, 0.138, 0.147, 0.198, 0.14, 0.223, 0.156, 0.239, 0.164, 0.163, 0.295, 0.184, 0.186, 0.333, 0.204, 0.197, 0.439]\n",
+    "# self_attn_state\n",
+    "# incremental = [0.122, 0.2, 0.133, 0.321, 0.136, 0.154, 0.141, 0.181, 0.212, 0.143, 0.313, 0.172, 0.134, 0.136, 0.356, 0.057, 0.078, 0.056, 0.139, 0.121, 0.158, 0.218, 0.203, 0.171, 0.208, 0.142, 0.24, 0.14, 0.141, 0.142, 0.158, 0.155, 0.184, 0.181, 0.221, 0.459, 0.198, 0.123, 0.126, 0.27, 0.067, 0.125, 0.163, 0.188, 0.129, 0.125, 0.239, 0.138, 0.136, 0.138, 0.133, 0.137, 0.381, 0.055, 0.129, 0.118, 0.166, 0.171, 0.131, 0.134, 0.278, 0.187, 0.229, 0.182, 0.129, 0.172, 0.128, 0.126, 0.248, 0.144, 0.217, 0.148, 0.14, 0.151, 0.266, 0.502, 0.155, 0.161, 0.211, 0.178, 0.176, 0.186, 0.187, 0.183, 0.211, 0.204, 0.22, 0.214, 0.206, 0.296, 0.252, 0.192, 0.063, 0.054, 0.056, 0.128, 0.124, 0.171, 0.263, 0.134, 0.198, 0.145, 0.26, 0.155, 0.123, 0.197, 0.127, 0.413, 0.209, 0.123, 0.194, 0.131, 0.128, 0.297, 0.053, 0.05, 0.057, 0.055, 0.046, 0.063, 0.06, 0.057, 0.057, 0.059, 0.135, 0.164, 0.121, 0.116, 0.258, 0.124, 0.118, 0.127, 0.285, 0.131, 0.164, 0.128, 0.134, 0.223, 0.208, 0.148, 0.124, 0.127, 0.263, 0.166, 0.128, 0.139, 0.162, 0.296, 0.189, 0.13, 0.135, 0.318, 0.049, 0.123, 0.17, 0.128, 0.125, 0.21, 0.214, 0.139, 0.14, 0.274, 0.051, 0.062, 0.131, 0.145, 0.162, 0.136, 0.125, 0.282, 0.13, 0.164, 0.205, 0.129, 0.156, 0.156, 0.251, 0.357, 0.139, 0.138, 0.149, 0.353, 0.19, 0.304, 0.165, 0.186, 0.194, 0.196, 0.183, 0.187, 0.194, 0.199, 0.206, 0.267, 0.212, 0.257, 0.228, 0.226, 0.225, 0.239, 0.3, 0.268, 0.272, 0.278, 0.303, 0.283, 0.274, 0.279, 0.274, 0.289, 0.303, 0.3, 0.303, 0.313, 0.305, 0.332, 0.344, 0.343, 0.342, 1.678, 0.062, 0.164, 0.13, 0.204, 0.171, 0.217, 0.171, 0.247, 0.207, 0.134, 0.13, 0.138, 0.138, 0.147, 0.187, 0.142, 0.242, 0.158, 0.16, 0.168, 0.214, 0.284, 0.185, 0.179, 0.365]\n",
+    "# incremental = [0.231, 0.134, 0.238, 0.195, 0.206, 0.147, 0.136, 0.149, 0.187, 0.214, 0.148, 0.354, 0.19, 0.213, 0.139, 0.138, 0.281, 0.061, 0.069, 0.178, 0.145, 0.133, 0.21, 0.208, 0.14, 0.287, 0.178, 0.173, 0.149, 0.27, 0.398, 0.376, 0.496, 0.164, 0.162, 0.178, 0.375, 0.273, 0.123, 0.126, 0.327, 0.058, 0.063, 0.134, 0.167, 0.127, 0.19, 0.167, 0.128, 0.132, 0.269, 0.14, 0.136, 0.388, 0.052, 0.245, 0.169, 0.126, 0.163, 0.126, 0.133, 0.264, 0.133, 0.246, 0.153, 0.164, 0.158, 0.119, 0.129, 0.254, 0.157, 0.268, 0.14, 0.135, 0.241, 0.143, 0.151, 0.155, 0.161, 0.188, 0.233, 0.191, 0.184, 0.179, 0.224, 0.213, 0.218, 0.226, 0.21, 0.211, 0.297, 0.253, 0.048, 0.056, 0.06, 0.059, 0.12, 0.169, 0.13, 0.233, 0.136, 0.21, 0.173, 0.162, 0.263, 0.146, 0.153, 0.139, 0.406, 0.191, 0.144, 0.163, 0.267, 0.139, 0.298, 0.055, 0.061, 0.049, 0.052, 0.055, 0.053, 0.051, 0.06, 0.053, 0.048, 0.052, 0.166, 0.123, 0.123, 0.26, 0.131, 0.161, 0.124, 0.277, 0.124, 0.159, 0.131, 0.131, 0.211, 0.135, 0.2, 0.135, 0.174, 0.285, 0.186, 0.179, 0.139, 0.133, 0.231, 0.181, 0.137, 0.199, 0.135, 0.306, 0.128, 0.173, 0.119, 0.116, 0.126, 0.285, 0.151, 0.136, 0.151, 0.363, 0.063, 0.173, 0.129, 0.169, 0.121, 0.129, 0.326, 0.13, 0.168, 0.212, 0.125, 0.136, 0.132, 0.247, 0.313, 0.144, 0.142, 0.149, 0.293, 0.178, 0.363, 0.176, 0.188, 0.185, 0.192, 0.178, 0.192, 0.213, 0.199, 0.204, 0.256, 0.21, 0.22, 0.208, 0.226, 0.242, 0.241, 0.556, 0.257, 0.262, 0.259, 0.26, 0.271, 0.379, 0.318, 0.282, 0.304, 0.331, 0.322, 0.295, 0.328, 0.328, 0.301, 0.345, 0.331, 0.333, 0.69, 0.056, 0.123, 0.167, 0.196, 0.127, 0.21, 0.129, 0.208, 0.234, 0.134, 0.141, 0.144, 0.137, 0.142, 0.139, 0.242, 0.152, 0.146, 0.16, 0.18, 0.287, 0.184, 0.18, 0.177, 0.321]\n",
+    "fig, ax = plt.subplots()\n",
+    "ax.boxplot([nonincremental, incremental])\n",
+    "ax.set_xticklabels([\"nonincremental\", \"incremental\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.262 0.20222357723577236\n",
+      "0.2530612244897959 0.0975609756097561\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(np.mean(nonincremental), np.mean(incremental))\n",
+    "print(sum([1 for x in nonincremental if x > 0.32])/len(nonincremental), sum([1 for x in incremental if x > 0.32])/len(incremental))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 75,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.41239999999999993"
+      ]
+     },
+     "execution_count": 75,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.percentile(nonincremental, 90)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.3105"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.percentile(incremental, 90)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "fairseq-20230220-seamless",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

seamless_server/src/transcoder_helpers.py

@@ -0,0 +1,43 @@
+import logging
+
+logger = logging.getLogger("socketio_server_pubsub")
+
+
+def get_transcoder_output_events(transcoder) -> list:
+    speech_and_text_output = transcoder.get_buffered_output()
+    if speech_and_text_output is None:
+        logger.debug("No output from transcoder.get_buffered_output()")
+        return []
+
+    logger.debug(f"We DID get output from the transcoder! {speech_and_text_output}")
+
+    lat = None
+
+    events = []
+
+    if speech_and_text_output.speech_samples:
+        events.append(
+            {
+                "event": "translation_speech",
+                "payload": speech_and_text_output.speech_samples,
+                "sample_rate": speech_and_text_output.speech_sample_rate,
+            }
+        )
+
+    if speech_and_text_output.text:
+        events.append(
+            {
+                "event": "translation_text",
+                "payload": speech_and_text_output.text,
+            }
+        )
+
+    for e in events:
+        e["eos"] = speech_and_text_output.final
+
+    # if not latency_sent:
+    #     lat = transcoder.first_translation_time()
+    #     latency_sent = True
+    #     to_send["latency"] = lat
+
+    return events

streaming-react-app/.eslintrc.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  root: true,
+  env: {browser: true, es2020: true},
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      {allowConstantExport: true},
+    ],
+  },
+};

streaming-react-app/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

streaming-react-app/README.md

@@ -0,0 +1,65 @@
+# 🚀 Streaming React App
+
+## Getting Started
+
+- `yarn run dev` - Run the app with a development server that supports hot module reloading
+
+## URL Parameters
+
+You can provide URL parameters in order to change the behavior of the app. Those are documented in [URLParams.ts](src/URLParams.ts).
+
+# Vite Information: React + TypeScript + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
+
+- Configure the top-level `parserOptions` property like this:
+
+```js
+   parserOptions: {
+    ecmaVersion: 'latest',
+    sourceType: 'module',
+    project: ['./tsconfig.json', './tsconfig.node.json'],
+    tsconfigRootDir: __dirname,
+   },
+```
+
+- Replace `plugin:@typescript-eslint/recommended` to `plugin:@typescript-eslint/recommended-type-checked` or `plugin:@typescript-eslint/strict-type-checked`
+- Optionally add `plugin:@typescript-eslint/stylistic-type-checked`
+- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and add `plugin:react/recommended` & `plugin:react/jsx-runtime` to the `extends` list
+
+# To Deploy to AWS
+
+1. Acquire AWS credentials (not needed if already on an EC2 instance with permissions)
+
+On your local mac use the following command.
+
+```
+eval $(corp_cloud aws get-creds 790537050551)
+```
+
+2. Deploy to AWS
+
+Build the react and copy the contents of [dist](dist) folder to s3 bucket and then invalidate the cloudfront (CDN) cache. Note step 2 has been automated using `yarn deploy_dev`
+
+To deploy to the (old) seamless-vc s3 bucket:
+
+```
+yarn build:dev_vc
+yarn deploy_dev_vc
+```
+
+To deploy to the (new) seamless-vr terraform-based s3 bucket:
+
+```
+yarn build:dev_vr
+yarn deploy_dev_vr
+```

streaming-react-app/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/src/assets/seamless.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Seamless Translation</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

streaming-react-app/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "streaming-react-app",
+  "private": true,
+  "version": "0.0.12",
+  "type": "module",
+  "scripts": {
+    "dev": "vite --host --strictPort",
+    "build": "vite build",
+    "build:dev_vr": "yarn build --mode deploy_dev_vr",
+    "build:dev_vc": "yarn build --mode deploy_dev_vc",
+    "preview": "vite preview",
+    "clean:node-modules": "rm -rf node_modules/",
+    "ts-check": "tsc --noEmit",
+    "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "prettier-check": "cd ../ && yarn run prettier-base --check streaming-react-app",
+    "signal": "concurrently --names \"TS,LINT,PRETTIER\" -c \"bgBlack.bold,bgRed.bold,bgCyan.bold\" \"yarn run ts-check\" \"yarn run lint\" \"yarn run prettier-check\"",
+    "deploy_dev_vr": "aws s3 sync dist/ s3://dev-seamless-vr-www && aws cloudfront create-invalidation --distribution-id E38ZTD4R79FGYF --paths \"/*\"",
+    "deploy_dev_vc": "aws s3 sync dist/ s3://seamless-vc.dev.metademolab.com && aws cloudfront create-invalidation --distribution-id E29D1W2ORBP77G --paths \"/*\""
+  },
+  "dependencies": {
+    "@emotion/react": "11.11.1",
+    "@emotion/styled": "11.11.0",
+    "@mui/icons-material": "5.14.3",
+    "@mui/material": "5.14.5",
+    "@react-three/drei": "^9.83.9",
+    "@react-three/fiber": "^8.14.1",
+    "@react-three/xr": "^5.7.1",
+    "amazon-cognito-identity-js": "^6.3.6",
+    "audiobuffer-to-wav": "^1.0.0",
+    "aws-sdk": "^2.1472.0",
+    "iso-639-1": "^3.1.0",
+    "js-cookie": "^3.0.5",
+    "lodash": "4.17.21",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-google-charts": "^4.0.1",
+    "socket.io-client": "^4.7.2",
+    "three": "^0.156.1",
+    "three-mesh-ui": "^6.5.4",
+    "uuid": "^9.0.0",
+    "zustand": "^4.4.3"
+  },
+  "devDependencies": {
+    "@types/node": "^20.5.3",
+    "@types/react": "^18.2.15",
+    "@types/react-dom": "^18.2.7",
+    "@types/uuid": "^9.0.2",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "@vitejs/plugin-react": "^4.0.3",
+    "concurrently": "8.2.1",
+    "eslint": "^8.45.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.3",
+    "typescript": "5.1.6",
+    "vite": "^4.4.5"
+  }
+}

streaming-react-app/src/App.tsx

@@ -0,0 +1,57 @@
+import SocketWrapper from './SocketWrapper';
+import {ThemeProvider} from '@mui/material/styles';
+import theme from './theme';
+import StreamingInterface from './StreamingInterface';
+import CssBaseline from '@mui/material/CssBaseline';
+import {createContext, useCallback, useState} from 'react';
+import packageJson from '../package.json';
+
+console.log(`Streaming React App version: ${packageJson?.version}`);
+
+// Roboto font for mui ui library
+// import '@fontsource/roboto/300.css';
+// import '@fontsource/roboto/400.css';
+// import '@fontsource/roboto/500.css';
+// import '@fontsource/roboto/700.css';
+
+export const AppResetKeyContext = createContext<(newKey: string) => void>(
+  () => {
+    throw new Error('AppResetKeyContext not initialized');
+  },
+);
+
+function App() {
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <SocketWrapper>
+        <StreamingInterface />
+      </SocketWrapper>
+    </ThemeProvider>
+  );
+}
+
+function AppWrapper() {
+  const [appResetKey, setAppResetKey] = useState<string>('[initial value]');
+  const setAppResetKeyHandler = useCallback((newKey: string) => {
+    setAppResetKey((prev) => {
+      console.warn(
+        `Resetting the app with appResetKey: ${newKey}; prevKey: ${prev}`,
+      );
+      if (prev === newKey) {
+        console.error(
+          `The appResetKey was the same as the previous key, so the app will not reset.`,
+        );
+      }
+      return newKey;
+    });
+  }, []);
+
+  return (
+    <AppResetKeyContext.Provider value={setAppResetKeyHandler}>
+      <App key={appResetKey} />
+    </AppResetKeyContext.Provider>
+  );
+}
+
+export default AppWrapper;

streaming-react-app/src/Blink.tsx

@@ -0,0 +1,41 @@
+import Box from '@mui/material/Box';
+import {useEffect, useState} from 'react';
+
+type Props = {
+  intervalMs: number;
+  children: React.ReactNode;
+  shouldBlink: boolean;
+  // display?: 'block' | 'inline' | 'inline-block';
+};
+
+export default function Blink({
+  // display = 'inline-block',
+  shouldBlink,
+  intervalMs,
+  children,
+}: Props): React.ReactElement {
+  const [cursorBlinkOn, setCursorBlinkOn] = useState(false);
+
+  useEffect(() => {
+    if (shouldBlink) {
+      const interval = setInterval(() => {
+        setCursorBlinkOn((prev) => !prev);
+      }, intervalMs);
+
+      return () => clearInterval(interval);
+    } else {
+      setCursorBlinkOn(false);
+    }
+  }, [intervalMs, shouldBlink]);
+
+  return (
+    <Box
+      component="span"
+      sx={{
+        display: 'inline-block',
+        visibility: cursorBlinkOn ? 'visible' : 'hidden',
+      }}>
+      {children}
+    </Box>
+  );
+}

streaming-react-app/src/DebugSection.tsx

@@ -0,0 +1,62 @@
+import {Chart} from 'react-google-charts';
+import debug from './debug';
+import {
+  Accordion,
+  AccordionDetails,
+  AccordionSummary,
+  Button,
+  Typography,
+} from '@mui/material';
+import {useState} from 'react';
+import ArrowDropDownIcon from '@mui/icons-material/ArrowDropDown';
+
+export default function DebugChart() {
+  const [showDebugTimings, setShowDebugTimings] = useState<boolean>(false);
+
+  const data = debug()?.getChartData();
+  const options = {
+    timeline: {
+      groupByRowLabel: true,
+    },
+  };
+
+  return (
+    <div className="horizontal-padding-sra text-chunk-sra">
+      <Accordion
+        expanded={showDebugTimings}
+        onChange={() => setShowDebugTimings(!showDebugTimings)}
+        elevation={0}
+        sx={{border: 1, borderColor: 'rgba(0, 0, 0, 0.3)'}}>
+        <AccordionSummary
+          expandIcon={<ArrowDropDownIcon />}
+          className="debug-section">
+          Debug Info
+        </AccordionSummary>
+        <AccordionDetails>
+          {data && data.length > 1 ? (
+            <>
+              <Chart
+                chartType="Timeline"
+                data={data}
+                width="100%"
+                height="400px"
+                options={options}
+              />
+              <Button
+                variant="contained"
+                sx={{marginBottom: 1}}
+                onClick={() => {
+                  debug()?.downloadInputAudio();
+                  debug()?.downloadOutputAudio();
+                }}>
+                Download Input / Ouput Audio
+              </Button>
+            </>
+          ) : (
+            <Typography>No input / output detected</Typography>
+          )}
+        </AccordionDetails>
+      </Accordion>
+    </div>
+  );
+}

streaming-react-app/src/RoomConfig.tsx

@@ -0,0 +1,263 @@
+import Stack from '@mui/material/Stack';
+import TextField from '@mui/material/TextField';
+import {isValidRoomID, isValidPartialRoomID} from './generateNewRoomID';
+import {useCallback, useEffect, useState} from 'react';
+import Button from '@mui/material/Button';
+import {useSocket} from './useSocket';
+import FormGroup from '@mui/material/FormGroup';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import Checkbox from '@mui/material/Checkbox';
+import {RoomState} from './types/RoomState';
+import setURLParam from './setURLParam';
+import {getURLParams} from './URLParams';
+import {
+  JoinRoomConfig,
+  Roles,
+  ServerState,
+  StreamingStatus,
+} from './types/StreamingTypes';
+import Alert from '@mui/material/Alert';
+
+function capitalize(str: string): string {
+  return str.charAt(0).toUpperCase() + str.slice(1);
+}
+
+type Props = {
+  roomState: RoomState | null;
+  serverState: ServerState | null;
+  onJoinRoomOrUpdateRoles?: () => void;
+  streamingStatus: StreamingStatus;
+};
+
+export default function RoomConfig({
+  roomState,
+  serverState,
+  onJoinRoomOrUpdateRoles,
+  streamingStatus,
+}: Props) {
+  const {socket, clientID} = useSocket();
+
+  const urlParams = getURLParams();
+  const roomIDParam = urlParams.roomID;
+  const autoJoinRoom = urlParams.autoJoin;
+
+  const [roomID, setRoomID] = useState<string>(
+    (roomIDParam ?? '').toUpperCase(),
+  );
+  const [roomIDError, setRoomIDError] = useState<boolean>(false);
+  const [roles, setRoles] = useState<{speaker: boolean; listener: boolean}>({
+    speaker: true,
+    listener: true,
+  });
+  const [lockServer, setLockServer] = useState<boolean>(false);
+  const [lockServerName, setLockServerName] = useState<string>('');
+
+  const [joinInProgress, setJoinInProgress] = useState<boolean>(false);
+  const [didAttemptAutoJoin, setDidAttemptAutoJoin] = useState<boolean>(false);
+
+  const isValidServerLock =
+    lockServer === false ||
+    (lockServerName != null && lockServerName.length > 0);
+  const isValidRoles = Object.values(roles).filter(Boolean).length > 0;
+  const isValidAllInputs =
+    isValidRoomID(roomID) && isValidRoles && isValidServerLock;
+  const roomIDFromServer = roomState?.room_id ?? null;
+
+  const onJoinRoom = useCallback(
+    (createNewRoom: boolean) => {
+      if (socket == null) {
+        console.error('Socket is null, cannot join room');
+        return;
+      }
+      console.debug(`Attempting to join roomID ${roomID}...`);
+
+      const lockServerValidated: string | null =
+        lockServer && roles['speaker'] ? lockServerName : null;
+
+      // TODO: Show error state if roomID isn't valid
+      setJoinInProgress(true);
+
+      const configObject: JoinRoomConfig = {
+        roles: (Object.keys(roles) as Array<Roles>).filter(
+          (role) => roles[role] === true,
+        ),
+        lockServerName: lockServerValidated,
+      };
+
+      socket.emit(
+        'join_room',
+        clientID,
+        createNewRoom ? null : roomID,
+        configObject,
+        (result) => {
+          console.log('join_room result:', result);
+          if (createNewRoom) {
+            setRoomID(result.roomID);
+          }
+          if (onJoinRoomOrUpdateRoles != null) {
+            onJoinRoomOrUpdateRoles();
+          }
+          setURLParam('roomID', result.roomID);
+          setJoinInProgress(false);
+        },
+      );
+    },
+    [
+      clientID,
+      lockServer,
+      lockServerName,
+      onJoinRoomOrUpdateRoles,
+      roles,
+      roomID,
+      socket,
+    ],
+  );
+
+  useEffect(() => {
+    if (
+      autoJoinRoom === true &&
+      didAttemptAutoJoin === false &&
+      socket != null
+    ) {
+      // We want to consider this an attempt whether or not we actually try to join, because
+      // we only want auto-join to happen on initial load
+      setDidAttemptAutoJoin(true);
+      if (
+        isValidAllInputs &&
+        joinInProgress === false &&
+        roomIDFromServer == null
+      ) {
+        console.debug('Attempting to auto-join room...');
+
+        onJoinRoom(false);
+      } else {
+        console.debug('Unable to auto-join room', {
+          isValidAllInputs,
+          joinInProgress,
+          roomIDFromServer,
+        });
+      }
+    }
+  }, [
+    autoJoinRoom,
+    didAttemptAutoJoin,
+    isValidAllInputs,
+    joinInProgress,
+    onJoinRoom,
+    roomIDFromServer,
+    socket,
+  ]);
+
+  return (
+    <Stack direction="column" spacing="12px">
+      <Stack direction="row" spacing="12px" sx={{alignItems: 'center'}}>
+        <TextField
+          size="small"
+          label="Room Code"
+          variant="outlined"
+          disabled={roomState?.room_id != null}
+          value={roomID}
+          error={roomIDError}
+          onChange={(e) => {
+            const id = e.target.value.toUpperCase();
+            if (isValidPartialRoomID(id)) {
+              setRoomIDError(false);
+              setRoomID(id);
+            } else {
+              setRoomIDError(true);
+            }
+          }}
+          sx={{width: '8em'}}
+        />
+
+        <div>
+          <Button
+            variant="contained"
+            disabled={
+              isValidAllInputs === false ||
+              joinInProgress ||
+              streamingStatus !== 'stopped'
+            }
+            onClick={() => onJoinRoom(false)}>
+            {roomState?.room_id != null ? 'Update Roles' : 'Join Room'}
+          </Button>
+        </div>
+
+        {roomState?.room_id == null && (
+          <div>
+            <Button
+              variant="contained"
+              disabled={
+                roomState?.room_id != null ||
+                joinInProgress ||
+                streamingStatus !== 'stopped'
+              }
+              onClick={() => onJoinRoom(true)}>
+              {'Create New Room'}
+            </Button>
+          </div>
+        )}
+      </Stack>
+
+      <FormGroup>
+        {Object.keys(roles).map((role) => {
+          return (
+            <FormControlLabel
+              disabled={streamingStatus !== 'stopped'}
+              key={role}
+              control={
+                <Checkbox
+                  checked={roles[role]}
+                  onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+                    setRoles((prevRoles) => ({
+                      ...prevRoles,
+                      [role]: event.target.checked,
+                    }));
+                  }}
+                />
+              }
+              label={capitalize(role)}
+            />
+          );
+        })}
+
+        {urlParams.enableServerLock && roles['speaker'] === true && (
+          <>
+            <FormControlLabel
+              disabled={streamingStatus !== 'stopped'}
+              control={
+                <Checkbox
+                  checked={lockServer}
+                  onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+                    setLockServer(event.target.checked);
+                  }}
+                />
+              }
+              label="Lock Server (prevent other users from streaming)"
+            />
+          </>
+        )}
+      </FormGroup>
+
+      {urlParams.enableServerLock &&
+        roles['speaker'] === true &&
+        lockServer && (
+          <TextField
+            disabled={streamingStatus !== 'stopped'}
+            label="Enter Your Name + Expected Lock End Time"
+            variant="outlined"
+            value={lockServerName}
+            onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+              setLockServerName(event.target.value);
+            }}
+            helperText="Locking the server will prevent anyone else from using it until you close the page, in order to maximize server performance. Please only use this for live demos."
+          />
+        )}
+
+      {serverState?.serverLock != null &&
+        serverState.serverLock.clientID === clientID && (
+          <Alert severity="success">{`The server is now locked for your use (${serverState?.serverLock?.name}). Close this window to release the lock so that others may use the server.`}</Alert>
+        )}
+    </Stack>
+  );
+}

streaming-react-app/src/SeamlessLogo.tsx

@@ -0,0 +1,33 @@
+function SeamlessLogo() {
+  return (
+    <svg
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg">
+      <circle cx="12" cy="12" r="12" fill="#1C2B33" />
+      <rect x="7" y="9" width="2" height="6" rx="1" fill="white" />
+      <rect
+        x="15"
+        y="9"
+        width="2"
+        height="6"
+        rx="1"
+        fill="white"
+        fill-opacity="0.5"
+      />
+      <rect
+        x="11"
+        y="6"
+        width="2"
+        height="12"
+        rx="1"
+        fill="white"
+        fill-opacity="0.5"
+      />
+    </svg>
+  );
+}
+
+export default SeamlessLogo;

streaming-react-app/src/SocketWrapper.tsx

@@ -0,0 +1,217 @@
+import {useContext, useEffect, useMemo, useRef, useState} from 'react';
+import socketIOClient, {Socket} from 'socket.io-client';
+import useStable from './useStable';
+import {v4 as uuidv4} from 'uuid';
+import {SocketContext} from './useSocket';
+import {AppResetKeyContext} from './App';
+import Backdrop from '@mui/material/Backdrop';
+import CircularProgress from '@mui/material/CircularProgress';
+import Typography from '@mui/material/Typography';
+import {getURLParams} from './URLParams';
+
+// The time to wait before showing a "disconnected" screen upon initial app load
+const INITIAL_DISCONNECT_SCREEN_DELAY = 2000;
+const SERVER_URL_DEFAULT = "localhost:8000"
+
+export default function SocketWrapper({children}) {
+  const [socket, setSocket] = useState<Socket | null>(null);
+  const [connected, setConnected] = useState<boolean | null>(null);
+  // Default to true:
+  const [willAttemptReconnect, setWillAttemptReconnect] =
+    useState<boolean>(true);
+  const serverIDRef = useRef<string | null>(null);
+
+  const setAppResetKey = useContext(AppResetKeyContext);
+
+  /**
+   * Previously we had stored the clientID in local storage, but in that case
+   * if a user refreshes their page they'll still have the same clientID, and
+   * will be put back into the same room, which may be confusing if they're trying
+   * to join a new room or reset the app interface. So now clientIDs persist only as
+   * long as the react app full lifecycle
+   */
+  const clientID = useStable<string>(() => {
+    const newID = uuidv4();
+    // Set the clientID in session storage so if the page reloads the person
+    // still retains their member/room config
+    return newID;
+  });
+
+  const socketObject = useMemo(
+    () => ({socket, clientID, connected: connected ?? false}),
+    [socket, clientID, connected],
+  );
+
+  useEffect(() => {
+    const queryParams = {
+      clientID: clientID,
+    };
+
+    const serverURLFromParams = getURLParams().serverURL;
+    const serverURL = serverURLFromParams ?? SERVER_URL_DEFAULT;
+
+    console.log(
+      `Opening socket connection to ${
+        serverURL?.length === 0 ? 'window.location.host' : serverURL
+      } with query params:`,
+      queryParams,
+    );
+
+    const newSocket: Socket = socketIOClient(serverURL, {
+      query: queryParams,
+      // Normally socket.io will fallback to http polling, but we basically never
+      // want that because that'd mean awful performance. It'd be better for the app
+      // to simply break in that case and not connect.
+      transports: ['websocket'],
+    });
+
+    const onServerID = (serverID: string) => {
+      console.debug('Received server ID:', serverID);
+      if (serverIDRef.current != null) {
+        if (serverIDRef.current !== serverID) {
+          console.error(
+            'Server ID changed. Resetting the app using the app key',
+          );
+          setAppResetKey(serverID);
+        }
+      }
+      serverIDRef.current = serverID;
+    };
+
+    newSocket.on('server_id', onServerID);
+
+    setSocket(newSocket);
+
+    return () => {
+      newSocket.off('server_id', onServerID);
+      console.log(
+        'Closing socket connection in the useEffect cleanup function...',
+      );
+      newSocket.disconnect();
+      setSocket(null);
+    };
+  }, [clientID, setAppResetKey]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onAny = (eventName: string, ...args) => {
+        console.debug(`[event: ${eventName}] args:`, ...args);
+      };
+
+      socket.onAny(onAny);
+
+      return () => {
+        socket.offAny(onAny);
+      };
+    }
+    return () => {};
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onConnect = (...args) => {
+        console.debug('Connected to server with args:', ...args);
+        setConnected(true);
+      };
+
+      const onConnectError = (err) => {
+        console.error(`Connection error due to ${err.message}`);
+      };
+
+      const onDisconnect = (reason) => {
+        setConnected(false);
+        console.log(`Disconnected due to ${reason}`);
+      };
+
+      socket.on('connect', onConnect);
+      socket.on('connect_error', onConnectError);
+      socket.on('disconnect', onDisconnect);
+
+      return () => {
+        socket.off('connect', onConnect);
+        socket.off('connect_error', onConnectError);
+        socket.off('disconnect', onDisconnect);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onReconnectError = (err) => {
+        console.log(`Reconnect error due to ${err.message}`);
+      };
+
+      socket.io.on('reconnect_error', onReconnectError);
+
+      const onError = (err) => {
+        console.log(`General socket error with message ${err.message}`);
+      };
+      socket.io.on('error', onError);
+
+      const onReconnect = (attempt) => {
+        console.log(`Reconnected after ${attempt} attempt(s)`);
+      };
+      socket.io.on('reconnect', onReconnect);
+
+      const disconnectOnBeforeUnload = () => {
+        console.log('Disconnecting due to beforeunload event...');
+        socket.disconnect();
+        setSocket(null);
+      };
+      window.addEventListener('beforeunload', disconnectOnBeforeUnload);
+
+      return () => {
+        socket.io.off('reconnect_error', onReconnectError);
+        socket.io.off('error', onError);
+        socket.io.off('reconnect', onReconnect);
+        window.removeEventListener('beforeunload', disconnectOnBeforeUnload);
+      };
+    }
+  }, [clientID, setAppResetKey, socket]);
+
+  /**
+   * Wait to show the disconnected screen on initial app load
+   */
+  useEffect(() => {
+    window.setTimeout(() => {
+      setConnected((prev) => {
+        if (prev === null) {
+          return false;
+        }
+        return prev;
+      });
+    }, INITIAL_DISCONNECT_SCREEN_DELAY);
+  }, []);
+
+  return (
+    <SocketContext.Provider value={socketObject}>
+      {children}
+
+      <Backdrop
+        open={connected === false && willAttemptReconnect === true}
+        sx={{
+          color: '#fff',
+          zIndex: (theme) => theme.zIndex.drawer + 1,
+        }}>
+        <div
+          style={{
+            alignItems: 'center',
+            flexDirection: 'column',
+            textAlign: 'center',
+          }}>
+          <CircularProgress color="inherit" />
+          <Typography
+            align="center"
+            fontSize={{sm: 18, xs: 16}}
+            sx={{
+              fontFamily:
+                'ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace',
+              fontWeight: 'bold',
+            }}>
+            {'Disconnected. Attempting to reconnect...'}
+          </Typography>
+        </div>
+      </Backdrop>
+    </SocketContext.Provider>
+  );
+}

streaming-react-app/src/StreamingInterface.css

@@ -0,0 +1,66 @@
+.app-wrapper-sra {
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+}
+
+.main-container-sra {
+  background-color: white;
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+  text-align: left;
+  margin: 16px;
+  margin-bottom: 36px;
+  border-radius: 8px;
+  box-shadow: 0px 24px 30px rgba(0, 0, 0, 0.3);
+  border: 1px solid rgba(0, 0, 0, 0.05);
+  min-height: 300px;
+  /* max-height: 95vh; */
+  /* max-width: 625px; */
+  /* min-width: 580px; */
+  overflow: hidden;
+}
+
+.top-section-sra {
+  padding-top: 24px;
+  margin-bottom: 24px;
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+}
+
+.horizontal-padding-sra {
+  padding-left: 20px;
+  padding-right: 20px;
+}
+
+.header-container-sra {
+  display: flex;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: center;
+  margin-bottom: 24px;
+}
+
+.header-icon-sra {
+  display: block;
+  margin-right: 12px;
+}
+
+.translation-text-container-sra {
+  background-color: #f8f8f8;
+  /* flex-grow: 1; make it expand to fill the available space */
+  padding-top: 12px;
+  padding-bottom: 4px;
+}
+
+.translation-text-sra {
+  /* overflow-y: scroll; */
+  /* max-height: 500px; */
+}
+
+.text-chunk-sra {
+  margin-bottom: 12px;
+}

streaming-react-app/src/StreamingInterface.tsx

@@ -0,0 +1,1149 @@
+import {useCallback, useEffect, useLayoutEffect, useRef, useState} from 'react';
+import Button from '@mui/material/Button';
+import Typography from '@mui/material/Typography';
+import InputLabel from '@mui/material/InputLabel';
+import FormControl from '@mui/material/FormControl';
+import Select, {SelectChangeEvent} from '@mui/material/Select';
+import MenuItem from '@mui/material/MenuItem';
+import Stack from '@mui/material/Stack';
+import seamlessLogoUrl from './assets/seamless.svg';
+import {
+  AgentCapabilities,
+  BaseResponse,
+  BrowserAudioStreamConfig,
+  DynamicConfig,
+  PartialDynamicConfig,
+  SUPPORTED_INPUT_SOURCES,
+  SUPPORTED_OUTPUT_MODES,
+  ServerExceptionData,
+  ServerSpeechData,
+  ServerState,
+  ServerTextData,
+  StartStreamEventConfig,
+  StreamingStatus,
+  SupportedInputSource,
+  SupportedOutputMode,
+  TranslationSentences,
+} from './types/StreamingTypes';
+import FormLabel from '@mui/material/FormLabel';
+import RadioGroup from '@mui/material/RadioGroup';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import Radio from '@mui/material/Radio';
+import './StreamingInterface.css';
+import RoomConfig from './RoomConfig';
+import Divider from '@mui/material/Divider';
+import {useSocket} from './useSocket';
+import {RoomState} from './types/RoomState';
+import useStable from './useStable';
+import float32To16BitPCM from './float32To16BitPCM';
+import createBufferedSpeechPlayer from './createBufferedSpeechPlayer';
+import Checkbox from '@mui/material/Checkbox';
+import Alert from '@mui/material/Alert';
+import ISO6391 from 'iso-639-1';
+import isScrolledToDocumentBottom from './isScrolledToDocumentBottom';
+import Box from '@mui/material/Box';
+import Slider from '@mui/material/Slider';
+import VolumeDown from '@mui/icons-material/VolumeDown';
+import VolumeUp from '@mui/icons-material/VolumeUp';
+import Mic from '@mui/icons-material/Mic';
+import MicOff from '@mui/icons-material/MicOff';
+import XRDialog from './react-xr/XRDialog';
+import getTranslationSentencesFromReceivedData from './getTranslationSentencesFromReceivedData';
+import {
+  sliceTranslationSentencesUpToIndex,
+  getTotalSentencesLength,
+} from './sliceTranslationSentencesUtils';
+import Blink from './Blink';
+import {CURSOR_BLINK_INTERVAL_MS} from './cursorBlinkInterval';
+import {getURLParams} from './URLParams';
+import debug from './debug';
+import DebugSection from './DebugSection';
+import Switch from '@mui/material/Switch';
+
+const AUDIO_STREAM_DEFAULTS: {
+  [key in SupportedInputSource]: BrowserAudioStreamConfig;
+} = {
+  userMedia: {
+    noiseSuppression: true,
+    echoCancellation: false,
+  },
+  displayMedia: {
+    noiseSuppression: false,
+    echoCancellation: false,
+  },
+};
+
+async function requestUserMediaAudioStream(
+  config: BrowserAudioStreamConfig = {
+    noiseSuppression: true,
+    echoCancellation: false,
+  },
+) {
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: {...config, channelCount: 1},
+  });
+  console.debug(
+    '[requestUserMediaAudioStream] stream created with settings:',
+    stream.getAudioTracks()?.[0]?.getSettings(),
+  );
+  return stream;
+}
+
+async function requestDisplayMediaAudioStream(
+  config: BrowserAudioStreamConfig = {
+    noiseSuppression: false,
+    echoCancellation: false,
+  },
+) {
+  const stream = await navigator.mediaDevices.getDisplayMedia({
+    audio: {...config, channelCount: 1},
+    // selfBrowserSurface: false, // don't allow the user to select the current tab as the source
+  });
+  console.debug(
+    '[requestDisplayMediaAudioStream] stream created with settings:',
+    stream.getAudioTracks()?.[0]?.getSettings(),
+  );
+  return stream;
+}
+
+const buttonLabelMap: {[key in StreamingStatus]: string} = {
+  stopped: 'Start Streaming',
+  running: 'Stop Streaming',
+  starting: 'Starting...',
+};
+
+const BUFFER_LIMIT = 1;
+
+const SCROLLED_TO_BOTTOM_THRESHOLD_PX = 36;
+
+const GAIN_MULTIPLIER_OVER_1 = 3;
+
+const getGainScaledValue = (value) =>
+  value > 1 ? (value - 1) * GAIN_MULTIPLIER_OVER_1 + 1 : value;
+
+const TOTAL_ACTIVE_TRANSCODER_WARNING_THRESHOLD = 2;
+
+const MAX_SERVER_EXCEPTIONS_TRACKED = 500;
+
+export const TYPING_ANIMATION_DELAY_MS = 6;
+
+export default function StreamingInterface() {
+  const urlParams = getURLParams();
+  const debugParam = urlParams.debug;
+  const animateTextDisplay = urlParams.animateTextDisplay;
+
+  const socketObject = useSocket();
+  const {socket, clientID} = socketObject;
+
+  const [serverState, setServerState] = useState<ServerState | null>(null);
+  const [agent, setAgent] = useState<AgentCapabilities | null>(null);
+  const model = agent?.name ?? null;
+  const agentsCapabilities: Array<AgentCapabilities> =
+    serverState?.agentsCapabilities ?? [];
+  const currentAgent: AgentCapabilities | null =
+    agentsCapabilities.find((agent) => agent.name === model) ?? null;
+
+  const [serverExceptions, setServerExceptions] = useState<
+    Array<ServerExceptionData>
+  >([]);
+  const [connectionError, setConnectionError] = useState<string | null>(null);
+  const [roomState, setRoomState] = useState<RoomState | null>(null);
+  const roomID = roomState?.room_id ?? null;
+  const isSpeaker =
+    (clientID != null && roomState?.speakers.includes(clientID)) ?? false;
+  const isListener =
+    (clientID != null && roomState?.listeners.includes(clientID)) ?? false;
+
+  const [streamingStatus, setStreamingStatus] =
+    useState<StreamingStatus>('stopped');
+
+  const isStreamConfiguredRef = useRef<boolean>(false);
+
+  const [outputMode, setOutputMode] = useState<SupportedOutputMode>('s2s&t');
+  const [inputSource, setInputSource] =
+    useState<SupportedInputSource>('userMedia');
+  const [enableNoiseSuppression, setEnableNoiseSuppression] = useState<
+    boolean | null
+  >(null);
+
+  // Dynamic Params:
+  const [targetLang, setTargetLang] = useState<string | null>(null);
+  const [enableExpressive, setEnableExpressive] = useState<boolean | null>(
+    null,
+  );
+
+  const [serverDebugFlag, setServerDebugFlag] = useState<boolean>(
+    debugParam ?? false,
+  );
+
+  const [receivedData, setReceivedData] = useState<Array<ServerTextData>>([]);
+  // const [translationSentencesAnimated, setTranslationSentencesAnimated] =
+  //   useState<TranslationSentences>([]);
+  const [
+    translationSentencesAnimatedIndex,
+    setTranslationSentencesAnimatedIndex,
+  ] = useState<number>(0);
+
+  const lastTranslationResultRef = useRef<HTMLDivElement | null>(null);
+
+  const [inputStream, setInputStream] = useState<MediaStream | null>(null);
+  const [inputStreamSource, setInputStreamSource] =
+    useState<MediaStreamAudioSourceNode | null>(null);
+  const audioContext = useStable<AudioContext>(() => new AudioContext());
+  const [scriptNodeProcessor, setScriptNodeProcessor] =
+    useState<ScriptProcessorNode | null>(null);
+
+  const [muted, setMuted] = useState<boolean>(false);
+  // The onaudioprocess script needs an up-to-date reference to the muted state, so
+  // we use a ref here and keep it in sync via useEffect
+  const mutedRef = useRef<boolean>(muted);
+  useEffect(() => {
+    mutedRef.current = muted;
+  }, [muted]);
+
+  const [gain, setGain] = useState<number>(1);
+
+  const isScrolledToBottomRef = useRef<boolean>(isScrolledToDocumentBottom());
+
+  // Some config options must be set when starting streaming and cannot be chaned dynamically.
+  // This controls whether they are disabled or not
+  const streamFixedConfigOptionsDisabled =
+    streamingStatus !== 'stopped' || roomID == null;
+
+  const bufferedSpeechPlayer = useStable(() => {
+    const player = createBufferedSpeechPlayer({
+      onStarted: () => {
+        console.debug('📢 PLAYBACK STARTED 📢');
+      },
+      onEnded: () => {
+        console.debug('🛑 PLAYBACK ENDED 🛑');
+      },
+    });
+
+    // Start the player now so it eagerly plays audio when it arrives
+    player.start();
+    return player;
+  });
+
+  const translationSentencesBase: TranslationSentences =
+    getTranslationSentencesFromReceivedData(receivedData);
+
+  const translationSentencesBaseTotalLength = getTotalSentencesLength(
+    translationSentencesBase,
+  );
+
+  const translationSentences: TranslationSentences = animateTextDisplay
+    ? sliceTranslationSentencesUpToIndex(
+        translationSentencesBase,
+        translationSentencesAnimatedIndex,
+      )
+    : translationSentencesBase;
+
+  // We want the blinking cursor to show before any text has arrived, so let's add an empty string so that the cursor shows up
+  const translationSentencesWithEmptyStartingString =
+    streamingStatus === 'running' && translationSentences.length === 0
+      ? ['']
+      : translationSentences;
+
+  /******************************************
+   * Event Handlers
+   ******************************************/
+
+  const setAgentAndUpdateParams = useCallback(
+    (newAgent: AgentCapabilities | null) => {
+      setAgent((prevAgent) => {
+        if (prevAgent?.name !== newAgent?.name) {
+          setTargetLang(newAgent?.targetLangs[0] ?? null);
+          setEnableExpressive(null);
+          // setOutputMode(newAgent.modalities[0]);
+        }
+        return newAgent;
+      });
+    },
+    [],
+  );
+
+  const onSetDynamicConfig = useCallback(
+    async (partialConfig: PartialDynamicConfig) => {
+      return new Promise<void>((resolve, reject) => {
+        if (socket == null) {
+          reject(new Error('[onSetDynamicConfig] socket is null '));
+          return;
+        }
+
+        socket.emit(
+          'set_dynamic_config',
+          partialConfig,
+          (result: BaseResponse) => {
+            console.log('[emit result: set_dynamic_config]', result);
+            if (result.status === 'ok') {
+              resolve();
+            } else {
+              reject();
+            }
+          },
+        );
+      });
+    },
+    [socket],
+  );
+
+  const configureStreamAsync = ({sampleRate}: {sampleRate: number}) => {
+    return new Promise<void>((resolve, reject) => {
+      if (socket == null) {
+        reject(new Error('[configureStreamAsync] socket is null '));
+        return;
+      }
+      const modelName = agent?.name ?? null;
+      if (modelName == null) {
+        reject(new Error('[configureStreamAsync] modelName is null '));
+        return;
+      }
+
+      const config: StartStreamEventConfig = {
+        event: 'config',
+        rate: sampleRate,
+        model_name: modelName,
+        // source_language: inputLang,
+        debug: serverDebugFlag,
+        // synchronous processing isn't implemented on the v2 pubsub server, so hardcode this to true
+        async_processing: true,
+        buffer_limit: BUFFER_LIMIT,
+        model_type: outputMode,
+      };
+
+      console.log('[configureStreamAsync] sending config', config);
+
+      socket.emit('configure_stream', config, (statusObject) => {
+        if (statusObject.status === 'ok') {
+          isStreamConfiguredRef.current = true;
+          console.debug(
+            '[configureStreamAsync] stream configured!',
+            statusObject,
+          );
+          resolve();
+        } else {
+          isStreamConfiguredRef.current = false;
+          reject(
+            new Error(
+              `[configureStreamAsync] configure_stream returned status: ${statusObject.status}`,
+            ),
+          );
+          return;
+        }
+      });
+    });
+  };
+
+  const startStreaming = async () => {
+    if (streamingStatus !== 'stopped') {
+      console.warn(
+        `Attempting to start stream when status is ${streamingStatus}`,
+      );
+      return;
+    }
+
+    setStreamingStatus('starting');
+
+    if (audioContext.state === 'suspended') {
+      console.warn('audioContext was suspended! resuming...');
+      await audioContext.resume();
+    }
+
+    let stream: MediaStream | null = null;
+
+    try {
+      if (inputSource === 'userMedia') {
+        stream = await requestUserMediaAudioStream({
+          noiseSuppression:
+            enableNoiseSuppression ??
+            AUDIO_STREAM_DEFAULTS['userMedia'].noiseSuppression,
+          echoCancellation: false,
+        });
+      } else if (inputSource === 'displayMedia') {
+        stream = await requestDisplayMediaAudioStream({
+          noiseSuppression:
+            enableNoiseSuppression ??
+            AUDIO_STREAM_DEFAULTS['displayMedia'].noiseSuppression,
+          echoCancellation: false,
+        });
+      } else {
+        throw new Error(`Unsupported input source requested: ${inputSource}`);
+      }
+      setInputStream(stream);
+    } catch (e) {
+      console.error('[startStreaming] media stream request failed:', e);
+      setStreamingStatus('stopped');
+      return;
+    }
+
+    const mediaStreamSource = audioContext.createMediaStreamSource(stream);
+    setInputStreamSource(mediaStreamSource);
+    /**
+     * NOTE: This currently uses a deprecated way of processing the audio (createScriptProcessor).
+     *
+     * Documentation for the deprecated way of doing it is here: https://developer.mozilla.org/en-US/docs/Web/API/BaseAudioContext/createScriptProcessor
+     *
+     * This should be migrated to something like this SO answer: https://stackoverflow.com/a/65448287
+     */
+    const scriptProcessor = audioContext.createScriptProcessor(16384, 1, 1);
+    setScriptNodeProcessor(scriptProcessor);
+
+    scriptProcessor.onaudioprocess = (event) => {
+      if (isStreamConfiguredRef.current === false) {
+        console.debug('[onaudioprocess] stream is not configured yet!');
+        return;
+      }
+      if (socket == null) {
+        console.warn('[onaudioprocess] socket is null in onaudioprocess');
+        return;
+      }
+      // console.debug('[onaudioprocess] event', event);
+
+      if (mutedRef.current) {
+        // We still want to send audio to the server when we're muted to ensure we
+        // get any remaining audio back from the server, so let's pass an array length 1 with a value of 0
+        const mostlyEmptyInt16Array = new Int16Array(1);
+        socket.emit('incoming_audio', mostlyEmptyInt16Array);
+      } else {
+        const float32Audio = event.inputBuffer.getChannelData(0);
+        const pcm16Audio = float32To16BitPCM(float32Audio);
+        socket.emit('incoming_audio', pcm16Audio);
+      }
+
+      debug()?.sentAudio(event);
+    };
+
+    mediaStreamSource.connect(scriptProcessor);
+    scriptProcessor.connect(audioContext.destination);
+
+    bufferedSpeechPlayer.start();
+
+    try {
+      if (targetLang == null) {
+        throw new Error('[startStreaming] targetLang cannot be nullish');
+      }
+
+      // When we are starting the stream we want to pass all the dynamic config values
+      // available before actually configuring and starting the stream
+      const fullDynamicConfig: DynamicConfig = {
+        targetLanguage: targetLang,
+        expressive: enableExpressive,
+      };
+
+      await onSetDynamicConfig(fullDynamicConfig);
+
+      // NOTE: this needs to be the *audioContext* sample rate, not the sample rate of the input stream. Not entirely sure why.
+      await configureStreamAsync({
+        sampleRate: audioContext.sampleRate,
+      });
+    } catch (e) {
+      console.error('configureStreamAsync failed', e);
+      setStreamingStatus('stopped');
+      return;
+    }
+
+    setStreamingStatus('running');
+  };
+
+  const stopStreaming = useCallback(async () => {
+    if (streamingStatus === 'stopped') {
+      console.warn(
+        `Attempting to stop stream when status is ${streamingStatus}`,
+      );
+      return;
+    }
+
+    // Stop the speech playback right away
+    bufferedSpeechPlayer.stop();
+
+    if (inputStreamSource == null || scriptNodeProcessor == null) {
+      console.error(
+        'inputStreamSource || scriptNodeProcessor is null in stopStreaming',
+      );
+    } else {
+      inputStreamSource.disconnect(scriptNodeProcessor);
+      scriptNodeProcessor.disconnect(audioContext.destination);
+
+      // From: https://stackoverflow.com/questions/65447236/scriptnode-onaudioprocess-is-deprecated-any-alternative
+      // do we also need this??
+      // recorder?.stop();
+
+      // Release the mic input so we stop showing the red recording icon in the browser
+      inputStream?.getTracks().forEach((track) => track.stop());
+    }
+
+    if (socket == null) {
+      console.warn('Unable to emit stop_stream because socket is null');
+    } else {
+      socket.emit('stop_stream', (result) => {
+        console.debug('[emit result: stop_stream]', result);
+      });
+    }
+
+    setStreamingStatus('stopped');
+  }, [
+    audioContext.destination,
+    bufferedSpeechPlayer,
+    inputStream,
+    inputStreamSource,
+    scriptNodeProcessor,
+    socket,
+    streamingStatus,
+  ]);
+
+  const onClearTranscriptForAll = useCallback(() => {
+    if (socket != null) {
+      socket.emit('clear_transcript_for_all');
+    }
+  }, [socket]);
+
+  /******************************************
+   * Effects
+   ******************************************/
+
+  useEffect(() => {
+    if (socket == null) {
+      return;
+    }
+
+    const onRoomStateUpdate = (roomState: RoomState) => {
+      // console.log('[event: room_state_update]', roomState);
+      setRoomState(roomState);
+    };
+
+    socket.on('room_state_update', onRoomStateUpdate);
+
+    return () => {
+      socket.off('room_state_update', onRoomStateUpdate);
+    };
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onTranslationText = (data: ServerTextData) => {
+        setReceivedData((prev) => [...prev, data]);
+        debug()?.receivedText(data.payload);
+      };
+
+      const onTranslationSpeech = (data: ServerSpeechData) => {
+        bufferedSpeechPlayer.addAudioToBuffer(data.payload, data.sample_rate);
+      };
+
+      socket.on('translation_text', onTranslationText);
+      socket.on('translation_speech', onTranslationSpeech);
+
+      return () => {
+        socket.off('translation_text', onTranslationText);
+        socket.off('translation_speech', onTranslationSpeech);
+      };
+    }
+  }, [bufferedSpeechPlayer, socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onServerStateUpdate = (newServerState: ServerState) => {
+        setServerState(newServerState);
+
+        // If a client creates a server lock, we want to stop streaming if we're not them
+        if (
+          newServerState.serverLock?.isActive === true &&
+          newServerState.serverLock?.clientID !== clientID &&
+          streamingStatus === 'running'
+        ) {
+          stopStreaming();
+        }
+
+        const firstAgentNullable = newServerState.agentsCapabilities[0];
+        if (agent == null && firstAgentNullable != null) {
+          setAgentAndUpdateParams(firstAgentNullable);
+        }
+      };
+
+      socket.on('server_state_update', onServerStateUpdate);
+
+      return () => {
+        socket.off('server_state_update', onServerStateUpdate);
+      };
+    }
+  }, [
+    agent,
+    clientID,
+    setAgentAndUpdateParams,
+    socket,
+    stopStreaming,
+    streamingStatus,
+  ]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onServerException = (
+        exceptionDataWithoutClientTime: ServerExceptionData,
+      ) => {
+        const exceptionData = {
+          ...exceptionDataWithoutClientTime,
+          timeStringClient: new Date(
+            exceptionDataWithoutClientTime['timeEpochMs'],
+          ).toLocaleString(),
+        };
+
+        setServerExceptions((prev) =>
+          [exceptionData, ...prev].slice(0, MAX_SERVER_EXCEPTIONS_TRACKED),
+        );
+        console.error(
+          `[server_exception] The server encountered an exception: ${exceptionData['message']}`,
+          exceptionData,
+        );
+      };
+
+      socket.on('server_exception', onServerException);
+
+      return () => {
+        socket.off('server_exception', onServerException);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onClearTranscript = () => {
+        setReceivedData([]);
+        setTranslationSentencesAnimatedIndex(0);
+      };
+
+      socket.on('clear_transcript', onClearTranscript);
+
+      return () => {
+        socket.off('clear_transcript', onClearTranscript);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    const onScroll = () => {
+      if (isScrolledToDocumentBottom(SCROLLED_TO_BOTTOM_THRESHOLD_PX)) {
+        // console.debug('scrolled to bottom!');
+        isScrolledToBottomRef.current = true;
+        return;
+      }
+      // console.debug('NOT scrolled to bottom!');
+      isScrolledToBottomRef.current = false;
+      return;
+    };
+
+    document.addEventListener('scroll', onScroll);
+
+    return () => {
+      document.removeEventListener('scroll', onScroll);
+    };
+  }, []);
+
+  useLayoutEffect(() => {
+    if (
+      lastTranslationResultRef.current != null &&
+      isScrolledToBottomRef.current
+    ) {
+      // Scroll the div to the most recent entry
+      lastTranslationResultRef.current.scrollIntoView();
+    }
+    // Run the effect every time data is received, so that
+    // we scroll to the bottom even if we're just adding text to
+    // a pre-existing chunk
+  }, [receivedData]);
+
+  useEffect(() => {
+    if (!animateTextDisplay) {
+      return;
+    }
+
+    if (
+      translationSentencesAnimatedIndex < translationSentencesBaseTotalLength
+    ) {
+      const timeout = setTimeout(() => {
+        setTranslationSentencesAnimatedIndex((prev) => prev + 1);
+        debug()?.startRenderText();
+      }, TYPING_ANIMATION_DELAY_MS);
+
+      return () => clearTimeout(timeout);
+    } else {
+      debug()?.endRenderText();
+    }
+  }, [
+    animateTextDisplay,
+    translationSentencesAnimatedIndex,
+    translationSentencesBaseTotalLength,
+  ]);
+
+  /******************************************
+   * Sub-components
+   ******************************************/
+
+  const volumeSliderNode = (
+    <Stack
+      spacing={2}
+      direction="row"
+      sx={{mb: 1, width: '100%'}}
+      alignItems="center">
+      <VolumeDown color="primary" />
+      <Slider
+        aria-label="Volume"
+        defaultValue={1}
+        scale={getGainScaledValue}
+        min={0}
+        max={3}
+        step={0.1}
+        marks={[
+          {value: 0, label: '0%'},
+          {value: 1, label: '100%'},
+          {value: 2, label: '400%'},
+          {value: 3, label: '700%'},
+        ]}
+        valueLabelFormat={(value) => `${(value * 100).toFixed(0)}%`}
+        valueLabelDisplay="auto"
+        value={gain}
+        onChange={(_event: Event, newValue: number | number[]) => {
+          // console.log({event, newValue});
+          if (typeof newValue === 'number') {
+            const scaledGain = getGainScaledValue(newValue);
+            // We want the actual gain node to use the scaled value
+            bufferedSpeechPlayer.setGain(scaledGain);
+            // But we want react state to keep track of the non-scaled value
+            setGain(newValue);
+          } else {
+            console.error(
+              `[volume slider] Unexpected non-number value: ${newValue}`,
+            );
+          }
+        }}
+      />
+      <VolumeUp color="primary" />
+    </Stack>
+  );
+
+  const xrDialogComponent = (
+    <XRDialog
+      animateTextDisplay={
+        animateTextDisplay &&
+        translationSentencesAnimatedIndex == translationSentencesBaseTotalLength
+      }
+      bufferedSpeechPlayer={bufferedSpeechPlayer}
+      translationSentences={translationSentences}
+      roomState={roomState}
+      roomID={roomID}
+      startStreaming={startStreaming}
+      stopStreaming={stopStreaming}
+      debugParam={debugParam}
+    />
+  );
+
+  return (
+    <div className="app-wrapper-sra">
+      <Box
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore Not sure why it's complaining about complexity here
+        minWidth={'550px'}
+        maxWidth={'660px'}>
+        <div className="main-container-sra">
+          <div className="top-section-sra horizontal-padding-sra">
+            <div className="header-container-sra">
+              <img
+                src={seamlessLogoUrl}
+                className="header-icon-sra"
+                alt="Seamless Translation Logo"
+                height={24}
+                width={24}
+              />
+
+              <div>
+                <Typography variant="h1" sx={{color: '#65676B'}}>
+                  Seamless Translation
+                </Typography>
+              </div>
+            </div>
+
+            <Stack spacing="22px" direction="column">
+              <Box>
+                <RoomConfig
+                  roomState={roomState}
+                  serverState={serverState}
+                  streamingStatus={streamingStatus}
+                  onJoinRoomOrUpdateRoles={() => {
+                    // If the user has switched from speaker to listener we need to tell the
+                    // player to play eagerly, since currently the listener doesn't have any stop/start controls
+                    bufferedSpeechPlayer.start();
+                  }}
+                />
+
+                {isListener && !isSpeaker && (
+                  <Box
+                    sx={{
+                      paddingX: 6,
+                      paddingBottom: 2,
+                      marginY: 2,
+                      display: 'flex',
+                      flexDirection: 'column',
+                      alignItems: 'center',
+                    }}>
+                    {volumeSliderNode}
+                  </Box>
+                )}
+              </Box>
+
+              {isSpeaker && (
+                <>
+                  <Divider />
+
+                  <Stack spacing="12px" direction="column">
+                    <FormLabel id="output-modes-radio-group-label">
+                      Model
+                    </FormLabel>
+                    <FormControl
+                      disabled={
+                        streamFixedConfigOptionsDisabled ||
+                        agentsCapabilities.length === 0
+                      }
+                      fullWidth
+                      sx={{minWidth: '14em'}}>
+                      <InputLabel id="model-selector-input-label">
+                        Model
+                      </InputLabel>
+                      <Select
+                        labelId="model-selector-input-label"
+                        label="Model"
+                        onChange={(e: SelectChangeEvent) => {
+                          const newAgent =
+                            agentsCapabilities.find(
+                              (agent) => e.target.value === agent.name,
+                            ) ?? null;
+                          if (newAgent == null) {
+                            console.error(
+                              'Unable to find agent with name',
+                              e.target.value,
+                            );
+                          }
+                          setAgentAndUpdateParams(newAgent);
+                        }}
+                        value={model ?? ''}>
+                        {agentsCapabilities.map((agent) => (
+                          <MenuItem value={agent.name} key={agent.name}>
+                            {agent.name}
+                          </MenuItem>
+                        ))}
+                      </Select>
+                    </FormControl>
+
+                    <Typography variant="body2">
+                      {`Supported Source Languages: ${
+                        currentAgent?.sourceLangs.join(', ') ?? 'None'
+                      }`}
+                    </Typography>
+                  </Stack>
+
+                  <Stack spacing={0.5}>
+                    <FormLabel id="output-modes-radio-group-label">
+                      Output
+                    </FormLabel>
+
+                    <Box sx={{paddingTop: 2, paddingBottom: 1}}>
+                      <FormControl fullWidth sx={{minWidth: '14em'}}>
+                        <InputLabel id="target-selector-input-label">
+                          Target Language
+                        </InputLabel>
+                        <Select
+                          labelId="target-selector-input-label"
+                          label="Target Language"
+                          onChange={(e: SelectChangeEvent) => {
+                            setTargetLang(e.target.value);
+                            onSetDynamicConfig({
+                              targetLanguage: e.target.value,
+                            });
+                          }}
+                          value={targetLang ?? ''}>
+                          {currentAgent?.targetLangs.map((langCode) => (
+                            <MenuItem value={langCode} key={langCode}>
+                              {`${ISO6391.getName(langCode)} (${langCode})`}
+                            </MenuItem>
+                          ))}
+                        </Select>
+                      </FormControl>
+                    </Box>
+
+                    <Stack direction="row" spacing={3.5}>
+                      <FormControl disabled={streamFixedConfigOptionsDisabled}>
+                        <RadioGroup
+                          aria-labelledby="output-modes-radio-group-label"
+                          value={outputMode}
+                          onChange={(e) =>
+                            setOutputMode(e.target.value as SupportedOutputMode)
+                          }
+                          name="output-modes-radio-buttons-group">
+                          {
+                            // TODO: Use supported modalities from agentCapabilities
+                            SUPPORTED_OUTPUT_MODES.map(({value, label}) => (
+                              <FormControlLabel
+                                key={value}
+                                value={value}
+                                control={<Radio />}
+                                label={label}
+                              />
+                            ))
+                          }
+                        </RadioGroup>
+                      </FormControl>
+
+                      <Stack
+                        direction="column"
+                        spacing={1}
+                        alignItems="flex-start"
+                        sx={{flexGrow: 1}}>
+                        {currentAgent?.dynamicParams?.includes(
+                          'expressive',
+                        ) && (
+                          <FormControlLabel
+                            control={
+                              <Switch
+                                checked={enableExpressive ?? false}
+                                onChange={(
+                                  event: React.ChangeEvent<HTMLInputElement>,
+                                ) => {
+                                  const newValue = event.target.checked;
+                                  setEnableExpressive(newValue);
+                                  onSetDynamicConfig({expressive: newValue});
+                                }}
+                              />
+                            }
+                            label="Expressive"
+                          />
+                        )}
+
+                        {isListener && (
+                          <Box
+                            sx={{
+                              flexGrow: 1,
+                              paddingX: 1.5,
+                              paddingY: 1.5,
+                              width: '100%',
+                            }}>
+                            {volumeSliderNode}
+                          </Box>
+                        )}
+                      </Stack>
+                    </Stack>
+                  </Stack>
+
+                  <Stack
+                    direction="row"
+                    spacing={2}
+                    justifyContent="space-between">
+                    <div>
+                      <FormControl disabled={streamFixedConfigOptionsDisabled}>
+                        <FormLabel id="input-source-radio-group-label">
+                          Input Source
+                        </FormLabel>
+                        <RadioGroup
+                          aria-labelledby="input-source-radio-group-label"
+                          value={inputSource}
+                          onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
+                            setInputSource(
+                              e.target.value as SupportedInputSource,
+                            )
+                          }
+                          name="input-source-radio-buttons-group">
+                          {SUPPORTED_INPUT_SOURCES.map(({label, value}) => (
+                            <FormControlLabel
+                              key={value}
+                              value={value}
+                              control={<Radio />}
+                              label={label}
+                            />
+                          ))}
+                        </RadioGroup>
+                      </FormControl>
+                    </div>
+                    <div>
+                      <FormControl disabled={streamFixedConfigOptionsDisabled}>
+                        <FormLabel>Options</FormLabel>
+                        <FormControlLabel
+                          control={
+                            <Checkbox
+                              checked={
+                                enableNoiseSuppression ??
+                                AUDIO_STREAM_DEFAULTS[inputSource]
+                                  .noiseSuppression
+                              }
+                              onChange={(
+                                event: React.ChangeEvent<HTMLInputElement>,
+                              ) =>
+                                setEnableNoiseSuppression(event.target.checked)
+                              }
+                            />
+                          }
+                          label="Noise Suppression (Browser)"
+                        />
+                        <FormControlLabel
+                          control={
+                            <Checkbox
+                              checked={serverDebugFlag}
+                              onChange={(
+                                event: React.ChangeEvent<HTMLInputElement>,
+                              ) => setServerDebugFlag(event.target.checked)}
+                            />
+                          }
+                          label="Server Debug Flag"
+                        />
+                      </FormControl>
+                    </div>
+                  </Stack>
+
+                  <Stack direction="row" spacing={2}>
+                    {streamingStatus === 'stopped' ? (
+                      <Button
+                        variant="contained"
+                        onClick={startStreaming}
+                        disabled={
+                          roomID == null ||
+                          // Prevent users from starting streaming if there is a server lock with an active session
+                          (serverState?.serverLock?.isActive === true &&
+                            serverState.serverLock.clientID !== clientID)
+                        }>
+                        {buttonLabelMap[streamingStatus]}
+                      </Button>
+                    ) : (
+                      <Button
+                        variant="contained"
+                        color={
+                          streamingStatus === 'running' ? 'error' : 'primary'
+                        }
+                        disabled={
+                          streamingStatus === 'starting' || roomID == null
+                        }
+                        onClick={stopStreaming}>
+                        {buttonLabelMap[streamingStatus]}
+                      </Button>
+                    )}
+
+                    <Box>
+                      <Button
+                        variant="contained"
+                        aria-label={muted ? 'Unmute' : 'Mute'}
+                        color={muted ? 'info' : 'primary'}
+                        onClick={() => setMuted((prev) => !prev)}
+                        sx={{
+                          borderRadius: 100,
+                          paddingX: 0,
+                          minWidth: '36px',
+                        }}>
+                        {muted ? <MicOff /> : <Mic />}
+                      </Button>
+                    </Box>
+
+                    {roomID == null ? null : (
+                      <Box
+                        sx={{
+                          flexGrow: 1,
+                          display: 'flex',
+                          justifyContent: 'flex-end',
+                        }}>
+                        {xrDialogComponent}
+                      </Box>
+                    )}
+                  </Stack>
+
+                  {serverExceptions.length > 0 && (
+                    <div>
+                      <Alert severity="error">
+                        {`The server encountered an exception. See the browser console for details. You may need to refresh the page to continue using the app.`}
+                      </Alert>
+                    </div>
+                  )}
+
+                  {serverState != null &&
+                    serverState.totalActiveTranscoders >=
+                      TOTAL_ACTIVE_TRANSCODER_WARNING_THRESHOLD && (
+                      <div>
+                        <Alert severity="warning">
+                          {`The server currently has ${serverState?.totalActiveTranscoders} active streaming sessions. Performance may be degraded.`}
+                        </Alert>
+                      </div>
+                    )}
+
+                  {serverState?.serverLock != null &&
+                    serverState.serverLock.clientID !== clientID && (
+                      <div>
+                        <Alert severity="warning">
+                          {`The server is currently locked by "${serverState.serverLock.name}". Priority will be given to that client when they are streaming, and your streaming session may be halted abruptly.`}
+                        </Alert>
+                      </div>
+                    )}
+                </>
+              )}
+            </Stack>
+
+            {isListener && !isSpeaker && (
+              <Box sx={{marginBottom: 1, marginTop: 2}}>
+                {xrDialogComponent}
+              </Box>
+            )}
+          </div>
+
+          {debugParam && roomID != null && <DebugSection />}
+
+          <div className="translation-text-container-sra horizontal-padding-sra">
+            <Stack
+              direction="row"
+              spacing={2}
+              sx={{mb: '16px', alignItems: 'center'}}>
+              <Typography variant="h1" sx={{fontWeight: 700, flexGrow: 1}}>
+                Transcript
+              </Typography>
+              {isSpeaker && (
+                <Button
+                  variant="text"
+                  size="small"
+                  onClick={onClearTranscriptForAll}>
+                  Clear Transcript for All
+                </Button>
+              )}
+            </Stack>
+            <Stack direction="row">
+              <div className="translation-text-sra">
+                {translationSentencesWithEmptyStartingString.map(
+                  (sentence, index, arr) => {
+                    const isLast = index === arr.length - 1;
+                    const maybeRef = isLast
+                      ? {ref: lastTranslationResultRef}
+                      : {};
+                    return (
+                      <div className="text-chunk-sra" key={index} {...maybeRef}>
+                        <Typography variant="body1">
+                          {sentence}
+                          {animateTextDisplay && isLast && (
+                            <Blink
+                              intervalMs={CURSOR_BLINK_INTERVAL_MS}
+                              shouldBlink={
+                                (roomState?.activeTranscoders ?? 0) > 0
+                              }>
+                              <Typography
+                                component="span"
+                                variant="body1"
+                                sx={{
+                                  display: 'inline-block',
+                                  transform: 'scaleY(1.25) translateY(-1px)',
+                                }}>
+                                {'|'}
+                              </Typography>
+                            </Blink>
+                          )}
+                        </Typography>
+                      </div>
+                    );
+                  },
+                )}
+              </div>
+            </Stack>
+          </div>
+        </div>
+      </Box>
+    </div>
+  );
+}

streaming-react-app/src/URLParams.ts

@@ -0,0 +1,50 @@
+import {getBooleanParamFlag, getStringParamFlag} from './getParamFlag';
+import {URLParamsObject} from './types/URLParamsTypes';
+
+/**
+ * These are the URL parameters you can provide to the app to change its behavior.
+ *
+ * Boolean flags can be set by just providing the flag name (`?autoJoin`), or by
+ * explicitly setting it to 1 (true) or 0 (false): `?autoJoin=1` or `?autoJoin=0`
+ *
+ * String flags require an explicit value: `?roomID=ABCD`
+ *
+ * Examples:
+ *
+ * - `http://localhost:5173/?roomID=BBCD&autoJoin&debug`
+ * - `http://localhost:5173/?serverURL=localhost:8000`
+
+ * @returns
+ */
+
+export function getURLParams(): URLParamsObject {
+  return {
+    // animate the translation text when it arrives, typing it out one letter at a time
+    animateTextDisplay: getBooleanParamFlag('animateTextDisplay', true), // default to true;
+
+    // automatically join the room when the app loads. requires roomID to be set via url param as well
+    autoJoin: getBooleanParamFlag('autoJoin', false),
+
+    // automatically check the server debug flag as true
+    debug: getBooleanParamFlag('debug', false),
+
+    // Enable UI on the client that allows locking out other users of the server when it's being used for high profile demos
+    // NOTE: There is an escape hatch for disabling a server lock by setting the name field to remove_server_lock
+    enableServerLock: getBooleanParamFlag('enableServerLock', false),
+
+    // Pre-populate the Room Code field with the provided roomID. Can be used in conjunction with autoJoin to jump straight into the room
+    roomID: getStringParamFlag('roomID'),
+
+    // Use an alternate server URL as the streaming server (useful for pointing to dev servers: http://localhost:5173/?serverURL=localhost:8000)
+    serverURL: getStringParamFlag('serverURL'),
+
+    // Skip the popup dialog that displays within VR, which is mostly redundant with the web based dialog
+    skipARIntro: getBooleanParamFlag('skipARIntro', true), // default to true
+
+    // Shows the translation text in AR in front of an opaque panel covering all the text area
+    // single_block = original single text block with background
+    // lines = each line is a separate block and animates
+    // lines_with_background = adds a panel behind lines
+    ARTranscriptionType: getStringParamFlag('ARTranscriptionType') || 'lines',
+  };
+}

streaming-react-app/src/createBufferedSpeechPlayer.ts

@@ -0,0 +1,177 @@
+import debug from './debug';
+
+type AddAudioToBufferFunction = (
+  samples: Array<number>,
+  sampleRate: number,
+) => void;
+
+export type BufferedSpeechPlayer = {
+  addAudioToBuffer: AddAudioToBufferFunction;
+  setGain: (gain: number) => void;
+  start: () => void;
+  stop: () => void;
+};
+
+type Options = {
+  onEnded?: () => void;
+  onStarted?: () => void;
+};
+
+export default function createBufferedSpeechPlayer({
+  onStarted,
+  onEnded,
+}: Options): BufferedSpeechPlayer {
+  const audioContext = new AudioContext();
+  const gainNode = audioContext.createGain();
+  gainNode.connect(audioContext.destination);
+
+  let unplayedAudioBuffers: Array<AudioBuffer> = [];
+
+  let currentPlayingBufferSource: AudioBufferSourceNode | null = null;
+
+  let isPlaying = false;
+
+  // This means that the player starts in the 'stopped' state, and you need to call player.start() for it to start playing
+  let shouldPlayWhenAudioAvailable = false;
+
+  const setGain = (gain: number) => {
+    gainNode.gain.setValueAtTime(gain, audioContext.currentTime);
+  };
+
+  const start = () => {
+    shouldPlayWhenAudioAvailable = true;
+    debug()?.start();
+    playNextBufferIfNotAlreadyPlaying();
+  };
+
+  // Stop will stop the audio and clear the buffers
+  const stop = () => {
+    shouldPlayWhenAudioAvailable = false;
+
+    // Stop the current buffers
+    currentPlayingBufferSource?.stop();
+    currentPlayingBufferSource = null;
+
+    unplayedAudioBuffers = [];
+
+    onEnded != null && onEnded();
+    isPlaying = false;
+    return;
+  };
+
+  const playNextBufferIfNotAlreadyPlaying = () => {
+    if (!isPlaying) {
+      playNextBuffer();
+    }
+  };
+
+  const playNextBuffer = () => {
+    if (shouldPlayWhenAudioAvailable === false) {
+      console.debug(
+        '[BufferedSpeechPlayer][playNextBuffer] Not playing any more audio because shouldPlayWhenAudioAvailable is false.',
+      );
+      // NOTE: we do not need to set isPlaying = false or call onEnded because that will be handled in the stop() function
+      return;
+    }
+    if (unplayedAudioBuffers.length === 0) {
+      console.debug(
+        '[BufferedSpeechPlayer][playNextBuffer] No buffers to play.',
+      );
+      if (isPlaying) {
+        isPlaying = false;
+        onEnded != null && onEnded();
+      }
+      return;
+    }
+
+    // If isPlaying is false, then we are starting playback fresh rather than continuing it, and should call onStarted
+    if (isPlaying === false) {
+      isPlaying = true;
+      onStarted != null && onStarted();
+    }
+
+    const source = audioContext.createBufferSource();
+
+    // Get the first unplayed buffer from the array, and remove it from the array
+    const buffer = unplayedAudioBuffers.shift() ?? null;
+    source.buffer = buffer;
+    console.debug(
+      `[BufferedSpeechPlayer] Playing buffer with ${source.buffer?.length} samples`,
+    );
+
+    source.connect(gainNode);
+    // the gain node is already connected to audioContext.destination
+    // source.connect(audioContext.destination);
+    const startTime = new Date().getTime();
+    source.start();
+    currentPlayingBufferSource = source;
+    // This is probably not necessary, but it doesn't hurt
+    isPlaying = true;
+
+    // TODO: consider changing this to a while loop to avoid deep recursion
+    const onThisBufferPlaybackEnded = () => {
+      console.debug(
+        `[BufferedSpeechPlayer] Buffer with ${source.buffer?.length} samples ended.`,
+      );
+      source.removeEventListener('ended', onThisBufferPlaybackEnded);
+      const endTime = new Date().getTime();
+      debug()?.playedAudio(startTime, endTime, buffer);
+      currentPlayingBufferSource = null;
+
+      // TODO: should we disconnect source from gain node here?
+      // source.disconnect(gainNode);
+
+      // We don't set isPlaying = false here because we are attempting to continue playing. It will get set to false if there are no more buffers to play
+      playNextBuffer();
+    };
+
+    source.addEventListener('ended', onThisBufferPlaybackEnded);
+  };
+
+  const addAudioToBuffer: AddAudioToBufferFunction = (samples, sampleRate) => {
+    const incomingArrayBufferChunk = audioContext.createBuffer(
+      // 1 channel
+      1,
+      samples.length,
+      sampleRate,
+    );
+
+    incomingArrayBufferChunk.copyToChannel(
+      new Float32Array(samples),
+      // first channel
+      0,
+    );
+
+    console.debug(
+      `[addAudioToBufferAndPlay] Adding buffer with ${incomingArrayBufferChunk.length} samples to queue.`,
+    );
+
+    unplayedAudioBuffers.push(incomingArrayBufferChunk);
+    debug()?.receivedAudio(
+      incomingArrayBufferChunk.length / incomingArrayBufferChunk.sampleRate,
+    );
+    const audioBuffersTableInfo = unplayedAudioBuffers.map((buffer, i) => {
+      return {
+        index: i,
+        duration: buffer.length / buffer.sampleRate,
+        samples: buffer.length,
+      };
+    });
+    const totalUnplayedDuration = unplayedAudioBuffers.reduce((acc, buffer) => {
+      return acc + buffer.length / buffer.sampleRate;
+    }, 0);
+
+    console.debug(
+      `[addAudioToBufferAndPlay] Current state of incoming audio buffers (${totalUnplayedDuration.toFixed(
+        1,
+      )}s unplayed):`,
+    );
+    console.table(audioBuffersTableInfo);
+
+    if (shouldPlayWhenAudioAvailable) {
+      playNextBufferIfNotAlreadyPlaying();
+    }
+  };
+
+  return {addAudioToBuffer, setGain, stop, start};
+}

streaming-react-app/src/cursorBlinkInterval.ts

@@ -0,0 +1 @@
+export const CURSOR_BLINK_INTERVAL_MS = 500;

streaming-react-app/src/debug.ts

@@ -0,0 +1,257 @@
+import {TYPING_ANIMATION_DELAY_MS} from './StreamingInterface';
+import {getURLParams} from './URLParams';
+import audioBuffertoWav from 'audiobuffer-to-wav';
+import './StreamingInterface.css';
+
+type StartEndTime = {
+  start: number;
+  end: number;
+};
+
+type StartEndTimeWithAudio = StartEndTime & {
+  float32Audio: Float32Array;
+};
+
+type Text = {
+  time: number;
+  chars: number;
+};
+
+type DebugTimings = {
+  receivedAudio: StartEndTime[];
+  playedAudio: StartEndTimeWithAudio[];
+  receivedText: Text[];
+  renderedText: StartEndTime[];
+  sentAudio: StartEndTimeWithAudio[];
+  startRenderTextTime: number | null;
+  startRecordingTime: number | null;
+  receivedAudioSampleRate: number | null;
+};
+
+function getInitialTimings(): DebugTimings {
+  return {
+    receivedAudio: [],
+    playedAudio: [],
+    receivedText: [],
+    renderedText: [],
+    sentAudio: [],
+    startRenderTextTime: null,
+    startRecordingTime: null,
+    receivedAudioSampleRate: null,
+  };
+}
+
+function downloadAudioBuffer(audioBuffer: AudioBuffer, fileName: string): void {
+  const wav = audioBuffertoWav(audioBuffer);
+  const wavBlob = new Blob([new DataView(wav)], {
+    type: 'audio/wav',
+  });
+  const url = URL.createObjectURL(wavBlob);
+  const anchor = document.createElement('a');
+  anchor.href = url;
+  anchor.target = '_blank';
+  anchor.download = fileName;
+  anchor.click();
+}
+
+// Uncomment for debugging without download
+// function playAudioBuffer(audioBuffer: AudioBuffer): void {
+//   const audioContext = new AudioContext();
+//   const source = audioContext.createBufferSource();
+
+//   source.buffer = audioBuffer;
+//   source.connect(audioContext.destination);
+//   source.start();
+// }
+
+// Accumulate timings and audio / text translation samples for debugging and exporting
+class DebugTimingsManager {
+  timings: DebugTimings = getInitialTimings();
+
+  start(): void {
+    this.timings = getInitialTimings();
+    this.timings.startRecordingTime = new Date().getTime();
+  }
+
+  sentAudio(event: AudioProcessingEvent): void {
+    const end = new Date().getTime();
+    const start = end - event.inputBuffer.duration * 1000;
+    // Copy or else buffer seems to be re-used
+    const float32Audio = new Float32Array(event.inputBuffer.getChannelData(0));
+    this.timings.sentAudio.push({
+      start,
+      end,
+      float32Audio,
+    });
+  }
+
+  receivedText(text: string): void {
+    this.timings.receivedText.push({
+      time: new Date().getTime(),
+      chars: text.length,
+    });
+  }
+
+  startRenderText(): void {
+    if (this.timings.startRenderTextTime == null) {
+      this.timings.startRenderTextTime = new Date().getTime();
+    }
+  }
+
+  endRenderText(): void {
+    if (this.timings.startRenderTextTime == null) {
+      console.warn(
+        'Wrong timings of start / end rendering text. startRenderText is null',
+      );
+      return;
+    }
+
+    this.timings.renderedText.push({
+      start: this.timings.startRenderTextTime as number,
+      end: new Date().getTime(),
+    });
+    this.timings.startRenderTextTime = null;
+  }
+
+  receivedAudio(duration: number): void {
+    const start = new Date().getTime();
+    this.timings.receivedAudio.push({
+      start,
+      end: start + duration * 1000,
+    });
+  }
+
+  playedAudio(start: number, end: number, buffer: AudioBuffer | null): void {
+    if (buffer != null) {
+      if (this.timings.receivedAudioSampleRate == null) {
+        this.timings.receivedAudioSampleRate = buffer.sampleRate;
+      }
+      if (this.timings.receivedAudioSampleRate != buffer.sampleRate) {
+        console.error(
+          'Sample rates of received audio are unequal, will fail to reconstruct debug audio',
+          this.timings.receivedAudioSampleRate,
+          buffer.sampleRate,
+        );
+      }
+    }
+    this.timings.playedAudio.push({
+      start,
+      end,
+      float32Audio:
+        buffer == null
+          ? new Float32Array()
+          : new Float32Array(buffer.getChannelData(0)),
+    });
+  }
+
+  getChartData() {
+    const columns = [
+      {type: 'string', id: 'Series'},
+      {type: 'date', id: 'Start'},
+      {type: 'date', id: 'End'},
+    ];
+    return [
+      columns,
+      ...this.timings.sentAudio.map((sentAudio) => [
+        'Sent Audio',
+        new Date(sentAudio.start),
+        new Date(sentAudio.end),
+      ]),
+      ...this.timings.receivedAudio.map((receivedAudio) => [
+        'Received Audio',
+        new Date(receivedAudio.start),
+        new Date(receivedAudio.end),
+      ]),
+      ...this.timings.playedAudio.map((playedAudio) => [
+        'Played Audio',
+        new Date(playedAudio.start),
+        new Date(playedAudio.end),
+      ]),
+      // Best estimate duration by multiplying length with animation duration for each letter
+      ...this.timings.receivedText.map((receivedText) => [
+        'Received Text',
+        new Date(receivedText.time),
+        new Date(
+          receivedText.time + receivedText.chars * TYPING_ANIMATION_DELAY_MS,
+        ),
+      ]),
+      ...this.timings.renderedText.map((renderedText) => [
+        'Rendered Text',
+        new Date(renderedText.start),
+        new Date(renderedText.end),
+      ]),
+    ];
+  }
+
+  downloadInputAudio() {
+    const audioContext = new AudioContext();
+    const totalLength = this.timings.sentAudio.reduce((acc, cur) => {
+      return acc + cur?.float32Audio?.length ?? 0;
+    }, 0);
+    if (totalLength === 0) {
+      return;
+    }
+
+    const incomingArrayBuffer = audioContext.createBuffer(
+      1, // 1 channel
+      totalLength,
+      audioContext.sampleRate,
+    );
+
+    const buffer = incomingArrayBuffer.getChannelData(0);
+    let i = 0;
+    this.timings.sentAudio.forEach((sentAudio) => {
+      sentAudio.float32Audio.forEach((bytes) => {
+        buffer[i++] = bytes;
+      });
+    });
+
+    // Play for debugging
+    // playAudioBuffer(incomingArrayBuffer);
+    downloadAudioBuffer(incomingArrayBuffer, `input_audio.wav`);
+  }
+
+  downloadOutputAudio() {
+    const playedAudio = this.timings.playedAudio;
+    const sampleRate = this.timings.receivedAudioSampleRate;
+    if (
+      playedAudio.length === 0 ||
+      this.timings.startRecordingTime == null ||
+      sampleRate == null
+    ) {
+      return null;
+    }
+
+    let previousEndTime = this.timings.startRecordingTime;
+    const audioArray: number[] = [];
+    playedAudio.forEach((audio) => {
+      const delta = (audio.start - previousEndTime) / 1000;
+      for (let i = 0; i < delta * sampleRate; i++) {
+        audioArray.push(0.0);
+      }
+      audio.float32Audio.forEach((bytes) => audioArray.push(bytes));
+      previousEndTime = audio.end;
+    });
+    const audioContext = new AudioContext();
+    const incomingArrayBuffer = audioContext.createBuffer(
+      1, // 1 channel
+      audioArray.length,
+      sampleRate,
+    );
+
+    incomingArrayBuffer.copyToChannel(
+      new Float32Array(audioArray),
+      0, // first channel
+    );
+
+    // Play for debugging
+    // playAudioBuffer(incomingArrayBuffer);
+    downloadAudioBuffer(incomingArrayBuffer, 'output_audio.wav');
+  }
+}
+
+const debugSingleton = new DebugTimingsManager();
+export default function debug(): DebugTimingsManager | null {
+  const debugParam = getURLParams().debug;
+  return debugParam ? debugSingleton : null;
+}

streaming-react-app/src/float32To16BitPCM.ts

@@ -0,0 +1,16 @@
+export default function float32To16BitPCM(
+  float32Arr: Float32Array,
+): Int16Array {
+  const pcm16bit = new Int16Array(float32Arr.length);
+  for (let i = 0; i < float32Arr.length; ++i) {
+    // force number in [-1,1]
+    const s = Math.max(-1, Math.min(1, float32Arr[i]));
+
+    /**
+     * convert 32 bit float to 16 bit int pcm audio
+     * 0x8000 = minimum int16 value, 0x7fff = maximum int16 value
+     */
+    pcm16bit[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+  }
+  return pcm16bit;
+}

streaming-react-app/src/generateNewRoomID.ts

@@ -0,0 +1,60 @@
+import {random} from 'lodash';
+
+// const USABLE_CHARACTERS = 'BCDFGHJKMPQRTVWXY2346789';
+const USABLE_CHARACTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+const ID_LENGTH = 4;
+
+export function isValidRoomID(id: string | null | undefined): boolean {
+  if (id == null) {
+    return false;
+  }
+  if (id.length !== ID_LENGTH) {
+    return false;
+  }
+  return isValidPartialRoomID(id);
+}
+
+export function isValidPartialRoomID(roomID: string): boolean {
+  return (
+    roomID.length <= ID_LENGTH &&
+    roomID.split('').every((char) => USABLE_CHARACTERS.includes(char))
+  );
+}
+
+export default function generateNewRoomID(): string {
+  return Array.from(
+    {length: ID_LENGTH},
+    () => USABLE_CHARACTERS[random(USABLE_CHARACTERS.length - 1)],
+  ).join('');
+}
+
+export function getSequentialRoomIDForTestingGenerator(): () => string {
+  let counter = 0;
+
+  return function generateNextRoomID(): string {
+    const counterInBase: string = Number(counter)
+      .toString(USABLE_CHARACTERS.length)
+      .padStart(ID_LENGTH, '0');
+
+    if (counterInBase.length > ID_LENGTH) {
+      throw new Error(
+        'Ran out of unique room IDs from the sequential generator',
+      );
+    }
+
+    const result = counterInBase
+      .split('')
+      .map(
+        (digit) => USABLE_CHARACTERS[parseInt(digit, USABLE_CHARACTERS.length)],
+      )
+      .join('');
+
+    counter++;
+
+    return result;
+  };
+}
+
+// const generator = getSequentialRoomIDForTestingGenerator();
+
+// Array.from({length: 200}, () => console.log(generator()));

streaming-react-app/src/getParamFlag.ts

@@ -0,0 +1,39 @@
+import type {URLParamNames} from './types/URLParamsTypes';
+
+export function getBooleanParamFlag(
+  flag: URLParamNames,
+  defaultValue?: boolean,
+): boolean {
+  const paramFlagValue = getBooleanParamFlagWithoutDefault(flag);
+
+  if (paramFlagValue == null) {
+    // The default value for paramFlags is false, unless they explicitly provide a
+    // defaultValue via the config
+    return defaultValue ?? false;
+  }
+
+  return paramFlagValue;
+}
+
+export function getBooleanParamFlagWithoutDefault(
+  flag: URLParamNames,
+): boolean | null {
+  const urlParams = new URLSearchParams(window.location.search);
+
+  if (urlParams.get(flag) == null) {
+    return null;
+  }
+
+  return urlParams.get(flag) !== '0';
+}
+
+export function getStringParamFlag(
+  flag: URLParamNames,
+  defaultValue?: string,
+): string | null {
+  const urlParams = new URLSearchParams(window.location.search);
+
+  const param = urlParams.get(flag);
+
+  return param ?? defaultValue ?? null;
+}

streaming-react-app/src/getTranslationSentencesFromReceivedData.ts

@@ -0,0 +1,23 @@
+import {ServerTextData, TranslationSentences} from './types/StreamingTypes';
+
+export default function getTranslationSentencesFromReceivedData(
+  receivedData: Array<ServerTextData>,
+): TranslationSentences {
+  return receivedData
+    .reduce(
+      (acc, data) => {
+        // TODO: Add special handling if the payload starts/ends with an apostrophe?
+        const newAcc = [
+          ...acc.slice(0, -1),
+          acc[acc.length - 1].trim() + ' ' + data.payload,
+        ];
+        if (data.eos) {
+          newAcc.push('');
+        }
+
+        return newAcc;
+      },
+      [''],
+    )
+    .filter((s) => s.trim().length !== 0);
+}

streaming-react-app/src/isScrolledToDocumentBottom.ts

@@ -0,0 +1,11 @@
+export default function isScrolledToDocumentBottom(
+  bufferPx: number = 0,
+): boolean {
+  if (
+    window.innerHeight + window.scrollY >=
+    document.body.offsetHeight - bufferPx
+  ) {
+    return true;
+  }
+  return false;
+}

streaming-react-app/src/main.tsx

@@ -0,0 +1,10 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App.tsx';
+import './index.css';
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+);

streaming-react-app/src/react-xr/Button.tsx

@@ -0,0 +1,117 @@
+import {useRef, useEffect} from 'react';
+import * as THREE from 'three';
+import {extend} from '@react-three/fiber';
+import ThreeMeshUI from 'three-mesh-ui';
+import ThreeMeshUIText, {ThreeMeshUITextType} from './ThreeMeshUIText';
+import {Interactive} from '@react-three/xr';
+
+/**
+ * Using `?url` at the end of this import tells vite this is a static asset, and
+ * provides us a URL to the hashed version of the file when the project is built.
+ * See: https://vitejs.dev/guide/assets.html#explicit-url-imports
+ */
+import robotoFontFamilyJson from '../assets/RobotoMono-Regular-msdf.json?url';
+import robotoFontTexture from '../assets/RobotoMono-Regular.png';
+
+extend(ThreeMeshUI);
+
+/**
+ * Button component that renders as a three-mesh-ui block
+ */
+export default function Button({
+  onClick,
+  content,
+  width,
+  height,
+  fontSize,
+  borderRadius,
+  padding,
+}) {
+  const button = useRef<JSX.IntrinsicElements['block']>();
+  const textRef = useRef<ThreeMeshUITextType>();
+
+  useEffect(() => {
+    if (textRef.current != null) {
+      textRef.current.set({content});
+    }
+  }, [textRef, content]);
+
+  useEffect(() => {
+    if (!button.current) {
+      return;
+    }
+    button.current.setupState({
+      state: 'hovered',
+      attributes: {
+        offset: 0.002,
+        backgroundColor: new THREE.Color(0x607b8f),
+        fontColor: new THREE.Color(0xffffff),
+      },
+    });
+    button.current.setupState({
+      state: 'idle',
+      attributes: {
+        offset: 0.001,
+        backgroundColor: new THREE.Color(0x465a69),
+        fontColor: new THREE.Color(0xffffff),
+      },
+    });
+    button.current.setupState({
+      state: 'selected',
+      attributes: {
+        offset: 0.005,
+        backgroundColor: new THREE.Color(0x000000),
+        fontColor: new THREE.Color(0xffffff),
+      },
+    });
+    button.current.setState('idle');
+  }, []);
+
+  const args = [
+    {
+      width,
+      height,
+      fontSize,
+      padding,
+      justifyContent: 'end',
+      textAlign: 'center',
+      alignItems: 'center',
+      borderRadius,
+      fontFamily: robotoFontFamilyJson,
+      fontTexture: robotoFontTexture,
+      backgroundOpacity: 1,
+      backgroundColor: new THREE.Color(0x779092),
+      fontColor: new THREE.Color(0x000000),
+    },
+  ];
+
+  return (
+    <Interactive
+      // These are for XR mode
+      onSelect={() => {
+        onClick();
+      }}
+      onHover={() => button.current.setState('hovered')}
+      onBlur={() => button.current.setState('idle')}
+      onSelectStart={() => button.current.setState('selected')}
+      onSelectEnd={() => button.current.setState('idle')}>
+      <block
+        // These are for non-XR modes
+        onPointerEnter={() => button.current.setState('hovered')}
+        onPointerLeave={() => button.current.setState('idle')}
+        onPointerDown={() => button.current.setState('selected')}
+        onPointerUp={() => {
+          button.current.setState('hovered');
+          onClick();
+        }}>
+        <block args={args} ref={button}>
+          <ThreeMeshUIText
+            ref={textRef}
+            fontColor={new THREE.Color(0xffffff)}
+            content={content}
+          />
+        </block>
+      </block>
+    </Interactive>
+  );
+}

streaming-react-app/src/react-xr/Colors.ts

@@ -0,0 +1,6 @@
+import * as THREE from 'three';
+
+export const WHITE = new THREE.Color('#FFFFFF');
+export const BLACK = new THREE.Color('#000000');
+export const RED = new THREE.Color('red');
+export const BLUE = new THREE.Color('blue');

streaming-react-app/src/react-xr/MovementController.tsx

@@ -0,0 +1,64 @@
+import {useRef} from 'react';
+import {useFrame} from '@react-three/fiber';
+import {useController, useXR} from '@react-three/xr';
+import * as THREE from 'three';
+
+const USE_HORIZONTAL = true;
+const USE_VERTICAL = true;
+const USE_ROTATION = true;
+const HORIZONTAL_AXIS = 2;
+const VERTICAL_AXIS = 3;
+const ROTATION_AXIS = 2;
+const SENSITIVITY = 0.05;
+const DEADZONE = 0.05;
+
+/**
+ * Component to add into the ThreeJS canvas that reads controller (Quest) inputs to change camera position
+ */
+export default function MovementController() {
+  const xr = useXR();
+  const controller = useController('right');
+  const forward = useRef(new THREE.Vector3());
+  const horizontal = useRef(new THREE.Vector3());
+
+  useFrame(() => {
+    const player = xr.player;
+    const camera = xr.player.children[0];
+    const cameraMatrix = camera.matrixWorld.elements;
+    forward.current
+      .set(-cameraMatrix[8], -cameraMatrix[9], -cameraMatrix[10])
+      .normalize();
+
+    const axes = controller?.inputSource?.gamepad?.axes ?? [0, 0, 0, 0];
+
+    if (USE_HORIZONTAL) {
+      horizontal.current.copy(forward.current);
+      horizontal.current.cross(camera.up).normalize();
+
+      player.position.add(
+        horizontal.current.multiplyScalar(
+          (Math.abs(axes[HORIZONTAL_AXIS]) > DEADZONE
+            ? axes[HORIZONTAL_AXIS]
+            : 0) * SENSITIVITY,
+        ),
+      );
+    }
+
+    if (USE_VERTICAL) {
+      player.position.add(
+        forward.current.multiplyScalar(
+          (Math.abs(axes[VERTICAL_AXIS]) > DEADZONE ? axes[VERTICAL_AXIS] : 0) *
+            SENSITIVITY,
+        ),
+      );
+    }
+
+    if (USE_ROTATION) {
+      player.rotation.y -=
+        (Math.abs(axes[ROTATION_AXIS]) > DEADZONE ? axes[ROTATION_AXIS] : 0) *
+        SENSITIVITY;
+    }
+  });
+
+  return <></>;
+}

streaming-react-app/src/react-xr/Playground.tsx

@@ -0,0 +1,133 @@
+/**
+ * EXPERIMENTAL components to play around with but not officially use in the demo while
+ * we develop.
+ */
+import {useEffect, useState} from 'react';
+import {Object3DNode, extend} from '@react-three/fiber';
+import ThreeMeshUI from 'three-mesh-ui';
+
+import {} from '@react-three/xr';
+import {Sparkles, Shadow} from '@react-three/drei';
+
+// import FontImage from './assets/Roboto-msdf.png';
+import {FontLoader} from 'three/examples/jsm/loaders/FontLoader.js';
+import {TextGeometry} from 'three/examples/jsm/geometries/TextGeometry.js';
+import ThreeMeshUIText from './ThreeMeshUIText';
+import {ContactShadows, BakeShadows} from '@react-three/drei';
+
+extend({TextGeometry});
+extend(ThreeMeshUI);
+
+declare module '@react-three/fiber' {
+  interface ThreeElements {
+    textGeometry: Object3DNode<TextGeometry, typeof TextGeometry>;
+  }
+}
+
+// This is for textGeometry.. not using three-mesh-ui to display text
+export function TitleMesh() {
+  const font = new FontLoader().parse();
+  console.log('font', font);
+  const [text, setText] = useState('Text');
+
+  useEffect(() => {
+    setTimeout(() => {
+      setText(text + ' more ');
+      console.log('adding more tex..', text);
+    }, 1000);
+  }, [text]);
+
+  return (
+    <mesh>
+      <textGeometry args={[text, {font, size: 5, height: 1}]} />
+      <meshPhysicalMaterial attach={'material'} color={'white'} />
+    </mesh>
+  );
+}
+
+export function Sphere({
+  size = 1,
+  amount = 50,
+  color = 'white',
+  emissive,
+  ...props
+}) {
+  return (
+    <mesh {...props}>
+      <sphereGeometry args={[size, 64, 64]} />
+      <meshPhysicalMaterial
+        roughness={0}
+        color={color}
+        emissive={emissive || color}
+        envMapIntensity={0.2}
+      />
+      <Sparkles count={amount} scale={size * 2} size={6} speed={0.4} />
+      <Shadow
+        rotation={[-Math.PI / 2, 0, 0]}
+        scale={size}
+        position={[0, -size, 0]}
+        color={emissive}
+        opacity={0.5}
+      />
+    </mesh>
+  );
+}
+
+export function Title({accentColor}) {
+  return (
+    <block
+      args={[
+        {
+          width: 1,
+          height: 0.25,
+          backgroundOpacity: 0,
+          justifyContent: 'center',
+        },
+      ]}>
+      <ThreeMeshUIText content={'Hello '} />
+      <ThreeMeshUIText content={'world!'} args={[{fontColor: accentColor}]} />
+    </block>
+  );
+}
+
+export function RandomComponents() {
+  return (
+    <>
+      <color args={['#eee']} attach={'background'} />
+      <Sphere
+        color="white"
+        amount={50}
+        emissive="green"
+        glow="lightgreen"
+        position={[1, 1, -1]}
+      />
+      <Sphere
+        color="white"
+        amount={30}
+        emissive="purple"
+        glow="#ff90f0"
+        size={0.5}
+        position={[-1.5, 0.5, -2]}
+      />
+      <Sphere
+        color="lightpink"
+        amount={20}
+        emissive="orange"
+        glow="#ff9f50"
+        size={0.25}
+        position={[-1, 0.25, 1]}
+      />
+      <ContactShadows
+        renderOrder={2}
+        color="black"
+        resolution={1024}
+        frames={1}
+        scale={10}
+        blur={1.5}
+        opacity={0.65}
+        far={0.5}
+      />
+      <BakeShadows />
+    </>
+  );
+}

streaming-react-app/src/react-xr/TextBlocks.tsx

@@ -0,0 +1,235 @@
+import {JSX, useEffect, useRef, useState} from 'react';
+import robotoFontFamilyJson from '../assets/RobotoMono-Regular-msdf.json?url';
+import robotoFontTexture from '../assets/RobotoMono-Regular.png';
+import ThreeMeshUIText, {ThreeMeshUITextType} from './ThreeMeshUIText';
+import {getURLParams} from '../URLParams';
+import {CURSOR_BLINK_INTERVAL_MS} from '../cursorBlinkInterval';
+
+const NUM_LINES = 3;
+
+export const CHARS_PER_LINE = 37;
+const MAX_WIDTH = 0.89;
+const CHAR_WIDTH = 0.0235;
+const Y_COORD_START = -0.38;
+const Z_COORD = -1.3;
+const LINE_HEIGHT = 0.062;
+const BLOCK_SPACING = 0.02;
+const FONT_SIZE = 0.038;
+
+const SCROLL_Y_DELTA = 0.001;
+
+// Overlay an extra block for padding due to inflexibilities of native padding
+const OFFSET = 0.01;
+const OFFSET_WIDTH = OFFSET * 3;
+
+type Props = {
+  content: string;
+  // The actual position or end position when animating
+  y: number;
+  // The start position when animating
+  startY: number;
+  width: number;
+  height: number;
+  textOpacity: number;
+  backgroundOpacity: number;
+  // Use this to keep track of sentence + line position for animation
+  index: string;
+  enableAnimation: boolean;
+};
+
+function TextBlock({
+  content,
+  y,
+  startY,
+  width,
+  height,
+  textOpacity,
+  backgroundOpacity,
+  index,
+  enableAnimation,
+}: Props) {
+  const [scrollY, setScrollY] = useState<number>(y);
+
+  // We are reusing text blocks so this keeps track of when we changed rows so we can restart animation
+  const lastIndex = useRef<string>(index);
+  useEffect(() => {
+    if (index != lastIndex.current) {
+      lastIndex.current = index;
+      enableAnimation && setScrollY(startY);
+    } else if (scrollY < y) {
+      setScrollY((prev) => prev + SCROLL_Y_DELTA);
+    }
+  }, [enableAnimation, index, scrollY, setScrollY, startY, y]);
+
+  // This is needed to update text content (doesn't work if we just update the content prop)
+  const textRef = useRef<ThreeMeshUITextType>();
+  useEffect(() => {
+    if (textRef.current != null) {
+      textRef.current.set({content});
+    }
+  }, [content, textRef, y, startY]);
+
+  // Width starts from 0 and goes 1/2 in each direction
+  const xPosition = width / 2 - MAX_WIDTH / 2 + OFFSET_WIDTH;
+  return (
+    <>
+      <block
+        args={[
+          {
+            backgroundOpacity,
+            width: width + OFFSET_WIDTH,
+            height: height,
+            borderRadius: 0,
+          },
+        ]}
+        position={[-OFFSET_WIDTH + xPosition, scrollY, Z_COORD]}></block>
+      <block
+        args={[{padding: 0, backgroundOpacity: 0, width, height}]}
+        position={[xPosition, scrollY + OFFSET, Z_COORD]}>
+        <block
+          args={[
+            {
+              width,
+              height,
+              fontSize: FONT_SIZE,
+              textAlign: 'left',
+              backgroundOpacity: 0,
+              // TODO: support more language charsets
+              // This renders using MSDF format supported in WebGL. Renderable characters are defined in the "charset" json
+              // Currently supports most default keyboard inputs but this would exclude many non latin charset based languages.
+              // You can use https://msdf-bmfont.donmccurdy.com/ for easily generating these files
+              // fontFamily: '/src/assets/Roboto-msdf.json',
+              // fontTexture: '/src/assets/Roboto-msdf.png'
+              fontFamily: robotoFontFamilyJson,
+              fontTexture: robotoFontTexture,
+            },
+          ]}>
+          <ThreeMeshUIText ref={textRef} content="" fontOpacity={textOpacity} />
+        </block>
+      </block>
+    </>
+  );
+}
+
+// Background behind the text so it covers any missing spaces
+function TranscriptionPanel() {
+  const panelHeight = LINE_HEIGHT * NUM_LINES + 2 * BLOCK_SPACING + 2 * OFFSET;
+  const xPosition = OFFSET_WIDTH;
+  return (
+    <block
+      args={[
+        {
+          backgroundOpacity: 1,
+          width:
+            MAX_WIDTH * ((CHARS_PER_LINE + 2) / CHARS_PER_LINE) +
+            2 * OFFSET_WIDTH,
+          height: panelHeight,
+          borderRadius: 0,
+        },
+      ]}
+      position={[
+        -OFFSET + xPosition,
+        Y_COORD_START + panelHeight / 2 - 2 * OFFSET,
+        Z_COORD,
+      ]}></block>
+  );
+}
+
+export default function TextBlocks({
+  sentences,
+  blinkCursor,
+}: {
+  sentences: string[][];
+  blinkCursor: boolean;
+}) {
+  const showTranscriptionPanel =
+    getURLParams().ARTranscriptionType === 'lines_with_background';
+  const textBlocks: JSX.Element[] = [];
+
+  const [cursorBlinkOn, setCursorBlinkOn] = useState(false);
+  useEffect(() => {
+    if (blinkCursor) {
+      const interval = setInterval(() => {
+        setCursorBlinkOn((prev) => !prev);
+      }, CURSOR_BLINK_INTERVAL_MS);
+
+      return () => clearInterval(interval);
+    } else {
+      setCursorBlinkOn(false);
+    }
+  }, [blinkCursor]);
+
+  // Start from bottom and populate most recent sentences by line until we fill max lines.
+  let currentY = Y_COORD_START;
+  for (let i = sentences.length - 1; i >= 0; i--) {
+    const sentenceLines = sentences[i];
+    for (let j = sentenceLines.length - 1; j >= 0; j--) {
+      if (textBlocks.length == NUM_LINES) {
+        if (showTranscriptionPanel) {
+          textBlocks.push(<TranscriptionPanel key={textBlocks.length} />);
+        }
+        return textBlocks;
+      }
+
+      const isBottomSentence = i === sentences.length - 1;
+      const isBottomLine = isBottomSentence && textBlocks.length === 0;
+      const y = currentY + LINE_HEIGHT / 2;
+      let textBlockLine = sentenceLines[j];
+      const numChars = textBlockLine.length;
+
+      if (cursorBlinkOn && isBottomLine) {
+        textBlockLine = textBlockLine + '|';
+      }
+
+      // Accounting for potential cursor for block width (the +1)
+      const blockWidth =
+        (numChars + (isBottomLine ? 1.1 : 0) + (numChars < 10 ? 1 : 0)) *
+        CHAR_WIDTH;
+      const textOpacity = 1 - 0.1 * textBlocks.length;
+      textBlocks.push(
+        <TextBlock
+          key={textBlocks.length}
+          y={y}
+          startY={currentY}
+          index={`${sentences.length - i},${j}`}
+          textOpacity={textOpacity}
+          backgroundOpacity={0.98}
+          height={LINE_HEIGHT}
+          width={blockWidth}
+          // content={"BLOCK " + textBlocks.length + ": " + content}
+          content={textBlockLine}
+          enableAnimation={!isBottomLine}
+        />,
+      );
+
+      currentY = y + LINE_HEIGHT / 2;
+    }
+    currentY += showTranscriptionPanel ? BLOCK_SPACING / 3 : BLOCK_SPACING;
+  }
+
+  const numRemainingBlocks = textBlocks.length - NUM_LINES;
+  if (numRemainingBlocks > 0) {
+    Array.from({length: numRemainingBlocks}).forEach(() => {
+      // Push in non display blocks because mesh UI crashes if elements are add / removed from screen.
+      textBlocks.push(
+        <TextBlock
+          key={textBlocks.length}
+          y={Y_COORD_START}
+          startY={0}
+          index="0,0"
+          textOpacity={0}
+          backgroundOpacity={0}
+          enableAnimation={false}
+          width={MAX_WIDTH}
+          height={LINE_HEIGHT}
+          content=""
+        />,
+      );
+    });
+  }
+
+  if (showTranscriptionPanel) {
+    textBlocks.push(<TranscriptionPanel key={textBlocks.length} />);
+  }
+  return textBlocks;
+}

streaming-react-app/src/react-xr/ThreeMeshUIText.tsx

@@ -0,0 +1,22 @@
+import {extend} from '@react-three/fiber';
+import {forwardRef} from 'react';
+import ThreeMeshUI, {TextOptions} from 'three-mesh-ui';
+
+extend(ThreeMeshUI);
+
+/**
+ * Hacky but component that wraps <text> since this has typescript issues because it collides with
+ * the native <text> SVG element. Simple enough so abstracting it away in this file
+ * so it could be used in other places with low risk. e.g:
+ * <ThreeMeshUIText content="Hello" />
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type ThreeMeshUITextType = any;
+
+const ThreeMeshUIText = forwardRef<ThreeMeshUITextType, TextOptions>(
+  function ThreeMeshUIText(props, ref) {
+    return <text {...props} ref={ref} />;
+  },
+);
+
+export default ThreeMeshUIText;

streaming-react-app/src/react-xr/XRConfig.tsx

@@ -0,0 +1,449 @@
+import {useCallback, useEffect, useRef, useState} from 'react';
+import {
+  Canvas,
+  createPortal,
+  extend,
+  useFrame,
+  useThree,
+} from '@react-three/fiber';
+import ThreeMeshUI from 'three-mesh-ui';
+
+import {ARButton, XR, Hands, XREvent} from '@react-three/xr';
+
+import {TextGeometry} from 'three/examples/jsm/geometries/TextGeometry.js';
+import {TranslationSentences} from '../types/StreamingTypes';
+import Button from './Button';
+import {RoomState} from '../types/RoomState';
+import ThreeMeshUIText, {ThreeMeshUITextType} from './ThreeMeshUIText';
+import {BLACK, WHITE} from './Colors';
+
+/**
+ * Using `?url` at the end of this import tells vite this is a static asset, and
+ * provides us a URL to the hashed version of the file when the project is built.
+ * See: https://vitejs.dev/guide/assets.html#explicit-url-imports
+ */
+import robotoFontFamilyJson from '../assets/RobotoMono-Regular-msdf.json?url';
+import robotoFontTexture from '../assets/RobotoMono-Regular.png';
+import {getURLParams} from '../URLParams';
+import TextBlocks, {CHARS_PER_LINE} from './TextBlocks';
+import {BufferedSpeechPlayer} from '../createBufferedSpeechPlayer';
+import {CURSOR_BLINK_INTERVAL_MS} from '../cursorBlinkInterval';
+
+// Adds on react JSX for add-on libraries to react-three-fiber
+extend(ThreeMeshUI);
+extend({TextGeometry});
+
+async function fetchSupportedCharSet(): Promise<Set<string>> {
+  try {
+    const response = await fetch(robotoFontFamilyJson);
+    const fontFamily = await response.json();
+
+    return new Set(fontFamily.info.charset);
+  } catch (e) {
+    console.error('Failed to fetch supported XR charset', e);
+    return new Set();
+  }
+}
+
+let supportedCharSet = new Set();
+fetchSupportedCharSet().then((result) => (supportedCharSet = result));
+
+// This component wraps any children so it is positioned relative to the camera, rather than from the origin
+function CameraLinkedObject({children}) {
+  const camera = useThree((state) => state.camera);
+  return createPortal(<>{children}</>, camera);
+}
+
+function ThreeMeshUIComponents({
+  translationSentences,
+  skipARIntro,
+  roomState,
+  animateTextDisplay,
+}: XRConfigProps & {skipARIntro: boolean}) {
+  // The "loop" for re-rendering required for threemeshUI
+  useFrame(() => {
+    ThreeMeshUI.update();
+  });
+  const [started, setStarted] = useState<boolean>(skipARIntro);
+  return (
+    <>
+      <CameraLinkedObject>
+        {getURLParams().ARTranscriptionType === 'single_block' ? (
+          <TranscriptPanelSingleBlock
+            started={started}
+            animateTextDisplay={animateTextDisplay}
+            roomState={roomState}
+            translationSentences={translationSentences}
+          />
+        ) : (
+          <TranscriptPanelBlocks
+            animateTextDisplay={animateTextDisplay}
+            translationSentences={translationSentences}
+          />
+        )}
+        {skipARIntro ? null : (
+          <IntroPanel started={started} setStarted={setStarted} />
+        )}
+      </CameraLinkedObject>
+    </>
+  );
+}
+
+// Original UI that just uses a single block to render 6 lines in a panel
+function TranscriptPanelSingleBlock({
+  animateTextDisplay,
+  started,
+  translationSentences,
+  roomState,
+}: {
+  animateTextDisplay: boolean;
+  started: boolean;
+  translationSentences: TranslationSentences;
+  roomState: RoomState | null;
+}) {
+  const textRef = useRef<ThreeMeshUITextType>();
+  const [didReceiveTranslationSentences, setDidReceiveTranslationSentences] =
+    useState(false);
+
+  const hasActiveTranscoders = (roomState?.activeTranscoders ?? 0) > 0;
+
+  const [cursorBlinkOn, setCursorBlinkOn] = useState(false);
+
+  // Normally we don't setState in render, but here we need to for computed state, and this if statement assures it won't loop infinitely
+  if (!didReceiveTranslationSentences && translationSentences.length > 0) {
+    setDidReceiveTranslationSentences(true);
+  }
+
+  const width = 1;
+  const height = 0.3;
+  const fontSize = 0.03;
+
+  useEffect(() => {
+    if (animateTextDisplay && hasActiveTranscoders) {
+      const interval = setInterval(() => {
+        setCursorBlinkOn((prev) => !prev);
+      }, CURSOR_BLINK_INTERVAL_MS);
+
+      return () => clearInterval(interval);
+    } else {
+      setCursorBlinkOn(false);
+    }
+  }, [animateTextDisplay, hasActiveTranscoders]);
+
+  useEffect(() => {
+    if (textRef.current != null) {
+      const initialPrompt =
+        'Welcome to the presentation. We are excited to share with you the work we have been doing... Our model can now translate languages in less than 2 second latency.';
+      // These are rough ratios based on spot checking
+      const maxLines = 6;
+      const charsPerLine = 55;
+
+      const transcriptSentences: string[] = didReceiveTranslationSentences
+        ? translationSentences
+        : [initialPrompt];
+
+      // The transcript is an array of sentences. For each sentence we break this down into an array of words per line.
+      // This is needed so we can "scroll" through without changing the order of words in the transcript
+      const linesToDisplay = transcriptSentences.flatMap((sentence, idx) => {
+        const blinkingCursor =
+          cursorBlinkOn && idx === transcriptSentences.length - 1 ? '|' : ' ';
+        const words = sentence.concat(blinkingCursor).split(/\s+/);
+        // Here we break each sentence up with newlines so all words per line fit within the panel
+        return words.reduce(
+          (wordChunks, currentWord) => {
+            const filteredWord = [...currentWord]
+              .filter((c) => {
+                if (supportedCharSet.has(c)) {
+                  return true;
+                }
+                console.error(
+                  `Unsupported char ${c} - make sure this is supported in the font family msdf file`,
+                );
+                return false;
+              })
+              .join('');
+            const lastLineSoFar = wordChunks[wordChunks.length - 1];
+            const charCount = lastLineSoFar.length + filteredWord.length + 1;
+            if (charCount <= charsPerLine) {
+              wordChunks[wordChunks.length - 1] =
+                lastLineSoFar + ' ' + filteredWord;
+            } else {
+              wordChunks.push(filteredWord);
+            }
+            return wordChunks;
+          },
+          [''],
+        );
+      });
+
+      // Only keep the last maxLines so new text keeps scrolling up from the bottom
+      linesToDisplay.splice(0, linesToDisplay.length - maxLines);
+      textRef.current.set({content: linesToDisplay.join('\n')});
+    }
+  }, [
+    translationSentences,
+    textRef,
+    didReceiveTranslationSentences,
+    cursorBlinkOn,
+  ]);
+
+  const opacity = started ? 1 : 0;
+  return (
+    <block
+      args={[{padding: 0.05, backgroundOpacity: opacity}]}
+      position={[0, -0.4, -1.3]}>
+      <block
+        args={[
+          {
+            width,
+            height,
+            fontSize,
+            textAlign: 'left',
+            backgroundOpacity: opacity,
+            // TODO: support more language charsets
+            // This renders using MSDF format supported in WebGL. Renderable characters are defined in the "charset" json
+            // Currently supports most default keyboard inputs but this would exclude many non latin charset based languages.
+            // You can use https://msdf-bmfont.donmccurdy.com/ for easily generating these files
+            // fontFamily: '/src/assets/Roboto-msdf.json',
+            // fontTexture: '/src/assets/Roboto-msdf.png'
+            fontFamily: robotoFontFamilyJson,
+            fontTexture: robotoFontTexture,
+          },
+        ]}>
+        <ThreeMeshUIText
+          ref={textRef}
+          content={'Transcript'}
+          fontOpacity={opacity}
+        />
+      </block>
+    </block>
+  );
+}
+
+// Splits up the lines into separate blocks to treat each one separately.
+// This allows changing of opacity, animating per line, changing height / width per line etc
+function TranscriptPanelBlocks({
+  animateTextDisplay,
+  translationSentences,
+}: {
+  animateTextDisplay: boolean;
+  translationSentences: TranslationSentences;
+}) {
+  const [didReceiveTranslationSentences, setDidReceiveTranslationSentences] =
+    // Currently causing issues with displaying dummy text, skip over
+    useState(false);
+
+  // Normally we don't setState in render, but here we need to for computed state, and this if statement assures it won't loop infinitely
+  if (!didReceiveTranslationSentences && translationSentences.length > 0) {
+    setDidReceiveTranslationSentences(true);
+  }
+
+  const initialPrompt = 'Listening...';
+  const transcriptSentences: string[] = didReceiveTranslationSentences
+    ? translationSentences
+    : [initialPrompt];
+
+  // The transcript is an array of sentences. For each sentence we break this down into an array of words per line.
+  // This is needed so we can "scroll" through without changing the order of words in the transcript
+  const sentenceLines = transcriptSentences.map((sentence) => {
+    const words = sentence.split(/\s+/);
+    // Here we break each sentence up with newlines so all words per line fit within the panel
+    return words.reduce(
+      (wordChunks, currentWord) => {
+        const filteredWord = [...currentWord]
+          .filter((c) => {
+            if (supportedCharSet.has(c)) {
+              return true;
+            }
+            console.error(
+              `Unsupported char ${c} - make sure this is supported in the font family msdf file`,
+            );
+            return false;
+          })
+          .join('');
+        const lastLineSoFar = wordChunks[wordChunks.length - 1];
+        const charCount = lastLineSoFar.length + filteredWord.length + 1;
+        if (charCount <= CHARS_PER_LINE) {
+          wordChunks[wordChunks.length - 1] =
+            lastLineSoFar + ' ' + filteredWord;
+        } else {
+          wordChunks.push(filteredWord);
+        }
+        return wordChunks;
+      },
+      [''],
+    );
+  });
+  return (
+    <TextBlocks sentences={sentenceLines} blinkCursor={animateTextDisplay} />
+  );
+}
+
+function IntroPanel({started, setStarted}) {
+  const width = 0.5;
+  const height = 0.4;
+  const padding = 0.03;
+
+  // Kind of hacky but making the panel disappear by moving it completely off the camera view.
+  // If we try to remove elements we end up throwing and stopping the experience
+  // opacity=0 also runs into weird bugs where not everything is invisible
+  const xCoordinate = started ? 1000000 : 0;
+
+  const commonArgs = {
+    backgroundColor: WHITE,
+    width,
+    height,
+    padding,
+    backgroundOpacity: 1,
+    textAlign: 'center',
+    fontFamily: robotoFontFamilyJson,
+    fontTexture: robotoFontTexture,
+  };
+  return (
+    <>
+      <block
+        args={[
+          {
+            ...commonArgs,
+            fontSize: 0.02,
+          },
+        ]}
+        position={[xCoordinate, -0.1, -0.5]}>
+        <ThreeMeshUIText
+          content="FAIR Seamless Streaming Demo"
+          fontColor={BLACK}
+        />
+      </block>
+      <block
+        args={[
+          {
+            ...commonArgs,
+            fontSize: 0.016,
+            backgroundOpacity: 0,
+          },
+        ]}
+        position={[xCoordinate, -0.15, -0.5001]}>
+        <ThreeMeshUIText
+          fontColor={BLACK}
+          content="Welcome to the Seamless team streaming demo experience! In this demo, you would experience AI powered text and audio translation in real time."
+        />
+      </block>
+      <block
+        args={[
+          {
+            width: 0.1,
+            height: 0.1,
+            backgroundOpacity: 1,
+            backgroundColor: BLACK,
+          },
+        ]}
+        position={[xCoordinate, -0.23, -0.5002]}>
+        <Button
+          onClick={() => setStarted(true)}
+          content={'Start Experience'}
+          width={0.2}
+          height={0.035}
+          fontSize={0.015}
+          padding={0.01}
+          borderRadius={0.01}
+        />
+      </block>
+    </>
+  );
+}
+
+export type XRConfigProps = {
+  animateTextDisplay: boolean;
+  bufferedSpeechPlayer: BufferedSpeechPlayer;
+  translationSentences: TranslationSentences;
+  roomState: RoomState | null;
+  roomID: string | null;
+  startStreaming: () => Promise<void>;
+  stopStreaming: () => Promise<void>;
+  debugParam: boolean | null;
+};
+
+export default function XRConfig(props: XRConfigProps) {
+  const {bufferedSpeechPlayer, debugParam} = props;
+  const skipARIntro = getURLParams().skipARIntro;
+  const defaultDimensions = {width: 500, height: 500};
+  const [dimensions, setDimensions] = useState(
+    debugParam ? defaultDimensions : {width: 0, height: 0},
+  );
+  const {width, height} = dimensions;
+
+  // Make sure to reset buffer when headset is taken off / on so we don't get an endless stream
+  // of audio. The oculus actually runs for some time after the headset is taken off.
+  const resetBuffers = useCallback(
+    (event: XREvent<XRSessionEvent>) => {
+      const session = event.target;
+      if (!(session instanceof XRSession)) {
+        return;
+      }
+      switch (session.visibilityState) {
+        case 'visible':
+          bufferedSpeechPlayer.start();
+          break;
+        case 'hidden':
+          bufferedSpeechPlayer.stop();
+          break;
+      }
+    },
+    [bufferedSpeechPlayer],
+  );
+
+  return (
+    <div style={{height, width, margin: '0 auto', border: '1px solid #ccc'}}>
+      {/* This is the button that triggers AR flow if available via a button */}
+      <ARButton
+        onError={(e) => console.error(e)}
+        onClick={() => setDimensions(defaultDimensions)}
+        style={{
+          position: 'absolute',
+          bottom: '24px',
+          left: '50%',
+          transform: 'translateX(-50%)',
+          padding: '12px 24px',
+          border: '1px solid white',
+          borderRadius: '4px',
+          backgroundColor: '#465a69',
+          color: 'white',
+          font: 'normal 0.8125rem sans-serif',
+          outline: 'none',
+          zIndex: 99999,
+          cursor: 'pointer',
+        }}
+      />
+      {/* Canvas to draw if in browser but if in AR mode displays in pass through mode */}
+      {/* The camera here just works in 2D mode. In AR mode it starts at at origin */}
+      {/* <Canvas camera={{position: [0, 0, 1], fov: 60}}> */}
+      <Canvas camera={{position: [0, 0, 0.001], fov: 60}}>
+        <color attach="background" args={['grey']} />
+        <XR referenceSpace="local" onVisibilityChange={resetBuffers}>
+          {/*
+            Uncomment this for controllers to show up
+            <Controllers />
+          */}
+          <Hands />
+
+          {/*
+            Uncomment this for moving with controllers
+            <MovementController />
+          */}
+          {/*
+            Uncomment this for turning the view in non-vr mode
+            <OrbitControls
+              autoRotateSpeed={0.85}
+              zoomSpeed={1}
+              minPolarAngle={Math.PI / 2.5}
+              maxPolarAngle={Math.PI / 2.55}
+            />
+          */}
+          <ThreeMeshUIComponents {...props} skipARIntro={skipARIntro} />
+          {/* Just for testing */}
+          {/* <RandomComponents /> */}
+        </XR>
+      </Canvas>
+    </div>
+  );
+}

streaming-react-app/src/react-xr/XRDialog.css

@@ -0,0 +1,13 @@
+.xr-dialog-button {
+  margin-bottom: 16px;
+  padding-left: 20px;
+  padding-right: 20px;
+}
+
+.xr-dialog-container {
+  min-height: 200px;
+}
+
+.xr-dialog-text-center {
+  text-align: center;
+}

streaming-react-app/src/react-xr/XRDialog.tsx

@@ -0,0 +1,49 @@
+import {
+  Button,
+  Dialog,
+  DialogContent,
+  DialogTitle,
+  IconButton,
+  Typography,
+} from '@mui/material';
+import CloseIcon from '@mui/icons-material/Close';
+import XRConfig, {XRConfigProps} from './XRConfig';
+import {useState} from 'react';
+import './XRDialog.css';
+
+export default function XRDialog(props: XRConfigProps) {
+  const [isDialogOpen, setIsDialogOpen] = useState<boolean>(false);
+  return (
+    <>
+      <Button variant="contained" onClick={() => setIsDialogOpen(true)}>
+        Enter AR Experience
+      </Button>
+      <Dialog onClose={() => setIsDialogOpen(false)} open={isDialogOpen}>
+        <DialogTitle sx={{m: 0, p: 2}} className="xr-dialog-text-center">
+          FAIR Seamless Streaming Demo
+        </DialogTitle>
+        <IconButton
+          aria-label="close"
+          onClick={() => setIsDialogOpen(false)}
+          sx={{
+            position: 'absolute',
+            right: 8,
+            top: 8,
+            color: (theme) => theme.palette.grey[500],
+          }}>
+          <CloseIcon />
+        </IconButton>
+        <DialogContent
+          dividers
+          className="xr-dialog-container xr-dialog-text-center">
+          <Typography gutterBottom>
+            Welcome to the Seamless team streaming demo experience! In this demo
+            you will experience AI powered text and audio translation in real
+            time.
+          </Typography>
+          <XRConfig {...props} />
+        </DialogContent>
+      </Dialog>
+    </>
+  );
+}

streaming-react-app/src/setURLParam.ts

@@ -0,0 +1,40 @@
+export default function setURLParam<T>(
+  paramName: string,
+  value: T,
+  // If there's no defaultValue specified then we always set the URL param explicitly
+  defaultValue?: T,
+): void {
+  const urlParams = new URLSearchParams(window.location.search);
+  if (defaultValue != null && value === defaultValue) {
+    urlParams.delete(paramName);
+  } else {
+    let stringValue: string;
+
+    switch (typeof value) {
+      case 'string':
+        stringValue = value;
+        break;
+      case 'boolean':
+        stringValue = value ? '1' : '0';
+        break;
+      default:
+        throw new Error(`Unsupported URL param type: ${typeof value}`);
+    }
+
+    if (urlParams.has(paramName)) {
+      urlParams.set(paramName, stringValue);
+    } else {
+      urlParams.append(paramName, stringValue);
+    }
+  }
+
+  const paramStringWithoutQuestionMark = urlParams.toString();
+
+  window.history.replaceState(
+    null,
+    '',
+    `${window.location.pathname}${
+      paramStringWithoutQuestionMark.length > 0 ? '?' : ''
+    }${paramStringWithoutQuestionMark}`,
+  );
+}

streaming-react-app/src/sliceTranslationSentencesUtils.ts

@@ -0,0 +1,30 @@
+import {TranslationSentences} from './types/StreamingTypes';
+
+export function getTotalSentencesLength(
+  translatedSentences: TranslationSentences,
+) {
+  return translatedSentences.reduce((acc, curr) => acc + curr.length, 0);
+}
+
+/**
+ * @returns A new array of strings where the total length of the strings === targetIndex,
+ * aka it's as if we joined all the strings together, called joined.slice(0, targetIndex), and then
+ * split the string back into an array of strings.
+ */
+export function sliceTranslationSentencesUpToIndex(
+  translatedSentences: TranslationSentences,
+  targetIndex: number,
+): TranslationSentences {
+  return translatedSentences.reduce<TranslationSentences>((acc, sentence) => {
+    const accTotalLength = getTotalSentencesLength(acc);
+    if (accTotalLength === targetIndex) {
+      return acc;
+    }
+    // If adding the current sentence does not exceed the targetIndex, then add the whole sentence
+    if (accTotalLength + sentence.length <= targetIndex) {
+      return [...acc, sentence];
+    }
+    // If adding the current sentence DOES exceed the targetIndex, then slice the sentence and add it
+    return [...acc, sentence.slice(0, targetIndex - accTotalLength)];
+  }, []);
+}