Initial checkin

Dockerfile

@@ -1,6 +1,7 @@
-FROM python:3.9
+FROM python:3.10
 
 WORKDIR /code
+RUN apt install -y ffmpeg
 
 COPY ./requirements.txt /code/requirements.txt
 
@@ -8,6 +9,8 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 COPY . .
 
+RUN python preload.py
+
 EXPOSE 7860
 
-CMD ["shiny", "run", "app.py", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["shiny", "run", "app.py", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,20 +1,34 @@
----
-title: Multimodal
-emoji: 🌍
-colorFrom: yellow
-colorTo: indigo
-sdk: docker
-pinned: false
-license: mit
----
+# Video in, audio out
 
-This is a templated Space for [Shiny for Python](https://shiny.rstudio.com/py/). 
+This is a [Shiny for Python](https://shiny.posit.co/py/) app for easily interacting with GPT-4o via short webcam recordings.
 
+## Installation
 
-To get started with a new app do the following:
+### ffmpeg
 
-1) Install Shiny with `pip install shiny`
-2) Create a new app with `shiny create .`
-3) Then run the app with `shiny run --reload`
+You will need the `ffmpeg` utility installed. Either use the [official installers](https://ffmpeg.org/download.html), or `brew install ffmpeg` (for macOS brew users) or `choco install ffmpeg` (for Windows chocolatey users).
+
+### OpenAI API key
+
+Create a file called `.env` in the root of the project and add the following line:
+
+```
+OPENAI_API_KEY=<your-api-key>
+```
+
+If you have an OpenAI account, you can generate an API key from [this page](https://platform.openai.com/api-keys).
+
+### Python dependencies
+
+```
+pip install -r requirements.txt
+```
+
+## Usage
+
+```
+shiny run app.py --port 0 --launch-browser
+```
+
+This will launch a browser window with a video preview. Press Record, speak your prompt, and press Stop. The video will be processed and the response will be read aloud.
 
-To learn more about this framework please see the [Documentation](https://shiny.rstudio.com/py/docs/overview.html).

app.py

@@ -1,151 +1,53 @@
-from pathlib import Path
-from typing import List, Dict, Tuple
-import matplotlib.colors as mpl_colors
+import base64
+import tempfile
 
-import pandas as pd
-import seaborn as sns
-import shinyswatch
+from openai import AsyncOpenAI
+from shiny.express import input, render, ui
 
-from shiny import App, Inputs, Outputs, Session, reactive, render, req, ui
+from videoinput import audio_spinner, input_video_clip, process_video
 
-sns.set_theme()
+client = AsyncOpenAI()
 
-www_dir = Path(__file__).parent.resolve() / "www"
+ui.page_opts(class_="py-5")
 
-df = pd.read_csv(Path(__file__).parent / "penguins.csv", na_values="NA")
-numeric_cols: List[str] = df.select_dtypes(include=["float64"]).columns.tolist()
-species: List[str] = df["Species"].unique().tolist()
-species.sort()
+input_video_clip("clip")
 
-app_ui = ui.page_fillable(
-    shinyswatch.theme.minty(),
-    ui.layout_sidebar(
-        ui.sidebar(
-            # Artwork by @allison_horst
-            ui.input_selectize(
-                "xvar",
-                "X variable",
-                numeric_cols,
-                selected="Bill Length (mm)",
-            ),
-            ui.input_selectize(
-                "yvar",
-                "Y variable",
-                numeric_cols,
-                selected="Bill Depth (mm)",
-            ),
-            ui.input_checkbox_group(
-                "species", "Filter by species", species, selected=species
-            ),
-            ui.hr(),
-            ui.input_switch("by_species", "Show species", value=True),
-            ui.input_switch("show_margins", "Show marginal plots", value=True),
-        ),
-        ui.output_ui("value_boxes"),
-        ui.output_plot("scatter", fill=True),
-        ui.help_text(
-            "Artwork by ",
-            ui.a("@allison_horst", href="https://twitter.com/allison_horst"),
-            class_="text-end",
-        ),
-    ),
-)
 
+@render.ui
+async def show_clip():
+    clip = input.clip()
+    mime_type = clip["type"]
+    bytes = base64.b64decode(clip["bytes"])
+    # TODO: Use correct file extension based on mime type
+    with tempfile.TemporaryDirectory() as tempdir:
+        filename = tempfile.mktemp(dir=tempdir, suffix=get_video_extension(mime_type))
+        with open(filename, "wb") as file:
+            file.write(bytes)
+            file.close()
 
-def server(input: Inputs, output: Outputs, session: Session):
-    @reactive.Calc
-    def filtered_df() -> pd.DataFrame:
-        """Returns a Pandas data frame that includes only the desired rows"""
+        with ui.Progress() as p:
 
-        # This calculation "req"uires that at least one species is selected
-        req(len(input.species()) > 0)
-
-        # Filter the rows so we only include the desired species
-        return df[df["Species"].isin(input.species())]
-
-    @output
-    @render.plot
-    def scatter():
-        """Generates a plot for Shiny to display to the user"""
-
-        # The plotting function to use depends on whether margins are desired
-        plotfunc = sns.jointplot if input.show_margins() else sns.scatterplot
-
-        plotfunc(
-            data=filtered_df(),
-            x=input.xvar(),
-            y=input.yvar(),
-            palette=palette,
-            hue="Species" if input.by_species() else None,
-            hue_order=species,
-            legend=False,
-        )
-
-    @output
-    @render.ui
-    def value_boxes():
-        df = filtered_df()
-
-        def penguin_value_box(title: str, count: int, bgcol: str, showcase_img: str):
-            return ui.value_box(
-                title,
-                count,
-                {"class_": "pt-1 pb-0"},
-                showcase=ui.fill.as_fill_item(
-                    ui.tags.img(
-                        {"style": "object-fit:contain;"},
-                        src=showcase_img,
-                    )
-                ),
-                theme_color=None,
-                style=f"background-color: {bgcol};",
+            mp3_data_uri = await process_video(
+                client,
+                filename,
+                callback=lambda status: p.set(message=status),
             )
+            return audio_spinner(src=mp3_data_uri)
 
-        if not input.by_species():
-            return penguin_value_box(
-                "Penguins",
-                len(df.index),
-                bg_palette["default"],
-                # Artwork by @allison_horst
-                showcase_img="penguins.png",
-            )
-
-        value_boxes = [
-            penguin_value_box(
-                name,
-                len(df[df["Species"] == name]),
-                bg_palette[name],
-                # Artwork by @allison_horst
-                showcase_img=f"{name}.png",
-            )
-            for name in species
-            # Only include boxes for _selected_ species
-            if name in input.species()
-        ]
-
-        return ui.layout_column_wrap(*value_boxes, width = 1 / len(value_boxes))
-
-
-# "darkorange", "purple", "cyan4"
-colors = [[255, 140, 0], [160, 32, 240], [0, 139, 139]]
-colors = [(r / 255.0, g / 255.0, b / 255.0) for r, g, b in colors]
-
-palette: Dict[str, Tuple[float, float, float]] = {
-    "Adelie": colors[0],
-    "Chinstrap": colors[1],
-    "Gentoo": colors[2],
-    "default": sns.color_palette()[0],  # type: ignore
-}
 
-bg_palette = {}
-# Use `sns.set_style("whitegrid")` to help find approx alpha value
-for name, col in palette.items():
-    # Adjusted n_colors until `axe` accessibility did not complain about color contrast
-    bg_palette[name] = mpl_colors.to_hex(sns.light_palette(col, n_colors=7)[1])  # type: ignore
+def get_video_extension(mime_type: str) -> str:
+    mime_type = mime_type.split(";")[0].strip()
 
+    # Dictionary to map MIME types to file extensions
+    mime_to_extension = {
+        "video/webm": ".webm",
+        "video/mp4": ".mp4",
+        "video/ogg": ".ogv",
+        "video/x-matroska": ".mkv",
+        "video/avi": ".avi",
+        "video/mpeg": ".mpeg",
+        "video/quicktime": ".mov",
+    }
 
-app = App(
-    app_ui,
-    server,
-    static_assets=str(www_dir),
-)
+    # Return the appropriate file extension for the given MIME type
+    return mime_to_extension.get(mime_type, "")

penguins.csv

@@ -1,345 +0,0 @@
-Species,Island,Bill Length (mm),Bill Depth (mm),Flipper Length (mm),Body Mass (g),Sex,Year
-Adelie,Torgersen,39.1,18.7,181,3750,male,2007
-Adelie,Torgersen,39.5,17.4,186,3800,female,2007
-Adelie,Torgersen,40.3,18,195,3250,female,2007
-Adelie,Torgersen,NA,NA,NA,NA,NA,2007
-Adelie,Torgersen,36.7,19.3,193,3450,female,2007
-Adelie,Torgersen,39.3,20.6,190,3650,male,2007
-Adelie,Torgersen,38.9,17.8,181,3625,female,2007
-Adelie,Torgersen,39.2,19.6,195,4675,male,2007
-Adelie,Torgersen,34.1,18.1,193,3475,NA,2007
-Adelie,Torgersen,42,20.2,190,4250,NA,2007
-Adelie,Torgersen,37.8,17.1,186,3300,NA,2007
-Adelie,Torgersen,37.8,17.3,180,3700,NA,2007
-Adelie,Torgersen,41.1,17.6,182,3200,female,2007
-Adelie,Torgersen,38.6,21.2,191,3800,male,2007
-Adelie,Torgersen,34.6,21.1,198,4400,male,2007
-Adelie,Torgersen,36.6,17.8,185,3700,female,2007
-Adelie,Torgersen,38.7,19,195,3450,female,2007
-Adelie,Torgersen,42.5,20.7,197,4500,male,2007
-Adelie,Torgersen,34.4,18.4,184,3325,female,2007
-Adelie,Torgersen,46,21.5,194,4200,male,2007
-Adelie,Biscoe,37.8,18.3,174,3400,female,2007
-Adelie,Biscoe,37.7,18.7,180,3600,male,2007
-Adelie,Biscoe,35.9,19.2,189,3800,female,2007
-Adelie,Biscoe,38.2,18.1,185,3950,male,2007
-Adelie,Biscoe,38.8,17.2,180,3800,male,2007
-Adelie,Biscoe,35.3,18.9,187,3800,female,2007
-Adelie,Biscoe,40.6,18.6,183,3550,male,2007
-Adelie,Biscoe,40.5,17.9,187,3200,female,2007
-Adelie,Biscoe,37.9,18.6,172,3150,female,2007
-Adelie,Biscoe,40.5,18.9,180,3950,male,2007
-Adelie,Dream,39.5,16.7,178,3250,female,2007
-Adelie,Dream,37.2,18.1,178,3900,male,2007
-Adelie,Dream,39.5,17.8,188,3300,female,2007
-Adelie,Dream,40.9,18.9,184,3900,male,2007
-Adelie,Dream,36.4,17,195,3325,female,2007
-Adelie,Dream,39.2,21.1,196,4150,male,2007
-Adelie,Dream,38.8,20,190,3950,male,2007
-Adelie,Dream,42.2,18.5,180,3550,female,2007
-Adelie,Dream,37.6,19.3,181,3300,female,2007
-Adelie,Dream,39.8,19.1,184,4650,male,2007
-Adelie,Dream,36.5,18,182,3150,female,2007
-Adelie,Dream,40.8,18.4,195,3900,male,2007
-Adelie,Dream,36,18.5,186,3100,female,2007
-Adelie,Dream,44.1,19.7,196,4400,male,2007
-Adelie,Dream,37,16.9,185,3000,female,2007
-Adelie,Dream,39.6,18.8,190,4600,male,2007
-Adelie,Dream,41.1,19,182,3425,male,2007
-Adelie,Dream,37.5,18.9,179,2975,NA,2007
-Adelie,Dream,36,17.9,190,3450,female,2007
-Adelie,Dream,42.3,21.2,191,4150,male,2007
-Adelie,Biscoe,39.6,17.7,186,3500,female,2008
-Adelie,Biscoe,40.1,18.9,188,4300,male,2008
-Adelie,Biscoe,35,17.9,190,3450,female,2008
-Adelie,Biscoe,42,19.5,200,4050,male,2008
-Adelie,Biscoe,34.5,18.1,187,2900,female,2008
-Adelie,Biscoe,41.4,18.6,191,3700,male,2008
-Adelie,Biscoe,39,17.5,186,3550,female,2008
-Adelie,Biscoe,40.6,18.8,193,3800,male,2008
-Adelie,Biscoe,36.5,16.6,181,2850,female,2008
-Adelie,Biscoe,37.6,19.1,194,3750,male,2008
-Adelie,Biscoe,35.7,16.9,185,3150,female,2008
-Adelie,Biscoe,41.3,21.1,195,4400,male,2008
-Adelie,Biscoe,37.6,17,185,3600,female,2008
-Adelie,Biscoe,41.1,18.2,192,4050,male,2008
-Adelie,Biscoe,36.4,17.1,184,2850,female,2008
-Adelie,Biscoe,41.6,18,192,3950,male,2008
-Adelie,Biscoe,35.5,16.2,195,3350,female,2008
-Adelie,Biscoe,41.1,19.1,188,4100,male,2008
-Adelie,Torgersen,35.9,16.6,190,3050,female,2008
-Adelie,Torgersen,41.8,19.4,198,4450,male,2008
-Adelie,Torgersen,33.5,19,190,3600,female,2008
-Adelie,Torgersen,39.7,18.4,190,3900,male,2008
-Adelie,Torgersen,39.6,17.2,196,3550,female,2008
-Adelie,Torgersen,45.8,18.9,197,4150,male,2008
-Adelie,Torgersen,35.5,17.5,190,3700,female,2008
-Adelie,Torgersen,42.8,18.5,195,4250,male,2008
-Adelie,Torgersen,40.9,16.8,191,3700,female,2008
-Adelie,Torgersen,37.2,19.4,184,3900,male,2008
-Adelie,Torgersen,36.2,16.1,187,3550,female,2008
-Adelie,Torgersen,42.1,19.1,195,4000,male,2008
-Adelie,Torgersen,34.6,17.2,189,3200,female,2008
-Adelie,Torgersen,42.9,17.6,196,4700,male,2008
-Adelie,Torgersen,36.7,18.8,187,3800,female,2008
-Adelie,Torgersen,35.1,19.4,193,4200,male,2008
-Adelie,Dream,37.3,17.8,191,3350,female,2008
-Adelie,Dream,41.3,20.3,194,3550,male,2008
-Adelie,Dream,36.3,19.5,190,3800,male,2008
-Adelie,Dream,36.9,18.6,189,3500,female,2008
-Adelie,Dream,38.3,19.2,189,3950,male,2008
-Adelie,Dream,38.9,18.8,190,3600,female,2008
-Adelie,Dream,35.7,18,202,3550,female,2008
-Adelie,Dream,41.1,18.1,205,4300,male,2008
-Adelie,Dream,34,17.1,185,3400,female,2008
-Adelie,Dream,39.6,18.1,186,4450,male,2008
-Adelie,Dream,36.2,17.3,187,3300,female,2008
-Adelie,Dream,40.8,18.9,208,4300,male,2008
-Adelie,Dream,38.1,18.6,190,3700,female,2008
-Adelie,Dream,40.3,18.5,196,4350,male,2008
-Adelie,Dream,33.1,16.1,178,2900,female,2008
-Adelie,Dream,43.2,18.5,192,4100,male,2008
-Adelie,Biscoe,35,17.9,192,3725,female,2009
-Adelie,Biscoe,41,20,203,4725,male,2009
-Adelie,Biscoe,37.7,16,183,3075,female,2009
-Adelie,Biscoe,37.8,20,190,4250,male,2009
-Adelie,Biscoe,37.9,18.6,193,2925,female,2009
-Adelie,Biscoe,39.7,18.9,184,3550,male,2009
-Adelie,Biscoe,38.6,17.2,199,3750,female,2009
-Adelie,Biscoe,38.2,20,190,3900,male,2009
-Adelie,Biscoe,38.1,17,181,3175,female,2009
-Adelie,Biscoe,43.2,19,197,4775,male,2009
-Adelie,Biscoe,38.1,16.5,198,3825,female,2009
-Adelie,Biscoe,45.6,20.3,191,4600,male,2009
-Adelie,Biscoe,39.7,17.7,193,3200,female,2009
-Adelie,Biscoe,42.2,19.5,197,4275,male,2009
-Adelie,Biscoe,39.6,20.7,191,3900,female,2009
-Adelie,Biscoe,42.7,18.3,196,4075,male,2009
-Adelie,Torgersen,38.6,17,188,2900,female,2009
-Adelie,Torgersen,37.3,20.5,199,3775,male,2009
-Adelie,Torgersen,35.7,17,189,3350,female,2009
-Adelie,Torgersen,41.1,18.6,189,3325,male,2009
-Adelie,Torgersen,36.2,17.2,187,3150,female,2009
-Adelie,Torgersen,37.7,19.8,198,3500,male,2009
-Adelie,Torgersen,40.2,17,176,3450,female,2009
-Adelie,Torgersen,41.4,18.5,202,3875,male,2009
-Adelie,Torgersen,35.2,15.9,186,3050,female,2009
-Adelie,Torgersen,40.6,19,199,4000,male,2009
-Adelie,Torgersen,38.8,17.6,191,3275,female,2009
-Adelie,Torgersen,41.5,18.3,195,4300,male,2009
-Adelie,Torgersen,39,17.1,191,3050,female,2009
-Adelie,Torgersen,44.1,18,210,4000,male,2009
-Adelie,Torgersen,38.5,17.9,190,3325,female,2009
-Adelie,Torgersen,43.1,19.2,197,3500,male,2009
-Adelie,Dream,36.8,18.5,193,3500,female,2009
-Adelie,Dream,37.5,18.5,199,4475,male,2009
-Adelie,Dream,38.1,17.6,187,3425,female,2009
-Adelie,Dream,41.1,17.5,190,3900,male,2009
-Adelie,Dream,35.6,17.5,191,3175,female,2009
-Adelie,Dream,40.2,20.1,200,3975,male,2009
-Adelie,Dream,37,16.5,185,3400,female,2009
-Adelie,Dream,39.7,17.9,193,4250,male,2009
-Adelie,Dream,40.2,17.1,193,3400,female,2009
-Adelie,Dream,40.6,17.2,187,3475,male,2009
-Adelie,Dream,32.1,15.5,188,3050,female,2009
-Adelie,Dream,40.7,17,190,3725,male,2009
-Adelie,Dream,37.3,16.8,192,3000,female,2009
-Adelie,Dream,39,18.7,185,3650,male,2009
-Adelie,Dream,39.2,18.6,190,4250,male,2009
-Adelie,Dream,36.6,18.4,184,3475,female,2009
-Adelie,Dream,36,17.8,195,3450,female,2009
-Adelie,Dream,37.8,18.1,193,3750,male,2009
-Adelie,Dream,36,17.1,187,3700,female,2009
-Adelie,Dream,41.5,18.5,201,4000,male,2009
-Gentoo,Biscoe,46.1,13.2,211,4500,female,2007
-Gentoo,Biscoe,50,16.3,230,5700,male,2007
-Gentoo,Biscoe,48.7,14.1,210,4450,female,2007
-Gentoo,Biscoe,50,15.2,218,5700,male,2007
-Gentoo,Biscoe,47.6,14.5,215,5400,male,2007
-Gentoo,Biscoe,46.5,13.5,210,4550,female,2007
-Gentoo,Biscoe,45.4,14.6,211,4800,female,2007
-Gentoo,Biscoe,46.7,15.3,219,5200,male,2007
-Gentoo,Biscoe,43.3,13.4,209,4400,female,2007
-Gentoo,Biscoe,46.8,15.4,215,5150,male,2007
-Gentoo,Biscoe,40.9,13.7,214,4650,female,2007
-Gentoo,Biscoe,49,16.1,216,5550,male,2007
-Gentoo,Biscoe,45.5,13.7,214,4650,female,2007
-Gentoo,Biscoe,48.4,14.6,213,5850,male,2007
-Gentoo,Biscoe,45.8,14.6,210,4200,female,2007
-Gentoo,Biscoe,49.3,15.7,217,5850,male,2007
-Gentoo,Biscoe,42,13.5,210,4150,female,2007
-Gentoo,Biscoe,49.2,15.2,221,6300,male,2007
-Gentoo,Biscoe,46.2,14.5,209,4800,female,2007
-Gentoo,Biscoe,48.7,15.1,222,5350,male,2007
-Gentoo,Biscoe,50.2,14.3,218,5700,male,2007
-Gentoo,Biscoe,45.1,14.5,215,5000,female,2007
-Gentoo,Biscoe,46.5,14.5,213,4400,female,2007
-Gentoo,Biscoe,46.3,15.8,215,5050,male,2007
-Gentoo,Biscoe,42.9,13.1,215,5000,female,2007
-Gentoo,Biscoe,46.1,15.1,215,5100,male,2007
-Gentoo,Biscoe,44.5,14.3,216,4100,NA,2007
-Gentoo,Biscoe,47.8,15,215,5650,male,2007
-Gentoo,Biscoe,48.2,14.3,210,4600,female,2007
-Gentoo,Biscoe,50,15.3,220,5550,male,2007
-Gentoo,Biscoe,47.3,15.3,222,5250,male,2007
-Gentoo,Biscoe,42.8,14.2,209,4700,female,2007
-Gentoo,Biscoe,45.1,14.5,207,5050,female,2007
-Gentoo,Biscoe,59.6,17,230,6050,male,2007
-Gentoo,Biscoe,49.1,14.8,220,5150,female,2008
-Gentoo,Biscoe,48.4,16.3,220,5400,male,2008
-Gentoo,Biscoe,42.6,13.7,213,4950,female,2008
-Gentoo,Biscoe,44.4,17.3,219,5250,male,2008
-Gentoo,Biscoe,44,13.6,208,4350,female,2008
-Gentoo,Biscoe,48.7,15.7,208,5350,male,2008
-Gentoo,Biscoe,42.7,13.7,208,3950,female,2008
-Gentoo,Biscoe,49.6,16,225,5700,male,2008
-Gentoo,Biscoe,45.3,13.7,210,4300,female,2008
-Gentoo,Biscoe,49.6,15,216,4750,male,2008
-Gentoo,Biscoe,50.5,15.9,222,5550,male,2008
-Gentoo,Biscoe,43.6,13.9,217,4900,female,2008
-Gentoo,Biscoe,45.5,13.9,210,4200,female,2008
-Gentoo,Biscoe,50.5,15.9,225,5400,male,2008
-Gentoo,Biscoe,44.9,13.3,213,5100,female,2008
-Gentoo,Biscoe,45.2,15.8,215,5300,male,2008
-Gentoo,Biscoe,46.6,14.2,210,4850,female,2008
-Gentoo,Biscoe,48.5,14.1,220,5300,male,2008
-Gentoo,Biscoe,45.1,14.4,210,4400,female,2008
-Gentoo,Biscoe,50.1,15,225,5000,male,2008
-Gentoo,Biscoe,46.5,14.4,217,4900,female,2008
-Gentoo,Biscoe,45,15.4,220,5050,male,2008
-Gentoo,Biscoe,43.8,13.9,208,4300,female,2008
-Gentoo,Biscoe,45.5,15,220,5000,male,2008
-Gentoo,Biscoe,43.2,14.5,208,4450,female,2008
-Gentoo,Biscoe,50.4,15.3,224,5550,male,2008
-Gentoo,Biscoe,45.3,13.8,208,4200,female,2008
-Gentoo,Biscoe,46.2,14.9,221,5300,male,2008
-Gentoo,Biscoe,45.7,13.9,214,4400,female,2008
-Gentoo,Biscoe,54.3,15.7,231,5650,male,2008
-Gentoo,Biscoe,45.8,14.2,219,4700,female,2008
-Gentoo,Biscoe,49.8,16.8,230,5700,male,2008
-Gentoo,Biscoe,46.2,14.4,214,4650,NA,2008
-Gentoo,Biscoe,49.5,16.2,229,5800,male,2008
-Gentoo,Biscoe,43.5,14.2,220,4700,female,2008
-Gentoo,Biscoe,50.7,15,223,5550,male,2008
-Gentoo,Biscoe,47.7,15,216,4750,female,2008
-Gentoo,Biscoe,46.4,15.6,221,5000,male,2008
-Gentoo,Biscoe,48.2,15.6,221,5100,male,2008
-Gentoo,Biscoe,46.5,14.8,217,5200,female,2008
-Gentoo,Biscoe,46.4,15,216,4700,female,2008
-Gentoo,Biscoe,48.6,16,230,5800,male,2008
-Gentoo,Biscoe,47.5,14.2,209,4600,female,2008
-Gentoo,Biscoe,51.1,16.3,220,6000,male,2008
-Gentoo,Biscoe,45.2,13.8,215,4750,female,2008
-Gentoo,Biscoe,45.2,16.4,223,5950,male,2008
-Gentoo,Biscoe,49.1,14.5,212,4625,female,2009
-Gentoo,Biscoe,52.5,15.6,221,5450,male,2009
-Gentoo,Biscoe,47.4,14.6,212,4725,female,2009
-Gentoo,Biscoe,50,15.9,224,5350,male,2009
-Gentoo,Biscoe,44.9,13.8,212,4750,female,2009
-Gentoo,Biscoe,50.8,17.3,228,5600,male,2009
-Gentoo,Biscoe,43.4,14.4,218,4600,female,2009
-Gentoo,Biscoe,51.3,14.2,218,5300,male,2009
-Gentoo,Biscoe,47.5,14,212,4875,female,2009
-Gentoo,Biscoe,52.1,17,230,5550,male,2009
-Gentoo,Biscoe,47.5,15,218,4950,female,2009
-Gentoo,Biscoe,52.2,17.1,228,5400,male,2009
-Gentoo,Biscoe,45.5,14.5,212,4750,female,2009
-Gentoo,Biscoe,49.5,16.1,224,5650,male,2009
-Gentoo,Biscoe,44.5,14.7,214,4850,female,2009
-Gentoo,Biscoe,50.8,15.7,226,5200,male,2009
-Gentoo,Biscoe,49.4,15.8,216,4925,male,2009
-Gentoo,Biscoe,46.9,14.6,222,4875,female,2009
-Gentoo,Biscoe,48.4,14.4,203,4625,female,2009
-Gentoo,Biscoe,51.1,16.5,225,5250,male,2009
-Gentoo,Biscoe,48.5,15,219,4850,female,2009
-Gentoo,Biscoe,55.9,17,228,5600,male,2009
-Gentoo,Biscoe,47.2,15.5,215,4975,female,2009
-Gentoo,Biscoe,49.1,15,228,5500,male,2009
-Gentoo,Biscoe,47.3,13.8,216,4725,NA,2009
-Gentoo,Biscoe,46.8,16.1,215,5500,male,2009
-Gentoo,Biscoe,41.7,14.7,210,4700,female,2009
-Gentoo,Biscoe,53.4,15.8,219,5500,male,2009
-Gentoo,Biscoe,43.3,14,208,4575,female,2009
-Gentoo,Biscoe,48.1,15.1,209,5500,male,2009
-Gentoo,Biscoe,50.5,15.2,216,5000,female,2009
-Gentoo,Biscoe,49.8,15.9,229,5950,male,2009
-Gentoo,Biscoe,43.5,15.2,213,4650,female,2009
-Gentoo,Biscoe,51.5,16.3,230,5500,male,2009
-Gentoo,Biscoe,46.2,14.1,217,4375,female,2009
-Gentoo,Biscoe,55.1,16,230,5850,male,2009
-Gentoo,Biscoe,44.5,15.7,217,4875,NA,2009
-Gentoo,Biscoe,48.8,16.2,222,6000,male,2009
-Gentoo,Biscoe,47.2,13.7,214,4925,female,2009
-Gentoo,Biscoe,NA,NA,NA,NA,NA,2009
-Gentoo,Biscoe,46.8,14.3,215,4850,female,2009
-Gentoo,Biscoe,50.4,15.7,222,5750,male,2009
-Gentoo,Biscoe,45.2,14.8,212,5200,female,2009
-Gentoo,Biscoe,49.9,16.1,213,5400,male,2009
-Chinstrap,Dream,46.5,17.9,192,3500,female,2007
-Chinstrap,Dream,50,19.5,196,3900,male,2007
-Chinstrap,Dream,51.3,19.2,193,3650,male,2007
-Chinstrap,Dream,45.4,18.7,188,3525,female,2007
-Chinstrap,Dream,52.7,19.8,197,3725,male,2007
-Chinstrap,Dream,45.2,17.8,198,3950,female,2007
-Chinstrap,Dream,46.1,18.2,178,3250,female,2007
-Chinstrap,Dream,51.3,18.2,197,3750,male,2007
-Chinstrap,Dream,46,18.9,195,4150,female,2007
-Chinstrap,Dream,51.3,19.9,198,3700,male,2007
-Chinstrap,Dream,46.6,17.8,193,3800,female,2007
-Chinstrap,Dream,51.7,20.3,194,3775,male,2007
-Chinstrap,Dream,47,17.3,185,3700,female,2007
-Chinstrap,Dream,52,18.1,201,4050,male,2007
-Chinstrap,Dream,45.9,17.1,190,3575,female,2007
-Chinstrap,Dream,50.5,19.6,201,4050,male,2007
-Chinstrap,Dream,50.3,20,197,3300,male,2007
-Chinstrap,Dream,58,17.8,181,3700,female,2007
-Chinstrap,Dream,46.4,18.6,190,3450,female,2007
-Chinstrap,Dream,49.2,18.2,195,4400,male,2007
-Chinstrap,Dream,42.4,17.3,181,3600,female,2007
-Chinstrap,Dream,48.5,17.5,191,3400,male,2007
-Chinstrap,Dream,43.2,16.6,187,2900,female,2007
-Chinstrap,Dream,50.6,19.4,193,3800,male,2007
-Chinstrap,Dream,46.7,17.9,195,3300,female,2007
-Chinstrap,Dream,52,19,197,4150,male,2007
-Chinstrap,Dream,50.5,18.4,200,3400,female,2008
-Chinstrap,Dream,49.5,19,200,3800,male,2008
-Chinstrap,Dream,46.4,17.8,191,3700,female,2008
-Chinstrap,Dream,52.8,20,205,4550,male,2008
-Chinstrap,Dream,40.9,16.6,187,3200,female,2008
-Chinstrap,Dream,54.2,20.8,201,4300,male,2008
-Chinstrap,Dream,42.5,16.7,187,3350,female,2008
-Chinstrap,Dream,51,18.8,203,4100,male,2008
-Chinstrap,Dream,49.7,18.6,195,3600,male,2008
-Chinstrap,Dream,47.5,16.8,199,3900,female,2008
-Chinstrap,Dream,47.6,18.3,195,3850,female,2008
-Chinstrap,Dream,52,20.7,210,4800,male,2008
-Chinstrap,Dream,46.9,16.6,192,2700,female,2008
-Chinstrap,Dream,53.5,19.9,205,4500,male,2008
-Chinstrap,Dream,49,19.5,210,3950,male,2008
-Chinstrap,Dream,46.2,17.5,187,3650,female,2008
-Chinstrap,Dream,50.9,19.1,196,3550,male,2008
-Chinstrap,Dream,45.5,17,196,3500,female,2008
-Chinstrap,Dream,50.9,17.9,196,3675,female,2009
-Chinstrap,Dream,50.8,18.5,201,4450,male,2009
-Chinstrap,Dream,50.1,17.9,190,3400,female,2009
-Chinstrap,Dream,49,19.6,212,4300,male,2009
-Chinstrap,Dream,51.5,18.7,187,3250,male,2009
-Chinstrap,Dream,49.8,17.3,198,3675,female,2009
-Chinstrap,Dream,48.1,16.4,199,3325,female,2009
-Chinstrap,Dream,51.4,19,201,3950,male,2009
-Chinstrap,Dream,45.7,17.3,193,3600,female,2009
-Chinstrap,Dream,50.7,19.7,203,4050,male,2009
-Chinstrap,Dream,42.5,17.3,187,3350,female,2009
-Chinstrap,Dream,52.2,18.8,197,3450,male,2009
-Chinstrap,Dream,45.2,16.6,191,3250,female,2009
-Chinstrap,Dream,49.3,19.9,203,4050,male,2009
-Chinstrap,Dream,50.2,18.8,202,3800,male,2009
-Chinstrap,Dream,45.6,19.4,194,3525,female,2009
-Chinstrap,Dream,51.9,19.5,206,3950,male,2009
-Chinstrap,Dream,46.8,16.5,189,3650,female,2009
-Chinstrap,Dream,45.7,17,195,3650,female,2009
-Chinstrap,Dream,55.8,19.8,207,4000,male,2009
-Chinstrap,Dream,43.5,18.1,202,3400,female,2009
-Chinstrap,Dream,49.6,18.2,193,3775,male,2009
-Chinstrap,Dream,50.8,19,210,4100,male,2009
-Chinstrap,Dream,50.2,18.7,198,3775,female,2009

preload.py

@@ -0,0 +1,4 @@
+from transformers import AutoProcessor, AutoModelForVision2Seq
+
+processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics2-8b")
+model = AutoModelForVision2Seq.from_pretrained("HuggingFaceM4/idefics2-8b")

requirements.txt

@@ -1,4 +1,12 @@
-shiny==0.9.0
-shinyswatch==0.6.0
-seaborn==0.12.2
-matplotlib==3.7.1
+--extra-index-url https://pypi.ngc.nvidia.com
+colorama
+faicons
+ffmpeg-python
+openai
+python-dotenv
+shiny
+nvidia-cuda-runtime-cu12
+nvidia-cudnn
+requests
+torch
+transformers

videoinput/__init__.py

@@ -0,0 +1,8 @@
+from .ui import input_video_clip, audio_spinner
+from .query import process_video
+
+__all__ = (
+    "input_video_clip",
+    "audio_spinner",
+    "process_video",
+)

videoinput/dist/index.css

@@ -0,0 +1,12 @@
+av-settings-menu {
+  display: block;
+  width: min-content;
+}
+
+/* The normal treatment of .dropdown-item.active is a little too much */
+av-settings-menu .dropdown-item.active,
+av-settings-menu .dropdown-menu > li > a.active {
+  color: inherit;
+  background-color: inherit;
+  font-weight: bold;
+}

videoinput/dist/index.js

@@ -0,0 +1,518 @@
+"use strict";
+
+// srcts/videoClipper.ts
+var VideoClipperElement = class extends HTMLElement {
+  constructor() {
+    super();
+    this.chunks = [];
+    this.attachShadow({ mode: "open" });
+    this.shadowRoot.innerHTML = `
+        <style>
+          :host {
+            display: grid;
+            grid-template-rows: 1fr;
+            grid-template-columns: 1fr;
+            width: 100%;
+            height: min-content;
+          }
+          video {
+            grid-column: 1 / 2;
+            grid-row: 1 / 2;
+            width: 100%;
+            object-fit: cover;
+            background-color: var(--video-clip-bg, black);
+            aspect-ratio: 16 / 9;
+            border-radius: var(--video-clip-border-radius, var(--bs-border-radius-lg));
+          }
+          video.mirrored {
+            transform: scaleX(-1);
+          }
+          .panel-settings {
+            grid-column: 1 / 2;
+            grid-row: 1 / 2;
+            justify-self: end;
+            margin: 0.5em;
+          }
+          .panel-buttons {
+            grid-column: 1 / 2;
+            grid-row: 1 / 2;
+            justify-self: end;
+            align-self: end;
+            margin: 0.5em;
+          }
+        </style>
+        <video part="video" muted></video>
+        <div class="panel-settings">
+          <slot name="settings"></slot>
+        </div>
+        <div class="panel-buttons">
+          <slot name="recording-controls"></slot>
+        </div>
+    `;
+    this.video = this.shadowRoot.querySelector("video");
+  }
+  connectedCallback() {
+    (async () => {
+      const slotSettings = this.shadowRoot.querySelector(
+        "slot[name=settings]"
+      );
+      slotSettings.addEventListener("slotchange", async () => {
+        this.avSettingsMenu = slotSettings.assignedElements()[0];
+        await this.#initializeMediaInput();
+        if (this.buttonRecord) {
+          this.#setEnabledButton(this.buttonRecord);
+        }
+      });
+      const slotControls = this.shadowRoot.querySelector(
+        "slot[name=recording-controls]"
+      );
+      slotControls.addEventListener("slotchange", () => {
+        const findButton = (selector) => {
+          for (const el of slotControls.assignedElements()) {
+            if (el.matches(selector)) {
+              return el;
+            }
+            const sub = el.querySelector(selector);
+            if (sub) {
+              return sub;
+            }
+          }
+          return null;
+        };
+        this.buttonRecord = findButton(".record-button");
+        this.buttonStop = findButton(".stop-button");
+        this.#setEnabledButton();
+        this.buttonRecord.addEventListener("click", () => {
+          this.#setEnabledButton(this.buttonStop);
+          this._beginRecord();
+        });
+        this.buttonStop.addEventListener("click", () => {
+          this._endRecord();
+          this.#setEnabledButton(this.buttonRecord);
+        });
+      });
+    })().catch((err) => {
+      console.error(err);
+    });
+  }
+  disconnectedCallback() {
+  }
+  #setEnabledButton(btn) {
+    this.buttonRecord.style.display = btn === this.buttonRecord ? "inline-block" : "none";
+    this.buttonStop.style.display = btn === this.buttonStop ? "inline-block" : "none";
+  }
+  async setMediaDevices(cameraId, micId) {
+    if (this.cameraStream) {
+      this.cameraStream.getTracks().forEach((track) => track.stop());
+    }
+    this.cameraStream = await navigator.mediaDevices.getUserMedia({
+      video: {
+        deviceId: cameraId || void 0,
+        facingMode: "user",
+        aspectRatio: 16 / 9
+      },
+      audio: {
+        deviceId: micId || void 0
+      }
+    });
+    const isSelfieCam = true;
+    this.video.classList.toggle("mirrored", isSelfieCam);
+    const aspectRatio = this.cameraStream.getVideoTracks()[0].getSettings().aspectRatio;
+    if (aspectRatio) {
+      this.video.style.aspectRatio = `${aspectRatio}`;
+    } else {
+      this.video.style.aspectRatio = "";
+    }
+    this.video.srcObject = this.cameraStream;
+    this.video.play();
+    return {
+      cameraId: this.cameraStream.getVideoTracks()[0].getSettings().deviceId,
+      micId: this.cameraStream.getAudioTracks()[0].getSettings().deviceId
+    };
+  }
+  async #initializeMediaInput() {
+    const savedCamera = window.localStorage.getItem("multimodal-camera");
+    const savedMic = window.localStorage.getItem("multimodal-mic");
+    const { cameraId, micId } = await this.setMediaDevices(
+      savedCamera,
+      savedMic
+    );
+    const devices = await navigator.mediaDevices.enumerateDevices();
+    this.avSettingsMenu.setCameras(
+      devices.filter((dev) => dev.kind === "videoinput")
+    );
+    this.avSettingsMenu.setMics(
+      devices.filter((dev) => dev.kind === "audioinput")
+    );
+    this.avSettingsMenu.cameraId = cameraId;
+    this.avSettingsMenu.micId = micId;
+    const handleDeviceChange = async (deviceType, deviceId) => {
+      if (!deviceId) return;
+      window.localStorage.setItem(`multimodal-${deviceType}`, deviceId);
+      await this.setMediaDevices(
+        this.avSettingsMenu.cameraId,
+        this.avSettingsMenu.micId
+      );
+    };
+    this.avSettingsMenu.addEventListener("camera-change", (e) => {
+      handleDeviceChange("camera", this.avSettingsMenu.cameraId);
+    });
+    this.avSettingsMenu.addEventListener("mic-change", (e) => {
+      handleDeviceChange("mic", this.avSettingsMenu.micId);
+    });
+  }
+  _beginRecord() {
+    this.recorder = new MediaRecorder(this.cameraStream, {});
+    this.recorder.addEventListener("error", (e) => {
+      console.error("MediaRecorder error:", e.error);
+    });
+    this.recorder.addEventListener("dataavailable", (e) => {
+      this.chunks.push(e.data);
+    });
+    this.recorder.addEventListener("start", () => {
+    });
+    this.recorder.start();
+  }
+  _endRecord(emit = true) {
+    this.recorder.stop();
+    if (!emit) {
+      this.chunks = [];
+    } else {
+      setTimeout(() => {
+        const blob = new Blob(this.chunks, { type: this.chunks[0].type });
+        const event = new BlobEvent("data", {
+          data: blob
+        });
+        try {
+          this.dispatchEvent(event);
+        } finally {
+          this.chunks = [];
+        }
+      }, 0);
+    }
+  }
+};
+customElements.define("video-clipper", VideoClipperElement);
+
+// srcts/avSettingsMenu.ts
+var DeviceChangeEvent = class extends CustomEvent {
+  constructor(type, detail) {
+    super(type, { detail });
+  }
+};
+var AVSettingsMenuElement = class extends HTMLElement {
+  constructor() {
+    super();
+    this.addEventListener("click", (e) => {
+      if (e.target instanceof HTMLAnchorElement) {
+        const a = e.target;
+        if (a.classList.contains("camera-device-item")) {
+          this.cameraId = a.dataset.deviceId;
+        } else if (a.classList.contains("mic-device-item")) {
+          this.micId = a.dataset.deviceId;
+        }
+      }
+    });
+  }
+  #setDevices(deviceType, devices) {
+    const deviceEls = devices.map(
+      (dev) => this.#createDeviceElement(dev, `${deviceType}-device-item`)
+    );
+    const header = this.querySelector(`.${deviceType}-header`);
+    header.after(...deviceEls);
+  }
+  setCameras(cameras) {
+    this.#setDevices("camera", cameras);
+  }
+  setMics(mics) {
+    this.#setDevices("mic", mics);
+  }
+  get cameraId() {
+    return this.#getSelectedDevice("camera");
+  }
+  set cameraId(id) {
+    this.#setSelectedDevice("camera", id);
+  }
+  get micId() {
+    return this.#getSelectedDevice("mic");
+  }
+  set micId(id) {
+    this.#setSelectedDevice("mic", id);
+  }
+  #createDeviceElement(dev, className) {
+    const li = this.ownerDocument.createElement("li");
+    const a = li.appendChild(this.ownerDocument.createElement("a"));
+    a.onclick = (e) => e.preventDefault();
+    a.href = "#";
+    a.textContent = dev.label;
+    a.dataset.deviceId = dev.deviceId;
+    a.className = className;
+    return li;
+  }
+  #getSelectedDevice(device) {
+    return this.querySelector(
+      `a.${device}-device-item.active`
+    )?.dataset.deviceId ?? null;
+  }
+  #setSelectedDevice(device, id) {
+    this.querySelectorAll(`a.${device}-device-item.active`).forEach(
+      (a) => a.classList.remove("active")
+    );
+    if (id) {
+      this.querySelector(
+        `a.${device}-device-item[data-device-id="${id}"]`
+      ).classList.add("active");
+    }
+    this.dispatchEvent(
+      new DeviceChangeEvent(`${device}-change`, {
+        deviceId: id
+      })
+    );
+  }
+};
+customElements.define("av-settings-menu", AVSettingsMenuElement);
+
+// srcts/audioSpinner.ts
+var AudioSpinnerElement = class extends HTMLElement {
+  #audio;
+  #canvas;
+  #ctx2d;
+  #analyzer;
+  #dataArray;
+  #smoother;
+  constructor() {
+    super();
+    this.attachShadow({ mode: "open" });
+    this.shadowRoot.innerHTML = `
+        <style>
+          :host {
+            display: block;
+            position: relative;
+          }
+          ::slotted(canvas) {
+            position: absolute;
+            top: 0;
+            left: 0;
+          }
+          ::slotted(audio) {
+            display: none;
+          }
+        </style>
+        <slot name="audio"></slot>
+        <slot name="canvas"></slot>
+        `;
+  }
+  connectedCallback() {
+    const audioSlot = this.shadowRoot.querySelector(
+      "slot[name=audio]"
+    );
+    this.#audio = this.ownerDocument.createElement("audio");
+    this.#audio.autoplay = true;
+    this.#audio.controls = false;
+    this.#audio.src = this.getAttribute("src");
+    this.#audio.slot = "audio";
+    audioSlot.assign(this.#audio);
+    this.#audio.addEventListener("play", () => {
+      this.#draw();
+    });
+    this.#audio.onpause = () => {
+      this.style.transition = "opacity 0.5s 1s";
+      this.classList.add("fade");
+      this.addEventListener("transitionend", () => {
+        this.remove();
+      });
+    };
+    const canvasSlot = this.shadowRoot.querySelector(
+      "slot[name=canvas]"
+    );
+    this.#canvas = this.ownerDocument.createElement("canvas");
+    this.#canvas.slot = "canvas";
+    this.#canvas.width = this.clientWidth * window.devicePixelRatio;
+    this.#canvas.height = this.clientHeight * window.devicePixelRatio;
+    this.#canvas.style.width = this.clientWidth + "px";
+    this.#canvas.style.height = this.clientHeight + "px";
+    this.appendChild(this.#canvas);
+    canvasSlot.assign(this.#canvas);
+    this.#ctx2d = this.#canvas.getContext("2d");
+    this.#ctx2d.scale(window.devicePixelRatio, window.devicePixelRatio);
+    new ResizeObserver(() => {
+      this.#canvas.width = this.clientWidth;
+      this.#canvas.height = this.clientHeight;
+    }).observe(this);
+    const audioCtx = new AudioContext();
+    const source = audioCtx.createMediaElementSource(this.#audio);
+    this.#analyzer = new AnalyserNode(audioCtx, {
+      fftSize: 2048
+    });
+    this.#dataArray = new Float32Array(this.#analyzer.frequencyBinCount);
+    source.connect(this.#analyzer);
+    this.#analyzer.connect(audioCtx.destination);
+    const dataArray2 = new Float32Array(this.#analyzer.frequencyBinCount);
+    this.#smoother = new Smoother(5, (samples) => {
+      for (let i = 0; i < dataArray2.length; i++) {
+        dataArray2[i] = 0;
+        for (let j = 0; j < samples.length; j++) {
+          dataArray2[i] += samples[j][i];
+        }
+        dataArray2[i] /= samples.length;
+      }
+      return dataArray2;
+    });
+    this.#draw();
+  }
+  #draw() {
+    if (!this.isConnected) {
+      return;
+    }
+    requestAnimationFrame(() => this.#draw());
+    const width = this.#canvas.width;
+    const height = this.#canvas.height;
+    this.#ctx2d.clearRect(0, 0, width, height);
+    this.#analyzer.getFloatTimeDomainData(this.#dataArray);
+    const smoothed = this.#smoother.add(new Float32Array(this.#dataArray));
+    const {
+      spinVelocity,
+      gap,
+      thickness,
+      minRadius,
+      radiusFactor,
+      steps,
+      blades
+    } = this.#getSettings(width, height);
+    const avg = smoothed.reduce((a, b) => a + Math.abs(b), 0) / smoothed.length * 4;
+    const radius = minRadius + avg * (height - minRadius) / radiusFactor;
+    for (let step = 0; step < steps; step++) {
+      const this_radius = radius - step * (radius / (steps + 1));
+      if (step === steps - 1) {
+        this.#drawPie(width, height, 0, Math.PI * 2, this_radius, thickness);
+      } else {
+        const seconds = (/* @__PURE__ */ new Date()).getTime() / 1e3;
+        const startAngle = seconds * spinVelocity % (Math.PI * 2);
+        for (let blade = 0; blade < blades; blade++) {
+          const angleOffset = Math.PI * 2 / blades * blade;
+          const sweep = Math.PI * 2 / blades - gap;
+          this.#drawPie(
+            width,
+            height,
+            startAngle + angleOffset,
+            sweep,
+            this_radius,
+            thickness
+          );
+        }
+      }
+    }
+  }
+  #drawPie(width, height, startAngle, sweep, radius, thickness) {
+    this.#ctx2d.beginPath();
+    this.#ctx2d.fillStyle = this.#canvas.computedStyleMap().get("color")?.toString();
+    if (!thickness) {
+      this.#ctx2d.moveTo(width / 2, height / 2);
+    }
+    this.#ctx2d.arc(
+      width / 2,
+      height / 2,
+      radius,
+      startAngle,
+      startAngle + sweep
+    );
+    if (!thickness) {
+      this.#ctx2d.lineTo(width / 2, height / 2);
+    } else {
+      this.#ctx2d.arc(
+        width / 2,
+        height / 2,
+        radius - thickness,
+        startAngle + sweep,
+        startAngle,
+        true
+      );
+    }
+    this.#ctx2d.fill();
+  }
+  #getSettings(width, height) {
+    const settings = {
+      spinVelocity: 5,
+      gap: Math.PI / 5,
+      thickness: 2.5,
+      minRadius: Math.min(width, height) / 6,
+      radiusFactor: 1.8,
+      steps: 3,
+      blades: 3
+    };
+    for (const key in settings) {
+      const value = tryParseFloat(this.dataset[key]);
+      if (typeof value !== "undefined") {
+        Object.assign(settings, { [key]: value });
+      }
+    }
+    return settings;
+  }
+};
+window.customElements.define("audio-spinner", AudioSpinnerElement);
+var Smoother = class {
+  #samples = [];
+  #smooth;
+  #size;
+  #pos;
+  constructor(size, smooth) {
+    this.#size = size;
+    this.#pos = 0;
+    this.#smooth = smooth;
+  }
+  add(sample) {
+    this.#samples[this.#pos] = sample;
+    this.#pos = (this.#pos + 1) % this.#size;
+    return this.smoothed();
+  }
+  smoothed() {
+    return this.#smooth(this.#samples);
+  }
+};
+function tryParseFloat(str) {
+  if (typeof str === "undefined") {
+    return void 0;
+  }
+  const parsed = parseFloat(str);
+  return isNaN(parsed) ? void 0 : parsed;
+}
+
+// srcts/index.ts
+var VideoClipperBinding = class extends Shiny.InputBinding {
+  #lastKnownValue = /* @__PURE__ */ new WeakMap();
+  #handlers = /* @__PURE__ */ new WeakMap();
+  find(scope) {
+    return $(scope).find("video-clipper");
+  }
+  getValue(el) {
+    return this.#lastKnownValue.get(el);
+  }
+  subscribe(el, callback) {
+    const handler = async (ev) => {
+      const blob = ev.data;
+      this.#lastKnownValue.set(el, {
+        type: blob.type,
+        bytes: await base64(blob)
+      });
+      callback(true);
+    };
+    el.addEventListener("data", handler);
+    this.#handlers.set(el, handler);
+  }
+  unsubscribe(el) {
+    const handler = this.#handlers.get(el);
+    el.removeEventListener("data", handler);
+    this.#handlers.delete(el);
+  }
+};
+window.Shiny.inputBindings.register(new VideoClipperBinding(), "video-clipper");
+async function base64(blob) {
+  const buf = await blob.arrayBuffer();
+  const results = [];
+  const CHUNKSIZE = 1024;
+  for (let i = 0; i < buf.byteLength; i += CHUNKSIZE) {
+    const chunk = buf.slice(i, i + CHUNKSIZE);
+    results.push(String.fromCharCode(...new Uint8Array(chunk)));
+  }
+  return btoa(results.join(""));
+}

videoinput/hf.py

@@ -0,0 +1,16 @@
+import requests
+import torch
+from PIL import Image
+from io import BytesIO
+
+from transformers import AutoProcessor, AutoModelForVision2Seq
+from transformers.image_utils import load_image
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+image1 = load_image(
+    "https://cdn.pixabay.com/photo/2015/04/23/22/00/tree-736885_960_720.jpg"
+)
+
+processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics2-8b")
+model = AutoModelForVision2Seq.from_pretrained("HuggingFaceM4/idefics2-8b").to(DEVICE)

videoinput/input.py

@@ -0,0 +1,70 @@
+import os
+import tempfile
+from pathlib import Path, PurePath
+
+import ffmpeg
+
+
+class DecodedInput:
+    audio: PurePath
+    images: tuple[PurePath, ...]
+
+    def __init__(
+        self,
+        audio: PurePath,
+        images: tuple[PurePath, ...],
+        tmpdir: tempfile.TemporaryDirectory,
+    ):
+        self.audio = audio
+        self.images = images
+        self.tmpdir = tmpdir
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        print("Cleaning up " + self.tmpdir.name)
+        self.tmpdir.cleanup()
+
+
+def decode_input(input_path: PurePath, fps: int = 2) -> DecodedInput:
+    outdir = tempfile.TemporaryDirectory()
+    audio = PurePath(outdir.name) / "audio.mp3"
+    (
+        ffmpeg.input(
+            str(input_path),
+        )
+        .output(
+            str(audio),
+            loglevel="error",
+            **{
+                # Use 64k bitrate for smaller file
+                "b:a": "64k",
+                # Only output one channel, again for smaller file
+                "ac": "1",
+            },
+        )
+        .run()
+    )
+    (
+        ffmpeg.input(str(input_path))
+        .output(
+            str(PurePath(outdir.name) / "frame-%04d.jpg"),
+            loglevel="error",
+            **{
+                # Use fps as specified, scale image to fit within 512x512
+                "vf": f"fps={fps},scale='if(gt(iw,ih),512,-1)':'if(gt(ih,iw),512,-1)'",
+                "q:v": "20",
+            },
+        )
+        .run()
+    )
+    images = list(Path(outdir.name).glob("*.jpg"))
+    images.sort()
+    return DecodedInput(audio, tuple(images), outdir)
+
+
+if __name__ == "__main__":
+    with decode_input(PurePath("data/question.mov")) as input:
+        print(input.audio)
+        print(input.images)

videoinput/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "multimodal",
+  "version": "1.0.0",
+  "description": "This is a simple script that takes a video file as input, and turns it into text plus images in a form GPT-4o can understand. The response from the model is then read aloud.",
+  "main": "dist/index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "esbuild srcts/index.ts --bundle --platform=node --outfile=dist/index.js",
+    "watch": "esbuild srcts/index.ts --bundle --platform=node --outfile=dist/index.js --watch",
+    "typecheck": "tsc --noEmit"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jcheng5/multimodal.git"
+  },
+  "author": "",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/jcheng5/multimodal/issues"
+  },
+  "homepage": "https://github.com/jcheng5/multimodal#readme",
+  "dependencies": {},
+  "devDependencies": {
+    "@types/jquery": "^3.5.30",
+    "@types/rstudio-shiny": "https://github.com/rstudio/shiny#v1.7.0",
+    "esbuild": "^0.21.3",
+    "typescript": "^5.4.5"
+  }
+}

videoinput/query.py

@@ -0,0 +1,114 @@
+from typing import Callable, Optional
+
+import dotenv
+from openai import AsyncOpenAI
+
+from .input import decode_input
+from .utils import NamedTemporaryFile, file_to_data_uri, timed
+
+# Load OpenAI API key from .env file
+dotenv.load_dotenv()
+
+TERSE_PROMPT = """
+The user you're responding to is EXTREMELY busy and cannot waste a single
+second. Above all, answers must be as concise as possible. Every wasted word
+will result in a huge deduction of points. In fact, use the absolute minimum
+number of words while still technically answering the question. Avoid
+adjectives, adverbs, fill words, and qualifiers.
+"""
+
+EXTRA_TERSE_PROMPT = """
+Definitely don't restate any part of the question in your answer, if it can
+possibly be avoided. Don't speak in complete sentences. Just get to the point as
+quickly as possible.
+"""
+
+SUBJECT_PROMPT = """
+If the user refers to "I" or "me" in the text input, you should assume that's
+referring to the most prominent person in the video.
+
+If the user refers to "you" in the text input, you should assume that's
+referring to you, the AI model.
+"""
+
+VIDEO_PROMPT = """
+The images are frames of a video at 2 frames per second. The user doesn't know
+the video is split into frames, so make sure your video refers to these images
+collectively as "the video", not "the images" or "the video frames".
+"""
+
+SPEAKING_PROMPT = """
+The user is asking you to speak the answer. Make sure your response is in the
+form of a friendly, casual spoken answer, not a formal written one.
+"""
+
+SYSTEM_PROMPT = (
+    VIDEO_PROMPT
+    + SUBJECT_PROMPT
+    + SPEAKING_PROMPT
+    # + TERSE_PROMPT
+    # + EXTRA_TERSE_PROMPT
+)
+
+
+async def process_video(
+    client: AsyncOpenAI, filepath: str, callback: Optional[Callable[[str], None]]
+) -> None:
+    if callback is None:
+        callback = lambda _: None
+
+    callback("Decoding input")
+    input = decode_input(filepath, fps=2)
+
+    with input:
+        callback("Decoding speech")
+        with open(str(input.audio), "rb") as audio_file:
+            transcription = await client.audio.transcriptions.create(
+                model="whisper-1", file=audio_file
+            )
+
+        callback("Processing video")
+        images = [file_to_data_uri(filename, "image/jpeg") for filename in input.images]
+
+        callback("Querying")
+        response = await client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": transcription.text},
+                        *[
+                            {
+                                "type": "image_url",
+                                "image_url": {"url": image, "detail": "auto"},
+                            }
+                            for image in images
+                        ],
+                    ],
+                },
+                {
+                    "role": "system",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": SYSTEM_PROMPT,
+                        }
+                    ],
+                },
+            ],
+        )
+
+        callback("Converting to speech")
+        audio = await client.audio.speech.create(
+            model="tts-1",
+            voice="nova",
+            input=response.choices[0].message.content,
+            response_format="mp3",
+        )
+
+        callback("Encoding audio")
+        with NamedTemporaryFile(suffix=".mp3", delete_on_close=False) as file:
+            file.close()
+            audio.write_to_file(file.name)
+            return file_to_data_uri(file.name, "audio/mpeg")

videoinput/srcts/audioSpinner.ts

@@ -0,0 +1,240 @@
+class AudioSpinnerElement extends HTMLElement {
+  #audio!: HTMLAudioElement;
+  #canvas!: HTMLCanvasElement;
+  #ctx2d!: CanvasRenderingContext2D;
+  #analyzer!: AnalyserNode;
+  #dataArray!: Float32Array;
+  #smoother!: Smoother<Float32Array>;
+
+  constructor() {
+    super();
+    this.attachShadow({ mode: "open" });
+    this.shadowRoot!.innerHTML = `
+        <style>
+          :host {
+            display: block;
+            position: relative;
+          }
+          ::slotted(canvas) {
+            position: absolute;
+            top: 0;
+            left: 0;
+          }
+          ::slotted(audio) {
+            display: none;
+          }
+        </style>
+        <slot name="audio"></slot>
+        <slot name="canvas"></slot>
+        `;
+  }
+
+  connectedCallback() {
+    // Create <audio>. This will play the sound.
+    const audioSlot = this.shadowRoot!.querySelector(
+      "slot[name=audio]"
+    )! as HTMLSlotElement;
+    this.#audio = this.ownerDocument.createElement("audio");
+    this.#audio.autoplay = true;
+    this.#audio.controls = false;
+    this.#audio.src = this.getAttribute("src")!;
+    this.#audio.slot = "audio";
+    audioSlot.assign(this.#audio);
+    this.#audio.addEventListener("play", () => {
+      this.#draw();
+    });
+    this.#audio.onpause = () => {
+      this.style.transition = "opacity 0.5s 1s";
+      this.classList.add("fade");
+      this.addEventListener("transitionend", () => {
+        this.remove();
+      });
+    };
+
+    // Create <canvas>. This will be the target of our vizualization.
+    const canvasSlot = this.shadowRoot!.querySelector(
+      "slot[name=canvas]"
+    )! as HTMLSlotElement;
+    this.#canvas = this.ownerDocument.createElement("canvas");
+    this.#canvas.slot = "canvas";
+    this.#canvas.width = this.clientWidth * window.devicePixelRatio;
+    this.#canvas.height = this.clientHeight * window.devicePixelRatio;
+    this.#canvas.style.width = this.clientWidth + "px";
+    this.#canvas.style.height = this.clientHeight + "px";
+    this.appendChild(this.#canvas);
+    canvasSlot.assign(this.#canvas);
+    this.#ctx2d = this.#canvas.getContext("2d")!;
+    this.#ctx2d.scale(window.devicePixelRatio, window.devicePixelRatio);
+    new ResizeObserver(() => {
+      this.#canvas.width = this.clientWidth;
+      this.#canvas.height = this.clientHeight;
+    }).observe(this);
+
+    // Initialize analyzer
+    const audioCtx = new AudioContext();
+    const source = audioCtx.createMediaElementSource(this.#audio);
+    this.#analyzer = new AnalyserNode(audioCtx, {
+      fftSize: 2048,
+    });
+    this.#dataArray = new Float32Array(this.#analyzer.frequencyBinCount);
+    source.connect(this.#analyzer);
+    this.#analyzer.connect(audioCtx.destination);
+
+    // Initialize persistent data structures needed for vizualization
+    const dataArray2 = new Float32Array(this.#analyzer.frequencyBinCount);
+    this.#smoother = new Smoother<Float32Array>(5, (samples) => {
+      for (let i = 0; i < dataArray2.length; i++) {
+        dataArray2[i] = 0;
+        for (let j = 0; j < samples.length; j++) {
+          dataArray2[i] += samples[j][i];
+        }
+        dataArray2[i] /= samples.length;
+      }
+      return dataArray2;
+    });
+
+    this.#draw();
+  }
+
+  #draw() {
+    if (!this.isConnected) {
+      return;
+    }
+
+    requestAnimationFrame(() => this.#draw());
+
+    const width = this.#canvas.width;
+    const height = this.#canvas.height;
+    this.#ctx2d.clearRect(0, 0, width, height);
+
+    this.#analyzer.getFloatTimeDomainData(this.#dataArray);
+    const smoothed = this.#smoother.add(new Float32Array(this.#dataArray));
+
+    const {
+      spinVelocity,
+      gap,
+      thickness,
+      minRadius,
+      radiusFactor,
+      steps,
+      blades,
+    } = this.#getSettings(width, height);
+
+    const avg =
+      (smoothed.reduce((a, b) => a + Math.abs(b), 0) / smoothed.length) * 4;
+
+    const radius = minRadius + (avg * (height - minRadius)) / radiusFactor;
+    for (let step = 0; step < steps; step++) {
+      const this_radius = radius - step * (radius / (steps + 1));
+      if (step === steps - 1) {
+        this.#drawPie(width, height, 0, Math.PI * 2, this_radius, thickness);
+      } else {
+        const seconds = new Date().getTime() / 1000;
+        const startAngle = (seconds * spinVelocity) % (Math.PI * 2);
+        for (let blade = 0; blade < blades; blade++) {
+          const angleOffset = ((Math.PI * 2) / blades) * blade;
+          const sweep = (Math.PI * 2) / blades - gap;
+          this.#drawPie(
+            width,
+            height,
+            startAngle + angleOffset,
+            sweep,
+            this_radius,
+            thickness
+          );
+        }
+      }
+    }
+  }
+
+  #drawPie(
+    width: number,
+    height: number,
+    startAngle: number,
+    sweep: number,
+    radius: number,
+    thickness?: number
+  ) {
+    this.#ctx2d.beginPath();
+    this.#ctx2d.fillStyle = this.#canvas
+      .computedStyleMap()
+      .get("color")
+      ?.toString()!;
+    if (!thickness) {
+      this.#ctx2d.moveTo(width / 2, height / 2);
+    }
+    this.#ctx2d.arc(
+      width / 2,
+      height / 2,
+      radius,
+      startAngle,
+      startAngle + sweep
+    );
+    if (!thickness) {
+      this.#ctx2d.lineTo(width / 2, height / 2);
+    } else {
+      this.#ctx2d.arc(
+        width / 2,
+        height / 2,
+        radius - thickness,
+        startAngle + sweep,
+        startAngle,
+        true
+      );
+    }
+    this.#ctx2d.fill();
+  }
+
+  #getSettings(width: number, height: number) {
+    // Visualization settings
+    const settings = {
+      spinVelocity: 5,
+      gap: Math.PI / 5,
+      thickness: 2.5,
+      minRadius: Math.min(width, height) / 6,
+      radiusFactor: 1.8,
+      steps: 3,
+      blades: 3,
+    };
+    for (const key in settings) {
+      const value = tryParseFloat(this.dataset[key]);
+      if (typeof value !== "undefined") {
+        Object.assign(settings, { [key]: value });
+      }
+    }
+    return settings;
+  }
+}
+
+window.customElements.define("audio-spinner", AudioSpinnerElement);
+
+class Smoother<T> {
+  #samples: T[] = [];
+  #smooth: (samples: T[]) => T;
+  #size: number;
+  #pos: number;
+
+  constructor(size: number, smooth: (samples: T[]) => T) {
+    this.#size = size;
+    this.#pos = 0;
+    this.#smooth = smooth;
+  }
+
+  add(sample: T): T {
+    this.#samples[this.#pos] = sample;
+    this.#pos = (this.#pos + 1) % this.#size;
+    return this.smoothed();
+  }
+
+  smoothed(): T {
+    return this.#smooth(this.#samples);
+  }
+}
+
+function tryParseFloat(str?: string): number | undefined {
+  if (typeof str === "undefined") {
+    return undefined;
+  }
+  const parsed = parseFloat(str);
+  return isNaN(parsed) ? undefined : parsed;
+}

videoinput/srcts/avSettingsMenu.ts

@@ -0,0 +1,91 @@
+class DeviceChangeEvent extends CustomEvent<{ deviceId: string | null }> {
+  constructor(type: string, detail: { deviceId: string | null }) {
+    super(type, { detail });
+  }
+}
+
+class AVSettingsMenuElement extends HTMLElement {
+  constructor() {
+    super();
+    this.addEventListener("click", (e) => {
+      if (e.target instanceof HTMLAnchorElement) {
+        const a = e.target;
+        if (a.classList.contains("camera-device-item")) {
+          this.cameraId = a.dataset.deviceId!;
+        } else if (a.classList.contains("mic-device-item")) {
+          this.micId = a.dataset.deviceId!;
+        }
+      }
+    });
+  }
+
+  #setDevices(deviceType: "camera" | "mic", devices: MediaDeviceInfo[]) {
+    const deviceEls = devices.map((dev) =>
+      this.#createDeviceElement(dev, `${deviceType}-device-item`)
+    );
+    const header = this.querySelector(`.${deviceType}-header`)!;
+    header.after(...deviceEls);
+  }
+
+  setCameras(cameras: MediaDeviceInfo[]) {
+    this.#setDevices("camera", cameras);
+  }
+
+  setMics(mics: MediaDeviceInfo[]) {
+    this.#setDevices("mic", mics);
+  }
+
+  get cameraId(): string | null {
+    return this.#getSelectedDevice("camera");
+  }
+
+  set cameraId(id: string | null) {
+    this.#setSelectedDevice("camera", id);
+  }
+
+  get micId(): string | null {
+    return this.#getSelectedDevice("mic");
+  }
+
+  set micId(id: string | null) {
+    this.#setSelectedDevice("mic", id);
+  }
+
+  #createDeviceElement(dev: MediaDeviceInfo, className: string): HTMLLIElement {
+    const li = this.ownerDocument.createElement("li");
+    const a = li.appendChild(this.ownerDocument.createElement("a"));
+    a.onclick = (e) => e.preventDefault();
+    a.href = "#";
+    a.textContent = dev.label;
+    a.dataset.deviceId = dev.deviceId!;
+    a.className = className;
+    return li;
+  }
+
+  #getSelectedDevice(device: "camera" | "mic"): string | null {
+    return (
+      (
+        this.querySelector(
+          `a.${device}-device-item.active`
+        ) as HTMLAnchorElement
+      )?.dataset.deviceId ?? null
+    );
+  }
+
+  #setSelectedDevice(device: "camera" | "mic", id: string | null) {
+    this.querySelectorAll(`a.${device}-device-item.active`).forEach((a) =>
+      a.classList.remove("active")
+    );
+    if (id) {
+      this.querySelector(
+        `a.${device}-device-item[data-device-id="${id}"]`
+      )!.classList.add("active");
+    }
+    this.dispatchEvent(
+      new DeviceChangeEvent(`${device}-change`, {
+        deviceId: id,
+      })
+    );
+  }
+}
+customElements.define("av-settings-menu", AVSettingsMenuElement);

videoinput/srcts/index.ts

@@ -0,0 +1,57 @@
+import { BindScope } from "rstudio-shiny/srcts/types/src/shiny/bind";
+
+// Register custom elements
+import "./videoClipper";
+import "./avSettingsMenu";
+import "./audioSpinner";
+
+// Create input binding to send video clips from <video-clipper> to Shiny
+class VideoClipperBinding extends Shiny.InputBinding {
+  #lastKnownValue = new WeakMap<HTMLElement, unknown>();
+  #handlers = new WeakMap<HTMLElement, (ev: Event) => Promise<void>>();
+
+  find(scope: BindScope): JQuery<HTMLElement> {
+    return $(scope).find("video-clipper");
+  }
+
+  getValue(el: HTMLElement): unknown {
+    return this.#lastKnownValue.get(el);
+  }
+
+  subscribe(el: HTMLElement, callback: (value: boolean) => void): void {
+    const handler = async (ev: Event) => {
+      const blob = (ev as BlobEvent).data;
+      this.#lastKnownValue.set(el, {
+        type: blob.type,
+        bytes: await base64(blob),
+      });
+      callback(true);
+    };
+    el.addEventListener("data", handler);
+    this.#handlers.set(el, handler);
+  }
+
+  unsubscribe(el: HTMLElement): void {
+    const handler = this.#handlers.get(el)!;
+    el.removeEventListener("data", handler);
+    this.#handlers.delete(el);
+  }
+}
+
+window.Shiny.inputBindings.register(new VideoClipperBinding(), "video-clipper");
+
+/**
+ * Encode a Blob as a base64 string
+ * @param blob The Blob to encode
+ * @returns A base64-encoded string
+ */
+async function base64(blob: Blob): Promise<string> {
+  const buf = await blob.arrayBuffer();
+  const results = [];
+  const CHUNKSIZE = 1024;
+  for (let i = 0; i < buf.byteLength; i += CHUNKSIZE) {
+    const chunk = buf.slice(i, i + CHUNKSIZE);
+    results.push(String.fromCharCode(...new Uint8Array(chunk)));
+  }
+  return btoa(results.join(""));
+}

videoinput/srcts/videoClipper.ts

@@ -0,0 +1,254 @@
+class VideoClipperElement extends HTMLElement {
+  video: HTMLVideoElement;
+  avSettingsMenu!: AVSettingsMenuElement;
+  buttonRecord!: HTMLButtonElement;
+  buttonStop!: HTMLButtonElement;
+
+  cameraStream?: MediaStream;
+  micStream?: MediaStream;
+
+  recorder?: MediaRecorder;
+  chunks: Blob[] = [];
+
+  constructor() {
+    super();
+    this.attachShadow({ mode: "open" });
+    this.shadowRoot!.innerHTML = `
+        <style>
+          :host {
+            display: grid;
+            grid-template-rows: 1fr;
+            grid-template-columns: 1fr;
+            width: 100%;
+            height: min-content;
+          }
+          video {
+            grid-column: 1 / 2;
+            grid-row: 1 / 2;
+            width: 100%;
+            object-fit: cover;
+            background-color: var(--video-clip-bg, black);
+            aspect-ratio: 16 / 9;
+            border-radius: var(--video-clip-border-radius, var(--bs-border-radius-lg));
+          }
+          video.mirrored {
+            transform: scaleX(-1);
+          }
+          .panel-settings {
+            grid-column: 1 / 2;
+            grid-row: 1 / 2;
+            justify-self: end;
+            margin: 0.5em;
+          }
+          .panel-buttons {
+            grid-column: 1 / 2;
+            grid-row: 1 / 2;
+            justify-self: end;
+            align-self: end;
+            margin: 0.5em;
+          }
+        </style>
+        <video part="video" muted></video>
+        <div class="panel-settings">
+          <slot name="settings"></slot>
+        </div>
+        <div class="panel-buttons">
+          <slot name="recording-controls"></slot>
+        </div>
+    `;
+    this.video = this.shadowRoot!.querySelector("video")!;
+  }
+  connectedCallback() {
+    (async () => {
+      const slotSettings = this.shadowRoot!.querySelector(
+        "slot[name=settings]"
+      )! as HTMLSlotElement;
+      slotSettings.addEventListener("slotchange", async () => {
+        this.avSettingsMenu =
+          slotSettings.assignedElements()[0] as AVSettingsMenuElement;
+        await this.#initializeMediaInput();
+        if (this.buttonRecord) {
+          this.#setEnabledButton(this.buttonRecord);
+        }
+      });
+
+      const slotControls = this.shadowRoot!.querySelector(
+        "slot[name=recording-controls]"
+      )! as HTMLSlotElement;
+      slotControls.addEventListener("slotchange", () => {
+        const findButton = (selector: string): HTMLElement | null => {
+          for (const el of slotControls.assignedElements()) {
+            if (el.matches(selector)) {
+              return el as HTMLElement;
+            }
+            const sub = el.querySelector(selector);
+            if (sub) {
+              return sub as HTMLElement;
+            }
+          }
+          return null;
+        };
+        this.buttonRecord = findButton(".record-button")! as HTMLButtonElement;
+        this.buttonStop = findButton(".stop-button")! as HTMLButtonElement;
+
+        this.#setEnabledButton();
+
+        this.buttonRecord.addEventListener("click", () => {
+          this.#setEnabledButton(this.buttonStop);
+          this._beginRecord();
+        });
+        this.buttonStop.addEventListener("click", () => {
+          this._endRecord();
+          this.#setEnabledButton(this.buttonRecord);
+        });
+      });
+    })().catch((err) => {
+      console.error(err);
+    });
+  }
+
+  disconnectedCallback() {}
+
+  #setEnabledButton(btn?: HTMLButtonElement) {
+    this.buttonRecord.style.display =
+      btn === this.buttonRecord ? "inline-block" : "none";
+    this.buttonStop.style.display =
+      btn === this.buttonStop ? "inline-block" : "none";
+  }
+
+  async setMediaDevices(
+    cameraId: string | null,
+    micId: string | null
+  ): Promise<{ cameraId: string; micId: string }> {
+    if (this.cameraStream) {
+      this.cameraStream.getTracks().forEach((track) => track.stop());
+    }
+
+    this.cameraStream = await navigator.mediaDevices.getUserMedia({
+      video: {
+        deviceId: cameraId || undefined,
+        facingMode: "user",
+        aspectRatio: 16 / 9,
+      },
+      audio: {
+        deviceId: micId || undefined,
+      },
+    });
+
+    // TODO: I can't figure out how to tell if this is actually a selfie cam.
+    // Ideally we wouldn't mirror unless we are sure.
+    const isSelfieCam = true; // this.cameraStream.getVideoTracks()[0].getSettings().facingMode === "user";
+    this.video.classList.toggle("mirrored", isSelfieCam);
+
+    /* Prevent the height from jumping around when switching cameras */
+    const aspectRatio = this.cameraStream
+      .getVideoTracks()[0]
+      .getSettings().aspectRatio;
+    if (aspectRatio) {
+      this.video.style.aspectRatio = `${aspectRatio}`;
+    } else {
+      this.video.style.aspectRatio = "";
+    }
+    this.video.srcObject = this.cameraStream!;
+    this.video.play();
+
+    return {
+      cameraId: this.cameraStream.getVideoTracks()[0].getSettings().deviceId!,
+      micId: this.cameraStream.getAudioTracks()[0].getSettings().deviceId!,
+    };
+  }
+
+  async #initializeMediaInput() {
+    // Retrieve the user's previous camera and mic settings, if they ever
+    // explicitly chose one
+    const savedCamera = window.localStorage.getItem("multimodal-camera");
+    const savedMic = window.localStorage.getItem("multimodal-mic");
+
+    // Initialize the camera and mic with the saved settings. It's important to
+    // request camera/mic access _before_ we attempt to enumerate devices,
+    // because if the user has not granted camera/mic access, enumerateDevices()
+    // will not prompt the user for permission and will instead return empty
+    // devices.
+    //
+    // The return values are the actual camera and mic IDs that were used, which
+    // may be different from the saved values if those devices are no longer
+    // available.
+    const { cameraId, micId } = await this.setMediaDevices(
+      savedCamera,
+      savedMic
+    );
+
+    // Populate the camera and mic dropdowns with the available devices
+    const devices = await navigator.mediaDevices.enumerateDevices();
+    this.avSettingsMenu.setCameras(
+      devices.filter((dev) => dev.kind === "videoinput")
+    );
+    this.avSettingsMenu.setMics(
+      devices.filter((dev) => dev.kind === "audioinput")
+    );
+
+    // Update the dropdown UI to reflect the actual devices that were used
+    this.avSettingsMenu.cameraId = cameraId;
+    this.avSettingsMenu.micId = micId;
+
+    // Listen for changes to the camera and mic dropdowns
+    const handleDeviceChange = async (
+      deviceType: string,
+      deviceId: string | null
+    ) => {
+      if (!deviceId) return;
+      window.localStorage.setItem(`multimodal-${deviceType}`, deviceId);
+      await this.setMediaDevices(
+        this.avSettingsMenu.cameraId,
+        this.avSettingsMenu.micId
+      );
+    };
+    this.avSettingsMenu.addEventListener("camera-change", (e) => {
+      handleDeviceChange("camera", this.avSettingsMenu.cameraId);
+    });
+    this.avSettingsMenu.addEventListener("mic-change", (e) => {
+      handleDeviceChange("mic", this.avSettingsMenu.micId);
+    });
+  }
+
+  _beginRecord() {
+    // Create a MediaRecorder object
+    this.recorder = new MediaRecorder(this.cameraStream!, {});
+
+    this.recorder.addEventListener("error", (e) => {
+      console.error("MediaRecorder error:", (e as ErrorEvent).error);
+    });
+    this.recorder.addEventListener("dataavailable", (e) => {
+      // console.log("chunk: ", e.data.size, e.data.type);
+      this.chunks.push(e.data);
+    });
+    this.recorder.addEventListener("start", () => {
+      // console.log("Recording started");
+    });
+    this.recorder.start();
+  }
+
+  _endRecord(emit: boolean = true) {
+    this.recorder!.stop();
+    if (!emit) {
+      this.chunks = [];
+    } else {
+      // Use setTimeout to give it a moment to finish processing the last chunk
+      setTimeout(() => {
+        // console.log("chunks: ", this.chunks.length);
+        const blob = new Blob(this.chunks, { type: this.chunks[0].type });
+
+        // emit blobevent
+        const event = new BlobEvent("data", {
+          data: blob,
+        });
+        try {
+          this.dispatchEvent(event);
+        } finally {
+          this.chunks = [];
+        }
+      }, 0);
+    }
+  }
+}
+customElements.define("video-clipper", VideoClipperElement);

videoinput/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./srcts",
+    "target": "es2020",
+    "module": "commonjs",
+    "lib": ["dom", "es2020"],
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["srcts/**/*"]
+}

videoinput/ui.py

@@ -0,0 +1,114 @@
+import math
+from pathlib import Path
+
+from faicons import icon_svg
+from htmltools import HTMLDependency
+from shiny import module, ui
+
+__all__ = (
+    "input_video_clip",
+    "audio_spinner",
+)
+
+multimodal_dep = HTMLDependency(
+    "multimodal",
+    "0.0.1",
+    source={
+        "subdir": str(Path(__file__).parent / "dist"),
+    },
+    script={"src": "index.js"},
+    stylesheet={"href": "index.css"},
+)
+
+
+def input_video_clip(id: str, **kwargs):
+    id = module.resolve_id(id)
+
+    return ui.Tag(
+        "video-clipper",
+        multimodal_dep,
+        ui.Tag(
+            "av-settings-menu",
+            ui.div(
+                ui.tags.button(
+                    icon_svg("gear").add_class("fw"),
+                    class_="btn btn-sm btn-secondary dropdown-toggle px-3 py-2",
+                    type="button",
+                    **{"data-bs-toggle": "dropdown"},
+                ),
+                ui.tags.ul(
+                    ui.tags.li(
+                        ui.tags.h6("Camera", class_="dropdown-header"),
+                        class_="camera-header",
+                    ),
+                    # Camera items will go here
+                    ui.tags.li(ui.tags.hr(class_="dropdown-divider")),
+                    ui.tags.li(
+                        ui.tags.h6("Microphone", class_="dropdown-header"),
+                        class_="mic-header",
+                    ),
+                    # Microphone items will go here
+                    class_="dropdown-menu",
+                ),
+                class_="btn-group",
+            ),
+            slot="settings",
+        ),
+        ui.div(
+            ui.tags.button(
+                ui.TagList(
+                    ui.tags.div(
+                        style="display: inline-block; background-color: red; width: 1rem; height: 1rem; border-radius: 100%; position: relative; top: 0.175rem; margin-right: 0.3rem;"
+                    ),
+                    "Record",
+                ),
+                style="display: block;",
+                class_="record-button btn btn-secondary px-3 mx-auto",
+            ),
+            ui.tags.button(
+                ui.TagList(
+                    ui.tags.div(
+                        style="display: inline-block; background-color: currentColor; width: 1rem; height: 1rem; position: relative; top: 0.175rem; margin-right: 0.3rem;"
+                    ),
+                    "Stop",
+                ),
+                style="display: block;",
+                class_="stop-button btn btn-secondary px-3 mx-auto",
+            ),
+            slot="recording-controls",
+            class_="btn-group",
+            **{"aria-label": "Recording controls"},
+        ),
+        id=id,
+        **kwargs,
+    )
+
+
+def audio_spinner(
+    *,
+    src: str,
+    spin_velocity: float = 1,
+    gap: float = math.pi / 5,
+    thickness: float = 2.5,
+    min_radius: float = 30,
+    radius_factor: float = 1.8,
+    steps: float = 3,
+    blades: float = 3,
+    **kwargs
+):
+    return ui.Tag(
+        "audio-spinner",
+        multimodal_dep,
+        src=src,
+        style="width: 125px; height: 125px;",
+        class_="mx-auto",
+        **{
+            "data-spin-velocity": spin_velocity,
+            "data-gap": gap,
+            "data-thickness": thickness,
+            "data-min-radius": min_radius,
+            "data-radius-factor": radius_factor,
+            "data-steps": steps,
+            "data-blades": blades,
+        },
+    )

videoinput/utils.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import base64
+import contextlib
+import os
+import tempfile
+import time
+
+import colorama
+
+__all__ = (
+    "timed",
+    "file_to_data_uri",
+)
+
+
+@contextlib.contextmanager
+def timed(msg):
+    start = time.perf_counter()
+    print(colorama.Style.DIM + f"╔ {msg}" + colorama.Style.RESET_ALL)
+    yield
+    elapsed = time.perf_counter() - start
+    print(
+        colorama.Style.DIM + f"╚ Finished in {elapsed:.3f}s" + colorama.Style.RESET_ALL
+    )
+
+
+def file_to_data_uri(file_path, mime_type):
+    with open(file_path, "rb") as file:
+        encoded_string = base64.b64encode(file.read()).decode("utf-8")
+        return f"data:{mime_type};base64,{encoded_string}"
+
+
+class NamedTemporaryFile(contextlib.AbstractContextManager):
+    """
+    tempfile.NamedTemporaryFile with an additional `delete_on_close` parameter.
+
+    The `delete_on_close` parameter was only added in Python 3.12, but we badly
+    need it on Windows: because file access on Windows is exclusive, we can't
+    write to and then read from a file without closing it in between. But
+    without `delete_on_close`, the file is deleted on close.
+
+    This class is a thin shim around tempfile.NamedTemporaryFile that adds the
+    parameter for older Python versions.
+    """
+
+    def __init__(
+        self,
+        mode: str = "w+b",
+        buffering: int = -1,
+        encoding: str | None = None,
+        newline: str | None = None,
+        suffix: str = "",
+        prefix: str = "tmp",
+        dir: str | None = None,
+        delete: bool = True,
+        *,
+        errors: str | None = None,
+        delete_on_close: bool = True,
+    ):
+        self._needs_manual_delete = delete and not delete_on_close
+        self._file = tempfile.NamedTemporaryFile(
+            mode=mode,
+            buffering=buffering,
+            encoding=encoding,
+            newline=newline,
+            suffix=suffix,
+            prefix=prefix,
+            dir=dir,
+            delete=delete and delete_on_close,
+            errors=errors,
+        )
+
+    def __enter__(self) -> tempfile.NamedTemporaryFile:
+        return self._file.__enter__()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._file.__exit__(exc_type, exc_val, exc_tb)
+        if self._needs_manual_delete:
+            os.unlink(self._file.name)

videoinput/yarn.lock

@@ -0,0 +1,225 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.21.3.tgz#78d3e6dcd19c1cb91f3940143e86dad1094aee81"
+  integrity sha512-yTgnwQpFVYfvvo4SvRFB0SwrW8YjOxEoT7wfMT7Ol5v7v5LDNvSGo67aExmxOb87nQNeWPVvaGBNfQ7BXcrZ9w==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.21.3.tgz#5eea56c21d61734942e050840d881eb7bedc3993"
+  integrity sha512-c+ty9necz3zB1Y+d/N+mC6KVVkGUUOcm4ZmT5i/Fk5arOaY3i6CA3P5wo/7+XzV8cb4GrI/Zjp8NuOQ9Lfsosw==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.21.3.tgz#7fda92e3231043c071ea6aa76c92accea86439fd"
+  integrity sha512-bviJOLMgurLJtF1/mAoJLxDZDL6oU5/ztMHnJQRejbJrSc9FFu0QoUoFhvi6qSKJEw9y5oGyvr9fuDtzJ30rNQ==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.21.3.tgz#625d139bde81b81f54ff493b1381ca0f540200f3"
+  integrity sha512-JReHfYCRK3FVX4Ra+y5EBH1b9e16TV2OxrPAvzMsGeES0X2Ndm9ImQRI4Ket757vhc5XBOuGperw63upesclRw==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.21.3.tgz#fa25f38a43ff4c469589d1dc93448d534d7f313b"
+  integrity sha512-U3fuQ0xNiAkXOmQ6w5dKpEvXQRSpHOnbw7gEfHCRXPeTKW9sBzVck6C5Yneb8LfJm0l6le4NQfkNPnWMSlTFUQ==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.21.3.tgz#2e450b8214f179a56b4559b2f107060e2b792c7e"
+  integrity sha512-3m1CEB7F07s19wmaMNI2KANLcnaqryJxO1fXHUV5j1rWn+wMxdUYoPyO2TnAbfRZdi7ADRwJClmOwgT13qlP3Q==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.3.tgz#f6b29e07bce25c545f6f7bb031d3be6a6ea1dc50"
+  integrity sha512-fsNAAl5pU6wmKHq91cHWQT0Fz0vtyE1JauMzKotrwqIKAswwP5cpHUCxZNSTuA/JlqtScq20/5KZ+TxQdovU/g==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.21.3.tgz#1a5da2bf89f8d67102820d893d271a270ae55751"
+  integrity sha512-tci+UJ4zP5EGF4rp8XlZIdq1q1a/1h9XuronfxTMCNBslpCtmk97Q/5qqy1Mu4zIc0yswN/yP/BLX+NTUC1bXA==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.21.3.tgz#355f6624c1ac6f5f68841a327ac90b98c679626c"
+  integrity sha512-vvG6R5g5ieB4eCJBQevyDMb31LMHthLpXTc2IGkFnPWS/GzIFDnaYFp558O+XybTmYrVjxnryru7QRleJvmZ6Q==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.21.3.tgz#872a476ca18a962a98700024c447a79279db1d45"
+  integrity sha512-f6kz2QpSuyHHg01cDawj0vkyMwuIvN62UAguQfnNVzbge2uWLhA7TCXOn83DT0ZvyJmBI943MItgTovUob36SQ==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.21.3.tgz#da713eb80ff6c011ed01aa4deebb5fc758906046"
+  integrity sha512-HjCWhH7K96Na+66TacDLJmOI9R8iDWDDiqe17C7znGvvE4sW1ECt9ly0AJ3dJH62jHyVqW9xpxZEU1jKdt+29A==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.21.3.tgz#a7c5dc9e961009018d23ec53a43baa8c03c5a1d5"
+  integrity sha512-BGpimEccmHBZRcAhdlRIxMp7x9PyJxUtj7apL2IuoG9VxvU/l/v1z015nFs7Si7tXUwEsvjc1rOJdZCn4QTU+Q==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.21.3.tgz#b97543f3d8655365729f3702ed07f6e41be5e48e"
+  integrity sha512-5rMOWkp7FQGtAH3QJddP4w3s47iT20hwftqdm7b+loe95o8JU8ro3qZbhgMRy0VuFU0DizymF1pBKkn3YHWtsw==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.21.3.tgz#23b9064d5bc0bf28a115a2f9cf69f3b01cdfe01c"
+  integrity sha512-h0zj1ldel89V5sjPLo5H1SyMzp4VrgN1tPkN29TmjvO1/r0MuMRwJxL8QY05SmfsZRs6TF0c/IDH3u7XYYmbAg==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.21.3.tgz#4f2536327f6d444c0573bd35bbd3a67897dbd5da"
+  integrity sha512-dkAKcTsTJ+CRX6bnO17qDJbLoW37npd5gSNtSzjYQr0svghLJYGYB0NF1SNcU1vDcjXLYS5pO4qOW4YbFama4A==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.21.3.tgz#05e6f3a12a0dcd60672f25e8789a83cd3affa487"
+  integrity sha512-vnD1YUkovEdnZWEuMmy2X2JmzsHQqPpZElXx6dxENcIwTu+Cu5ERax6+Ke1QsE814Zf3c6rxCfwQdCTQ7tPuXA==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.21.3.tgz#994d347e7f530c33628e35e48ccde8f299adbcb6"
+  integrity sha512-IOXOIm9WaK7plL2gMhsWJd+l2bfrhfilv0uPTptoRoSb2p09RghhQQp9YY6ZJhk/kqmeRt6siRdMSLLwzuT0KQ==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.21.3.tgz#309d8c323632e9c70ee92cf5414fa65b5eb7e00e"
+  integrity sha512-uTgCwsvQ5+vCQnqM//EfDSuomo2LhdWhFPS8VL8xKf+PKTCrcT/2kPPoWMTs22aB63MLdGMJiE3f1PHvCDmUOw==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.21.3.tgz#28820f9431fe00f2b04aac57511754213ff060eb"
+  integrity sha512-vNAkR17Ub2MgEud2Wag/OE4HTSI6zlb291UYzHez/psiKarp0J8PKGDnAhMBcHFoOHMXHfExzmjMojJNbAStrQ==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.21.3.tgz#a1f7f98b85bd221fe0f545d01abc0e6123ae60dc"
+  integrity sha512-W8H9jlGiSBomkgmouaRoTXo49j4w4Kfbl6I1bIdO/vT0+0u4f20ko3ELzV3hPI6XV6JNBVX+8BC+ajHkvffIJA==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.21.3.tgz#c6c3c0b1a1dfc6327ef4db6aa4fb6efd9df531f7"
+  integrity sha512-EjEomwyLSCg8Ag3LDILIqYCZAq/y3diJ04PnqGRgq8/4O3VNlXyMd54j/saShaN4h5o5mivOjAzmU6C3X4v0xw==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.21.3.tgz#471b8d2cad1bd6479eee5acf04bba2c0e4d37e24"
+  integrity sha512-WGiE/GgbsEwR33++5rzjiYsKyHywE8QSZPF7Rfx9EBfK3Qn3xyR6IjyCr5Uk38Kg8fG4/2phN7sXp4NPWd3fcw==
+
+"@esbuild/[email protected]":
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.21.3.tgz#899c03576c4c28c83228f0e64dfa10edae99c9a2"
+  integrity sha512-xRxC0jaJWDLYvcUvjQmHCJSfMrgmUuvsoXgDeU/wTorQ1ngDdUBuFtgY3W1Pc5sprGAvZBtWdJX7RPg/iZZUqA==
+
+"@types/[email protected]":
+  version "0.0.14"
+  resolved "https://registry.yarnpkg.com/@types/bootstrap-datepicker/-/bootstrap-datepicker-0.0.14.tgz#89bc5816dd0b802bf8d4923cf5461e2c98cbb450"
+  integrity sha512-ZBwEUuwYpehEHWtkaIDl55N3F6dPz6xEsoPW5esCZdtuhuZdNC/ebFV2n48kATcY5S2HD3V0O5/J8nuEgVk9fQ==
+  dependencies:
+    "@types/jquery" "*"
+
+"@types/[email protected]":
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/@types/bootstrap/-/bootstrap-3.4.0.tgz#7e74a2421b3984ce53ab459e979a99a78dd802c6"
+  integrity sha512-LS05hVAAsX86qbHg7W+ydwBlNHrVCoFw6wEP3/uW4eYmRXl08bWmPeN/+onM+8qZTFfDgUlG/OItJI8SW972oQ==
+  dependencies:
+    "@types/jquery" "*"
+
+"@types/datatables.net@^1.10.19":
+  version "1.12.0"
+  resolved "https://registry.yarnpkg.com/@types/datatables.net/-/datatables.net-1.12.0.tgz#cc0b0eb875997515aa10a59969ecb5fb03423ea4"
+  integrity sha512-HjvusoHf/XKdNI/pDP/qVDclUVOKF/PI4R/AlVeUdz3Z00+A4Dyu9N5hIXGk3cK36sy1A072UffYZ+1wwW0zWg==
+  dependencies:
+    datatables.net "*"
+
+"@types/[email protected]":
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/@types/ion-rangeslider/-/ion-rangeslider-2.3.0.tgz#dd089921c21acebc266c37e95d191e721ea21750"
+  integrity sha512-u1k2Ekfg7E/ByCrtqtMk98g3QMXnMyIjCKU397YfjhWnTrms2BteoTanMqe0MJ1zyBBiP6+bwbc7P4cU+RiL+w==
+
+"@types/jquery@*", "@types/jquery@^3.5.30":
+  version "3.5.30"
+  resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-3.5.30.tgz#888d584cbf844d3df56834b69925085038fd80f7"
+  integrity sha512-nbWKkkyb919DOUxjmRVk8vwtDb0/k8FKncmUKFi+NY+QXqWltooxTrswvz4LspQwxvLdvzBN1TImr6cw3aQx2A==
+  dependencies:
+    "@types/sizzle" "*"
+
+"@types/jquery@patch:@types/[email protected]#./srcts/patch/types-jquery.patch":
+  version "3.5.5"
+  resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-3.5.5.tgz#2c63f47c9c8d96693d272f5453602afd8338c903"
+  integrity sha512-6RXU9Xzpc6vxNrS6FPPapN1SxSHgQ336WC6Jj/N8q30OiaBZ00l1GBgeP7usjVZPivSkGUfL1z/WW6TX989M+w==
+  dependencies:
+    "@types/sizzle" "*"
+
+"@types/rstudio-shiny@https://github.com/rstudio/shiny#v1.7.0":
+  version "1.7.0"
+  resolved "https://github.com/rstudio/shiny#0fc861afb4c748b544ce1ac39be2e44563a7acf2"
+  dependencies:
+    "@types/bootstrap" "3.4.0"
+    "@types/bootstrap-datepicker" "0.0.14"
+    "@types/datatables.net" "^1.10.19"
+    "@types/ion-rangeslider" "2.3.0"
+    "@types/jquery" "patch:@types/[email protected]#./srcts/patch/types-jquery.patch"
+    "@types/selectize" "0.12.34"
+
+"@types/[email protected]":
+  version "0.12.34"
+  resolved "https://registry.yarnpkg.com/@types/selectize/-/selectize-0.12.34.tgz#24a79e3176103019d2371ae9fc8487dcd1336cea"
+  integrity sha512-OAXHsgsHBMJBHnB6QrFlHVa0+gCE5lgjdpwJDlJLRhrcsM4I2VXrm2vsG00//Rpdm4XgnX17vrBpjHlWOk1ALw==
+
+"@types/sizzle@*":
+  version "2.3.8"
+  resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.8.tgz#518609aefb797da19bf222feb199e8f653ff7627"
+  integrity sha512-0vWLNK2D5MT9dg0iOo8GlKguPAU02QjmZitPEsXRuJXU/OGIOt9vT9Fc26wtYuavLxtO45v9PGleoL9Z0k1LHg==
+
+datatables.net@*:
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/datatables.net/-/datatables.net-2.0.7.tgz#8af61ac17334c37ae3bd77fc2828a3571e052041"
+  integrity sha512-cyW+HZwkMzb4bLrao/SS9/i64ZHiw5nYhXl+OwuOPgddG+R9O11iOEke8wYsdiyOQmjv2mE6xkEmRMZwHZc8zw==
+  dependencies:
+    jquery ">=1.7"
+
+esbuild@^0.21.3:
+  version "0.21.3"
+  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.21.3.tgz#cbb10b100c768b0cfb35d61d9e70324553437c38"
+  integrity sha512-Kgq0/ZsAPzKrbOjCQcjoSmPoWhlcVnGAUo7jvaLHoxW1Drto0KGkR1xBNg2Cp43b9ImvxmPEJZ9xkfcnqPsfBw==
+  optionalDependencies:
+    "@esbuild/aix-ppc64" "0.21.3"
+    "@esbuild/android-arm" "0.21.3"
+    "@esbuild/android-arm64" "0.21.3"
+    "@esbuild/android-x64" "0.21.3"
+    "@esbuild/darwin-arm64" "0.21.3"
+    "@esbuild/darwin-x64" "0.21.3"
+    "@esbuild/freebsd-arm64" "0.21.3"
+    "@esbuild/freebsd-x64" "0.21.3"
+    "@esbuild/linux-arm" "0.21.3"
+    "@esbuild/linux-arm64" "0.21.3"
+    "@esbuild/linux-ia32" "0.21.3"
+    "@esbuild/linux-loong64" "0.21.3"
+    "@esbuild/linux-mips64el" "0.21.3"
+    "@esbuild/linux-ppc64" "0.21.3"
+    "@esbuild/linux-riscv64" "0.21.3"
+    "@esbuild/linux-s390x" "0.21.3"
+    "@esbuild/linux-x64" "0.21.3"
+    "@esbuild/netbsd-x64" "0.21.3"
+    "@esbuild/openbsd-x64" "0.21.3"
+    "@esbuild/sunos-x64" "0.21.3"
+    "@esbuild/win32-arm64" "0.21.3"
+    "@esbuild/win32-ia32" "0.21.3"
+    "@esbuild/win32-x64" "0.21.3"
+
+jquery@>=1.7:
+  version "3.7.1"
+  resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.7.1.tgz#083ef98927c9a6a74d05a6af02806566d16274de"
+  integrity sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg==
+
+typescript@^5.4.5:
+  version "5.4.5"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.4.5.tgz#42ccef2c571fdbd0f6718b1d1f5e6e5ef006f611"
+  integrity sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==