ported audio_utils and inference from colab

app.py

@@ -0,0 +1,33 @@
+import streamlit as st
+import os
+import librosa
+import soundfile as sf
+import numpy as np
+from inference import generate_drum_kit
+from audio_utils import play_audio
+
+# Streamlit UI
+st.title("Generate Drum Kits with Text")
+
+# User Inputs
+prompt = st.text_input("Describe your drum kit (e.g., 'warm vintage')", "8-bit video game")
+kit_size = st.slider("Number of sounds per instrument:", 1, 10, 4)
+
+# Run the inference
+if st.button("Generate Drum Kit"):
+    drum_kit = generate_drum_kit(prompt, kit_size)
+    st.session_state["drum_kit"] = drum_kit  # Store results
+
+# Display results
+if "drum_kit" in st.session_state:
+    drum_kit = st.session_state["drum_kit"]
+    st.subheader("Generated Drum Kit")
+
+    for instrument, sounds in drum_kit.items():
+        st.write(f"**{instrument}**")
+        cols = st.columns(len(sounds))
+
+        for i, sound_file in enumerate(sounds):
+            with cols[i]:
+                if st.button(f"▶️ {os.path.basename(sound_file)}", key=sound_file):
+                    play_audio(sound_file)

audio_utils.py

@@ -0,0 +1,12 @@
+import streamlit as st
+import soundfile as sf
+import librosa
+import numpy as np
+import io
+
+def play_audio(file_path):
+    """Load and play an audio file."""
+    audio, sr = librosa.load(file_path, sr=16000)
+    audio_buffer = io.BytesIO()
+    sf.write(audio_buffer, audio, sr, format="wav")
+    st.audio(audio_buffer, format="audio/wav")

dataset/audio_embeddings.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:734662f4d6f61035f7519a883918c52b6abc9d9eee0a03df1c4aaebb559d8408
+size 21482036

inference.py

@@ -0,0 +1,65 @@
+import laion_clap
+import numpy as np
+import librosa
+import pickle
+import os
+from sklearn.metrics.pairwise import cosine_similarity
+import pandas as pd
+import zipfile
+import json
+
+dataset_zip = "dataset/one_shot_percussive_sounds.zip"
+extracted_folder = "dataset/unzipped"
+metadata_path = "dataset/licenses.txt"
+audio_embeddings_path = "dataset/audio_embeddings.pkl"
+
+# Unzip if not already extracted
+if not os.path.exists(extracted_folder):
+    with zipfile.ZipFile(dataset_zip, "r") as zip_ref:
+        zip_ref.extractall("dataset")
+
+# Load the model
+model = laion_clap.CLAP_Module(enable_fusion=True)
+model.load_ckpt(model_id=3)
+
+# Load dataset metadata
+with open(metadata_path, "r") as file:
+    data = json.load(file)
+
+# Convert the JSON data into a Pandas DataFrame
+metadata = pd.DataFrame.from_dict(data, orient="index")
+metadata.index = metadata.index.astype(str) + '.wav'
+
+# Load precomputed audio embeddings (to avoid recomputing on every request)
+with open(audio_embeddings_path, "rb") as f:
+    audio_embeddings = pickle.load(f)
+
+def get_clap_embeddings_from_text(text):
+    """Convert user text input to a CLAP embedding."""
+    text_embed = model.get_text_embedding([text])
+    return text_embed[0, :]
+
+def find_top_sounds(text_embed, instrument, top_N=4):
+    """Finds the closest N sounds for an instrument."""
+    valid_sounds = metadata[metadata["Instrument"] == instrument].index.tolist()
+    relevant_embeddings = {k: v for k, v in audio_embeddings.items() if k in valid_sounds}
+
+    # Compute cosine similarity
+    all_embeds = np.array([v for v in relevant_embeddings.values()])
+    similarities = cosine_similarity([text_embed], all_embeds)[0]
+
+    # Get top N matches
+    top_indices = np.argsort(similarities)[-top_N:][::-1]
+    top_files = [valid_sounds[i] for i in top_indices]
+
+    return top_files
+
+def generate_drum_kit(prompt, kit_size=4):
+    """Generate a drum kit dictionary from user input."""
+    text_embed = get_clap_embeddings_from_text(prompt)
+    drum_kit = {}
+
+    for instrument in ["Kick", "Snare", "Hi-Hat", "Tom", "Cymbal", "Clap", "Percussion", "Other"]:
+        drum_kit[instrument] = find_top_sounds(text_embed, instrument, top_N=kit_size)
+
+    return drum_kit

requirements.txt

@@ -0,0 +1,79 @@
+altair==5.5.0
+annotated-types==0.7.0
+attrs==25.3.0
+audioread==3.0.1
+blinker==1.9.0
+braceexpand==0.1.7
+cachetools==5.5.2
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+click==8.1.8
+decorator==5.2.1
+docker-pycreds==0.4.0
+filelock==3.17.0
+fsspec==2025.3.0
+ftfy==6.3.1
+gitdb==4.0.12
+GitPython==3.1.44
+h5py==3.13.0
+huggingface-hub==0.29.3
+idna==3.10
+Jinja2==3.1.6
+joblib==1.4.2
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+laion_clap==1.1.6
+lazy_loader==0.4
+librosa==0.11.0
+llvmlite==0.43.0
+MarkupSafe==3.0.2
+msgpack==1.1.0
+narwhals==1.30.0
+numba==0.60.0
+numpy==1.23.5
+packaging==24.2
+pandas==2.2.3
+pillow==11.1.0
+platformdirs==4.3.6
+pooch==1.8.2
+progressbar==2.5
+protobuf==5.29.3
+psutil==7.0.0
+pyarrow==19.0.1
+pycparser==2.22
+pydantic==2.10.6
+pydantic_core==2.27.2
+pydeck==0.9.1
+python-dateutil==2.9.0.post0
+pytz==2025.1
+PyYAML==6.0.2
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+rpds-py==0.23.1
+safetensors==0.5.3
+scikit-learn==1.6.1
+scipy==1.15.2
+sentry-sdk==2.22.0
+setproctitle==1.3.5
+six==1.17.0
+smmap==5.0.2
+soundfile==0.13.1
+soxr==0.5.0.post1
+streamlit==1.43.2
+tenacity==9.0.0
+threadpoolctl==3.6.0
+tokenizers==0.21.1
+toml==0.10.2
+torchlibrosa==0.1.0
+tornado==6.4.2
+tqdm==4.67.1
+transformers==4.49.0
+typing_extensions==4.12.2
+tzdata==2025.1
+urllib3==2.3.0
+wandb==0.19.8
+wcwidth==0.2.13
+webdataset==0.2.111
+wget==3.2