# Mostly from: https://github.com/adefossez/seewav # Original author: adefossez import math import tempfile from pathlib import Path import subprocess import cairo import numpy as np import gradio as gr from pydub import AudioSegment import re import colorsys def read_audio(audio, seek=None, duration=None): """ Read the `audio` file, starting at `seek` (or 0) seconds for `duration` (or all) seconds. Returns `float[channels, samples]`. """ audio_segment = AudioSegment.from_file(audio) channels = audio_segment.channels samplerate = audio_segment.frame_rate if seek is not None: seek_ms = int(seek * 1000) audio_segment = audio_segment[seek_ms:] if duration is not None: duration_ms = int(duration * 1000) audio_segment = audio_segment[:duration_ms] samples = audio_segment.get_array_of_samples() wav = np.array(samples, dtype=np.float32) return wav.reshape(channels, -1), samplerate def sigmoid(x): return 1 / (1 + np.exp(-x)) def envelope(wav, window, stride): """ Extract the envelope of the waveform `wav` (float[samples]), using average pooling with `window` samples and the given `stride`. """ # pos = np.pad(np.maximum(wav, 0), window // 2) wav = np.pad(wav, window // 2) out = [] for off in range(0, len(wav) - window, stride): frame = wav[off : off + window] out.append(np.maximum(frame, 0).mean()) out = np.array(out) # Some form of audio compressor based on the sigmoid. out = 1.9 * (sigmoid(2.5 * out) - 0.5) return out def draw_env(envs, out, fg_colors, bg_color, size): """ Internal function, draw a single frame (two frames for stereo) using cairo and save it to the `out` file as png. envs is a list of envelopes over channels, each env is a float[bars] representing the height of the envelope to draw. Each entry will be represented by a bar. """ surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, *size) ctx = cairo.Context(surface) ctx.scale(*size) ctx.set_source_rgb(*bg_color) ctx.rectangle(0, 0, 1, 1) ctx.fill() K = len(envs) # Number of waves to draw (waves are stacked vertically) T = len(envs[0]) # Numbert of time steps pad_ratio = 0.1 # spacing ratio between 2 bars width = 1.0 / (T * (1 + 2 * pad_ratio)) pad = pad_ratio * width delta = 2 * pad + width ctx.set_line_width(width) for step in range(T): for i in range(K): half = 0.5 * envs[i][step] # (semi-)height of the bar half /= K # as we stack K waves vertically midrule = (1 + 2 * i) / (2 * K) # midrule of i-th wave ctx.set_source_rgb(*fg_colors[i]) ctx.move_to(pad + step * delta, midrule - half) ctx.line_to(pad + step * delta, midrule) ctx.stroke() ctx.set_source_rgba(*fg_colors[i], 0.8) ctx.move_to(pad + step * delta, midrule) ctx.line_to(pad + step * delta, midrule + 0.9 * half) ctx.stroke() surface.write_to_png(out) def interpole(x1, y1, x2, y2, x): return y1 + (y2 - y1) * (x - x1) / (x2 - x1) def visualize( progress, audio, tmp, out, seek=None, duration=None, rate=60, bars=50, speed=4, time=0.4, oversample=3, fg_color=(0.2, 0.2, 0.2), fg_color2=(0.5, 0.3, 0.6), bg_color=(1, 1, 1), size=(400, 400), stereo=False, ): """ Generate the visualisation for the `audio` file, using a `tmp` folder and saving the final video in `out`. `seek` and `durations` gives the extract location if any. `rate` is the framerate of the output video. `bars` is the number of bars in the animation. `speed` is the base speed of transition. Depending on volume, actual speed will vary between 0.5 and 2 times it. `time` amount of audio shown at once on a frame. `oversample` higher values will lead to more frequent changes. `fg_color` is the rgb color to use for the foreground. `fg_color2` is the rgb color to use for the second wav if stereo is set. `bg_color` is the rgb color to use for the background. `size` is the `(width, height)` in pixels to generate. `stereo` is whether to create 2 waves. """ try: wav, sr = read_audio(audio, seek=seek, duration=duration) except (IOError, ValueError) as err: raise gr.Error(err) # wavs is a list of wav over channels wavs = [] if stereo: assert wav.shape[0] == 2, "stereo requires stereo audio file" wavs.append(wav[0]) wavs.append(wav[1]) else: wav = wav.mean(0) wavs.append(wav) for i, wav in enumerate(wavs): wavs[i] = wav / wav.std() window = int(sr * time / bars) stride = int(window / oversample) # envs is a list of env over channels envs = [] for wav in wavs: env = envelope(wav, window, stride) env = np.pad(env, (bars // 2, 2 * bars)) envs.append(env) duration = len(wavs[0]) / sr frames = int(rate * duration) smooth = np.hanning(bars) gr.Info("Generating the frames...") for idx in progress(range(frames)): pos = (((idx / rate)) * sr) / stride / bars off = int(pos) loc = pos - off denvs = [] for env in envs: env1 = env[off * bars : (off + 1) * bars] env2 = env[(off + 1) * bars : (off + 2) * bars] # we want loud parts to be updated faster maxvol = math.log10(1e-4 + env2.max()) * 10 speedup = np.clip(interpole(-6, 0.5, 0, 2, maxvol), 0.5, 2) w = sigmoid(speed * speedup * (loc - 0.5)) denv = (1 - w) * env1 + w * env2 denv *= smooth denvs.append(denv) draw_env(denvs, tmp / f"{idx:06d}.png", (fg_color, fg_color2), bg_color, size) gr.Info("Encoding the animation video...") subprocess.run([ "ffmpeg", "-y", "-loglevel", "panic", "-r", str(rate), "-f", "image2", "-s", f"{size[0]}x{size[1]}", "-i", "%06d.png", "-i", audio, "-c:a", "aac", "-vcodec", "libx264", "-crf", "10", "-pix_fmt", "yuv420p", out.resolve() ], check=True, cwd=tmp) return out def parse_color(colorstr): """ Given a comma separated rgb(a) colors, returns a 4-tuple of float. """ try: r, g, b = [float(i) for i in colorstr.split(",")] return r, g, b except ValueError: raise gr.Error( "Format for color is 3 floats separated by commas 0.xx,0.xx,0.xx, rgb order" ) def hex_to_rgb(color): """ Convert color codes to RGBA format. Supports: - HEX codes with # (e.g., "#FFA07A" or "#FFA07A80" for alpha) - HEX codes without # (e.g., "FFA07A" or "FFA07A80") - RGB format (e.g., "rgb(255, 160, 122)") - RGBA format (e.g., "rgba(255, 160, 122, 0.5)") - HSV format (e.g., "hsv(17, 52, 100)") Returns: (r, g, b, a): Tuple of float values (0-1) """ print(f"Received color: {color}") # Debugging line # HEX with or without # if color.startswith("#") or (len(color) in [6, 3, 8, 4] and all(c in "0123456789ABCDEFabcdef" for c in color)): color = color.lstrip("#") # Remove # if present # Support 3-digit HEX (e.g., "FA5" -> "FFAA55") if len(color) == 3: color = "".join([c * 2 for c in color]) # Expand to 6-digit HEX # Support 4-digit HEX with alpha (e.g., "FA58" -> "FFAA5588") if len(color) == 4: color = "".join([c * 2 for c in color]) # Expand to 8-digit HEX # HEX without alpha if len(color) == 6: return (int(color[0:2], 16) / 255.0, int(color[2:4], 16) / 255.0, int(color[4:6], 16) / 255.0, ) # HEX with alpha (8-digit HEX) if len(color) == 8: return (int(color[0:2], 16) / 255.0, int(color[2:4], 16) / 255.0, int(color[4:6], 16) / 255.0 ) # RGB format (rgb(r, g, b)) match_rgb = re.match(r"rgb\((\d+),\s*(\d+),\s*(\d+)\)", color) if match_rgb: r, g, b = map(int, match_rgb.groups()) return (r / 255.0, g / 255.0, b / 255.0) # RGBA format (rgba(r, g, b, a)) match_rgba = re.match(r"rgba\((\d+),\s*(\d+),\s*(\d+),\s*([\d.]+)\)", color) if match_rgba: r, g, b, a = match_rgba.groups() return (int(r) / 255.0, int(g) / 255.0, int(b) / 255.0) # HSV format (hsv(h, s, v)) match_hsv = re.match(r"hsv\((\d+),\s*(\d+),\s*(\d+)\)", color) if match_hsv: h, s, v = map(int, match_hsv.groups()) r, g, b = colorsys.hsv_to_rgb(h / 360.0, s / 100.0, v / 100.0) return (r, g, b) raise ValueError(f"Invalid color format: {color}") def do_viz( inp_aud, inp_bgcolor, inp_color1, inp_nbars, inp_vidw, inp_vidh, progress=gr.Progress(), ): with tempfile.TemporaryDirectory() as tmp, tempfile.NamedTemporaryFile( suffix=".mp4", delete=False ) as out: return visualize( progress.tqdm, inp_aud, Path(tmp), Path(out.name), bars=inp_nbars, fg_color=hex_to_rgb(inp_color1), bg_color=hex_to_rgb(inp_bgcolor), size=(inp_vidw, inp_vidh), ) import gradio as gr ABOUT = """ # seewav GUI > Have an audio clip but need a video (e.g. for X/Twitter)? **Convert audio into a video!** An online graphical user interface for [seewav](https://github.com/adefossez/seewav). """ with gr.Blocks() as demo: gr.Markdown(ABOUT) with gr.Row(): with gr.Column(): inp_aud = gr.Audio(type='filepath') with gr.Group(): inp_color1 = gr.ColorPicker( label="Color", info="Color of the top waveform", value="#00237E", interactive=True, ) inp_bgcolor = gr.ColorPicker( label="Background Color", info="Color of the background", value="#000000", interactive=True, ) with gr.Accordion("Advanced Configuration", open=False): inp_nbars = gr.Slider( label="Num. Bars", value=50, interactive=True, minimum=5, maximum=1500, ) inp_vidw = gr.Slider( label="Video Width", value=400, interactive=True, minimum=100, maximum=3000, ) inp_vidh = gr.Slider( label="Video Height", value=400, interactive=True, minimum=100, maximum=3000, ) inp_go = gr.Button("Visualize", variant="primary") with gr.Column(): out_vid = gr.Video(interactive=False) inp_go.click( do_viz, inputs=[ inp_aud, inp_bgcolor, inp_color1, inp_nbars, inp_vidw, inp_vidh, ], outputs=[out_vid], ) demo.queue(api_open=False, default_concurrency_limit=20).launch(show_api=False)