update midi visualizer

app.py

@@ -281,7 +281,7 @@ if __name__ == "__main__":
         stop_btn = gr.Button("stop and output")
         output_midi_seq = gr.Variable()
         output_midi_visualizer = gr.HTML(elem_id="midi_visualizer_container")
-        output_audio = gr.Audio(label="output audio", format="mp3")
+        output_audio = gr.Audio(label="output audio", format="mp3", elem_id="midi_audio")
         output_midi = gr.File(label="output midi", file_types=[".mid"])
         run_event = run_btn.click(run, [input_model, tab_select, input_instruments, input_drum_kit, input_midi,
                                         input_midi_events, input_gen_events, input_temp, input_top_p, input_top_k,

javascript/app.js

@@ -58,12 +58,36 @@ document.addEventListener("DOMContentLoaded", function() {
             mse_receiver_inited = msg_receiver;
         }
     })
-})();
+})()
+
+function HSVtoRGB(h, s, v) {
+    let r, g, b, i, f, p, q, t;
+    i = Math.floor(h * 6);
+    f = h * 6 - i;
+    p = v * (1 - s);
+    q = v * (1 - f * s);
+    t = v * (1 - (1 - f) * s);
+    switch (i % 6) {
+        case 0: r = v; g = t; b = p; break;
+        case 1: r = q; g = v; b = p; break;
+        case 2: r = p; g = v; b = t; break;
+        case 3: r = p; g = q; b = v; break;
+        case 4: r = t; g = p; b = v; break;
+        case 5: r = v; g = p; b = q; break;
+    }
+    return {
+        r: Math.round(r * 255),
+        g: Math.round(g * 255),
+        b: Math.round(b * 255)
+    };
+}
 
 class MidiVisualizer extends HTMLElement{
     constructor() {
         super();
         this.midiEvents = [];
+        this.activeNotes = [];
+        this.midiTimes = [];
         this.wrapper = null;
         this.svg = null;
         this.timeLine = null;
@@ -71,10 +95,14 @@ class MidiVisualizer extends HTMLElement{
             noteHeight : 4,
             beatWidth: 32
         }
+        this.timePreBeat = 16
         this.svgWidth = 0;
         this.t1 = 0;
         this.playTime = 0
+        this.playTimeMs = 0
         this.colorMap = new Map();
+        this.playing = false;
+        this.timer = null;
         this.init();
     }
 
@@ -101,48 +129,46 @@ class MidiVisualizer extends HTMLElement{
         this.setPlayTime(0);
     }
 
-    setPlayTime(t){
-        this.playTime = t
-        let x = Math.round((t/16)*this.config.beatWidth)
-        this.timeLine.setAttribute('x1', `${x}`);
-        this.timeLine.setAttribute('y1', '0');
-        this.timeLine.setAttribute('x2', `${x}`);
-        this.timeLine.setAttribute('y2', `${this.config.noteHeight*128}`);
-    }
-
     clearMidiEvents(){
+        this.pause()
         this.midiEvents = [];
-        this.svgWidth = 0
-        this.svg.innerHTML = ''
+        this.activeNotes = [];
+        this.midiTimes = [];
         this.t1 = 0
         this.colorMap.clear()
-        this.svg.style.width = `${this.svgWidth}px`;
         this.setPlayTime(0);
+        this.playTimeMs = 0
+        this.svgWidth = 0
+        this.svg.innerHTML = ''
+        this.svg.style.width = `${this.svgWidth}px`;
         this.svg.appendChild(this.timeLine)
     }
 
     appendMidiEvent(midiEvent){
         if(midiEvent instanceof Array && midiEvent.length > 0){
-            this.midiEvents.push(midiEvent);
+
             this.t1 += midiEvent[1]
-            let t = this.t1*16 + midiEvent[2]
+            let t = this.t1*this.timePreBeat + midiEvent[2]
+            midiEvent = [midiEvent[0], t].concat(midiEvent.slice(3))
             if(midiEvent[0] === "note"){
-                let track = midiEvent[3]
-                let duration = midiEvent[4]
-                let channel = midiEvent[5]
-                let pitch = midiEvent[6]
-                let velocity = midiEvent[7]
-                let x = (t/16)*this.config.beatWidth
+                let track = midiEvent[2]
+                let duration = midiEvent[3]
+                let channel = midiEvent[4]
+                let pitch = midiEvent[5]
+                let velocity = midiEvent[6]
+                let x = (t/this.timePreBeat)*this.config.beatWidth
                 let y = (127 - pitch)*this.config.noteHeight
-                let w = (duration/16)*this.config.beatWidth
+                let w = (duration/this.timePreBeat)*this.config.beatWidth
                 let h = this.config.noteHeight
                 this.svgWidth = Math.ceil(Math.max(x + w, this.svgWidth))
                 let color = this.getColor(track, channel)
                 let opacity = Math.min(1, velocity/127 + 0.1).toFixed(2)
-                this.drawNote(x,y,w,h, `rgba(${color[0]}, ${color[1]}, ${color[2]}, ${opacity})`)
+                let rect = this.drawNote(x,y,w,h, `rgba(${color.r}, ${color.g}, ${color.b}, ${opacity})`)
+                midiEvent.push(rect)
                 this.setPlayTime(t);
                 this.wrapper.scrollTo(this.svgWidth - this.wrapper.offsetWidth, 0)
             }
+            this.midiEvents.push(midiEvent);
             this.svg.style.width = `${this.svgWidth}px`;
         }
 
@@ -154,7 +180,7 @@ class MidiVisualizer extends HTMLElement{
         if(!!color){
             return color;
         }
-        color = [Math.round(Math.random()*240) + 10, Math.round(Math.random()*240)+ 10, Math.round(Math.random()*240)+ 10];
+        color = HSVtoRGB(Math.random(),Math.random()*0.5 + 0.5,1);
         this.colorMap.set(key, color);
         return color;
     }
@@ -173,13 +199,116 @@ class MidiVisualizer extends HTMLElement{
         rect.setAttribute('height', `${Math.round(h)}`);
         this.svg.appendChild(rect);
         return rect
-  }
+    }
+
+    finishAppendMidiEvent(){
+        let midiEvents = this.midiEvents.sort((a, b)=>a[1]-b[1])
+        let tempo = (60 / 120) * 10 ** 3
+        let ms = 0
+        let lastT = 0
+        this.midiTimes.push({ms:ms, t: 0, tempo: tempo})
+        midiEvents.forEach((midiEvent)=>{
+            let t = midiEvent[1]
+            ms += ((t- lastT) / this.timePreBeat) * tempo
+            if(midiEvent[0]==="set_tempo"){
+                tempo = (60 / midiEvent[3]) * 10 ** 3
+                this.midiTimes.push({ms:ms, t: t, tempo: tempo})
+            }
+            lastT = t
+        })
+    }
+
+    setPlayTime(t){
+        this.playTime = t
+        let x = Math.round((t/this.timePreBeat)*this.config.beatWidth)
+        this.timeLine.setAttribute('x1', `${x}`);
+        this.timeLine.setAttribute('y1', '0');
+        this.timeLine.setAttribute('x2', `${x}`);
+        this.timeLine.setAttribute('y2', `${this.config.noteHeight*128}`);
+
+        this.wrapper.scrollTo(Math.max(0, x - this.wrapper.offsetWidth/2), 0)
+
+        if(this.playing){
+            let activeNotes = []
+            this.removeActiveNotes(this.activeNotes)
+            this.midiEvents.forEach((midiEvent)=>{
+                if(midiEvent[0] === "note"){
+                    let time = midiEvent[1]
+                    let duration = midiEvent[3]
+                    let note = midiEvent[midiEvent.length - 1]
+                    if(time <=this.playTime && time+duration>= this.playTime){
+                        activeNotes.push(note)
+                    }
+                }
+            })
+            this.addActiveNotes(activeNotes)
+        }
+    }
+
+    setPlayTimeMs(ms){
+        this.playTimeMs = ms
+        let playTime = 0
+        for(let i =0;i<this.midiTimes.length;i++){
+            let midiTime = this.midiTimes[i]
+            if(midiTime.ms>=ms){
+                break;
+            }
+            playTime = midiTime.t + (ms-midiTime.ms) * this.timePreBeat / midiTime.tempo
+        }
+        this.setPlayTime(playTime)
+    }
+
+    addActiveNotes(notes){
+        notes.forEach((note)=>{
+            this.activeNotes.push(note)
+            note.classList.add('active');
+        });
+    }
+
+    removeActiveNotes(notes){
+        notes.forEach((note)=>{
+            let idx = this.activeNotes.indexOf(note)
+            if(idx>-1)
+                this.activeNotes.splice(idx, 1);
+            note.classList.remove('active');
+        });
+    }
+
+    play(){
+        this.playing = true;
+        this.timer = setInterval(() => {
+            this.setPlayTimeMs(this.playTimeMs + 10)
+        }, 10);
+    }
+
+    pause(){
+        if(!!this.timer)
+            clearInterval(this.timer)
+        this.removeActiveNotes(this.activeNotes)
+        this.timer = null;
+        this.playing = false;
+    }
+
+
+    bindAudioPlayer(audio){
+        this.pause()
+        audio.addEventListener("play", (event)=>{
+            this.play()
+        })
+        audio.addEventListener("pause", (event)=>{
+            this.pause()
+        })
+        audio.addEventListener("timeupdate", (event)=>{
+            this.setPlayTimeMs(event.target.currentTime*10**3)
+        })
+    }
 }
 
 customElements.define('midi-visualizer', MidiVisualizer);
 
 (()=>{
     let midi_visualizer_container_inited = null
+    let midi_audio_inited = null;
     let midi_visualizer = document.createElement('midi-visualizer')
     onUiUpdate((m)=>{
         let app = gradioApp()
@@ -188,6 +317,11 @@ customElements.define('midi-visualizer', MidiVisualizer);
             midi_visualizer_container.appendChild(midi_visualizer)
             midi_visualizer_container_inited = midi_visualizer_container;
         }
+        let midi_audio = app.querySelector("#midi_audio > audio");
+        if(!!midi_audio && midi_audio_inited!==midi_audio){
+            midi_visualizer.bindAudioPlayer(midi_audio)
+            midi_audio_inited = midi_audio
+        }
     })
 
     function createProgressBar(progressbarContainer){
@@ -243,8 +377,9 @@ customElements.define('midi-visualizer', MidiVisualizer);
                 setProgressBar(midi_visualizer_container_inited, progress, total)
                 break;
             case "visualizer_end":
-                midi_visualizer.setPlayTime(0)
-                removeProgressBar(midi_visualizer_container_inited)
+                midi_visualizer.finishAppendMidiEvent()
+                midi_visualizer.setPlayTime(0);
+                removeProgressBar(midi_visualizer_container_inited);
                 break;
             default:
         }