trying to make a more reliable music mixer

src/bug-in-bun/aitube_ffmpeg/concatenate/concatenateAudio.ts

@@ -27,7 +27,7 @@ export async function concatenateAudio({
   audioTracks = [],
   audioFilePaths = [],
   crossfadeDurationInSec = 10,
-  outputFormat = "wav"
+  outputFormat = "mp3"
 }: ConcatenateAudioOptions): Promise<ConcatenateAudioOutput> {
   if (!Array.isArray(audioTracks)) {
     throw new Error("Audios must be provided in an array");
@@ -41,7 +41,7 @@ export async function concatenateAudio({
   if (audioTracks.length === 1 && audioTracks[0]) {
     const audioTrack = audioTracks[0]
     const outputFilePath = path.join(tempDir, `audio_0.${outputFormat}`)
-    await writeBase64ToFile(addBase64Header(audioTrack, "wav"), outputFilePath)
+    await writeBase64ToFile(addBase64Header(audioTrack, outputFormat), outputFilePath)
 
     // console.log("  |- there is only one track! so.. returning that")
     const { durationInSec } = await getMediaInfo(outputFilePath)
@@ -57,8 +57,8 @@ export async function concatenateAudio({
     let i = 0
     for (const track of audioTracks) {
       if (!track) { continue }
-      const audioFilePath = path.join(tempDir, `audio_${++i}.wav`);
-      await writeBase64ToFile(addBase64Header(track, "wav"), audioFilePath)
+      const audioFilePath = path.join(tempDir, `audio_${++i}.${outputFormat}`);
+      await writeBase64ToFile(addBase64Header(track, outputFormat), audioFilePath)
       
       audioFilePaths.push(audioFilePath);
     }

src/bug-in-bun/aitube_ffmpeg/concatenate/concatenateVideosWithAudio.ts

@@ -136,6 +136,7 @@ export const concatenateVideosWithAudio = async ({
     }
 
 
+    /*
     console.log("concatenateVideosWithAudio: DEBUG:", {
       videoTracksVolume,
       audioTrackVolume,
@@ -147,6 +148,7 @@ export const concatenateVideosWithAudio = async ({
       // additionalAudioVolume,
       finalOutputFilePath
      })
+     */
 
     // Set up event handlers for ffmpeg processing
     const promise = new Promise<string>((resolve, reject) => {

src/main.ts

@@ -8,12 +8,14 @@ import {
   concatenateVideosWithAudio,
   defaultExportFormat,
   type SupportedExportFormat,
-  type ConcatenateAudioOutput
+  type ConcatenateAudioOutput,
+  getMediaInfo
 // } from "@aitube/ffmpeg"
 } from "./bug-in-bun/aitube_ffmpeg"
 
 import { clapWithStoryboardsToVideoFile } from "./core/exporters/clapWithStoryboardsToVideoFile"
 import { clapWithVideosToVideoFile } from "./core/exporters/clapWithVideosToVideoFile"
+import { extractBase64 } from "@aitube/encoders"
 
 /**
  * Generate a .mp4 video inside a directory (if none is provided, it will be created in /tmp)
@@ -53,6 +55,14 @@ export async function clapToTmpVideoFilePath({
   const canUseVideos = videoSegments.length > 0
   const canUseStoryboards = !canUseVideos && storyboardSegments.length > 0
 
+  // we count the duration of the whole video
+  let totalDurationInMs = 0
+  clap.segments.forEach(s => {
+    if (s.endTimeInMs > totalDurationInMs) {
+      totalDurationInMs = s.endTimeInMs
+    }
+  })
+
   let videoFilePaths: string[] = []
 
   // two possibilities:
@@ -103,13 +113,38 @@ export async function clapToTmpVideoFilePath({
 
   console.log(`clapToTmpVideoFilePath: got ${musicSegments.length} music segments in total`)
   
+  // note: once we start with a certain type eg. mp3, there is no going to back
+  // another format like wav, we can't concatenate them together (well, not yet)
+  let detectedMusicTrackFormat = ''
+
+  // we count how much music has been generated
+  // if it is not enough to fill the full video, we will loop it (using cross-fading)
+  let availableMusicDurationInMs = 0
+
   for (const segment of musicSegments) {
-    audioTracks.push(
-      await writeBase64ToFile(
-        segment.assetUrl,
-        join(outputDir, `tmp_asset_${segment.id}.wav`)
-      )
+    const analysis = extractBase64(segment.assetUrl)
+    if (!detectedMusicTrackFormat) {
+      detectedMusicTrackFormat = analysis.extension
+    } else if (detectedMusicTrackFormat !== analysis.extension) {
+      throw new Error(`fatal error: concatenating a mixture of ${detectedMusicTrackFormat} and ${analysis.extension} tracks isn't supported yet`)
+    }
+
+    const { durationInMs, hasAudio } = await getMediaInfo(segment.assetUrl)
+
+    // we have to skip silent music tracks
+    if (!hasAudio) {
+      console.log(`skipping a silent music track`)
+      continue
+    }
+
+    const newTrackFileName = await writeBase64ToFile(
+      segment.assetUrl,
+      join(outputDir, `tmp_asset_${segment.id}.${analysis.extension}`)
     )
+
+    audioTracks.push(newTrackFileName)
+
+    availableMusicDurationInMs += durationInMs
   }
 
   let concatenatedAudio: ConcatenateAudioOutput | undefined = undefined
@@ -117,10 +152,37 @@ export async function clapToTmpVideoFilePath({
   if (audioTracks.length > 0) {
     console.log(`clapToTmpVideoFilePath: calling concatenateAudio over ${audioTracks.length} audio tracks`)
     
+    if (!detectedMusicTrackFormat) {
+      throw new Error(`uh that's weird, we couldn't detect the audio type`)
+    }
+
+    const availableMusicTracks = [...audioTracks]
+
+    // if we don't have enough music audio content
+    while (availableMusicDurationInMs < totalDurationInMs) {
+      let trackToUse = availableMusicTracks.shift()
+
+      // abort if there are no available tracks (for some reason)
+      if (!trackToUse) { break }
+
+      availableMusicTracks.push(trackToUse)
+
+      // we artificially duplicate it (note: this will be cross-faded)
+      const { durationInMs } = await getMediaInfo(trackToUse)
+
+      // let's abord if we have bad data
+      if (!durationInMs || durationInMs < 1000) { break }
+ 
+      audioTracks.push(trackToUse)
+  
+      availableMusicDurationInMs += durationInMs
+    }
+
     concatenatedAudio = await concatenateAudio({
-      output: join(outputDir, `tmp_asset_concatenated_audio.wav`),
+      output: join(outputDir, `tmp_asset_concatenated_audio.${detectedMusicTrackFormat}`),
       audioTracks,
-      crossfadeDurationInSec: 2 // 2 seconds
+      crossfadeDurationInSec: 2, // 2 seconds
+      outputFormat: detectedMusicTrackFormat
     })
     console.log(`clapToTmpVideoFilePath: concatenatedAudio = ${concatenatedAudio}`)
   }