another step for the Stories Factory (mp4 generation)

src/core/base64/extractBase64.mts

@@ -2,8 +2,13 @@
  * break a base64 string into sub-components
  */
 export function extractBase64(base64: string = ""): {
+
+  // file format eg. video/mp4 text/html audio/wave
   mimetype: string;
+
+  // file extension eg. .mp4 .html .wav
   extension: string;
+  
   data: string;
   buffer: Buffer;
   blob: Blob;

src/core/converters/htmlToBase64Png.mts

@@ -28,7 +28,7 @@ export async function htmlToBase64Png({
   }
 
   const browser = await puppeteer.launch({
-    headless: "new",
+    headless: true,
 
     // apparently we need those, see:
     // https://unix.stackexchange.com/questions/694734/puppeteer-in-alpine-docker-with-chromium-headless-dosent-seems-to-work

src/core/ffmpeg/addTextToVideo.mts

@@ -1,23 +1,37 @@
-import { createTextOverlayImage } from "./createTextOverlayImage.mts";
-import { addImageToVideo } from "./addImageToVideo.mts";
+import { createTextOverlayImage } from "./createTextOverlayImage.mts"
+import { addImageToVideo } from "./addImageToVideo.mts"
+import { deleteFile } from "../files/deleteFile.mts"
 
-export async function addTextToVideo() {
+export async function addTextToVideo({
+  inputVideoPath,
+  outputVideoPath,
+  text,
+  width,
+  height,
+}: {
+  inputVideoPath: string
+  outputVideoPath: string
+  text: string
+  width: number
+  height: number
+}): Promise<string> {
   
-  const inputVideoPath = "/Users/jbilcke/Downloads/use_me.mp4"
-
-  const { filePath } = await createTextOverlayImage({
-    text: "This tech is hot 🥵",
-    width: 1024 ,
-    height: 576,
+  const { filePath: temporaryImageOverlayFilePath } = await createTextOverlayImage({
+    text,
+    width,
+    height,
   })
-  console.log("filePath:", filePath)
 
-  /*
+  console.log("addTextToVideo: temporaryImageOverlayFilePath:", temporaryImageOverlayFilePath)
+
   const pathToVideo = await addImageToVideo({
     inputVideoPath,
-    inputImagePath: filePath,
+    inputImagePath: temporaryImageOverlayFilePath,
+    outputVideoPath,
   })
 
-  console.log("pathToVideo:", pathToVideo)
-  */
+  await deleteFile(temporaryImageOverlayFilePath)
+
+  console.log("addTextToVideo: outputVideoPath:", outputVideoPath)
+  return outputVideoPath
 }

src/core/files/deleteFile.mts

@@ -0,0 +1,14 @@
+import { unlink, rm } from "node:fs/promises"
+
+export async function deleteFile(filePath: string, debug?: boolean): Promise<boolean> {
+  try {
+    await rm(filePath, { recursive: true, force: true })
+    // await unlink(filePath)
+    return true
+  } catch (err) {
+    if (debug) {
+      console.error(`failed to unlink file at ${filePath}: ${err}`)
+    }
+  }
+  return false
+}

src/core/files/deleteFileWithName.mts

@@ -1,17 +1,11 @@
 import { promises as fs } from "node:fs"
 import path from "node:path"
+import { deleteFile } from "./deleteFile.mts"
 
 export const deleteFilesWithName = async (dir: string, name: string, debug?: boolean) => {
   for (const file of await fs.readdir(dir)) {
     if (file.includes(name)) {
-      const filePath = path.join(dir, file)
-      try {
-        await fs.unlink(filePath)
-      } catch (err) {
-        if (debug) {
-          console.error(`failed to unlink file in ${filePath}: ${err}`)
-        }
-      }
+      await deleteFile(path.join(dir, file))
     }
   }
 }

src/core/files/getRandomDirectory.mts

@@ -0,0 +1,8 @@
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { mkdtemp } from "node:fs/promises"
+import { v4 as uuidv4 } from "uuid"
+
+export async function getRandomDirectory(): Promise<string> {
+  return mkdtemp(join(tmpdir(), uuidv4()))
+}

src/core/files/removeTmpFiles.mts

@@ -1,22 +1,18 @@
 import { existsSync, promises as fs } from "node:fs"
 
-import { keepTemporaryFiles } from "../config.mts"
-
 // note: this function will never fail
 export async function removeTemporaryFiles(filesPaths: string[]) {
   try {
-    if (!keepTemporaryFiles) {
-      // Cleanup temporary files - you could choose to do this or leave it to the user
-      await Promise.all(filesPaths.map(async (filePath) => {
-        try {
-          if (existsSync(filePath)) {
-            await fs.unlink(filePath)
-          }
-        } catch (err) {
-          //
+    // Cleanup temporary files - you could choose to do this or leave it to the user
+    await Promise.all(filesPaths.map(async (filePath) => {
+      try {
+        if (existsSync(filePath)) {
+          await fs.rm(filePath)
         }
-      }))
-    }
+      } catch (err) {
+        //
+      }
+    }))
   } catch (err) {
     // no big deal, except a bit of tmp file leak
     // although.. if delete failed, it could also indicate

src/core/utils/startOfSegment1IsWithinSegment2.mts

@@ -0,0 +1,6 @@
+import { ClapSegment } from "../clap/types.mts";
+
+export function startOfSegment1IsWithinSegment2(s1: ClapSegment, s2: ClapSegment) {
+  const startOfSegment1 = s1.startTimeInMs 
+  return s2.startTimeInMs <= startOfSegment1 && startOfSegment1 <= s2.endTimeInMs
+}

src/index.mts

@@ -4,6 +4,8 @@ import { Blob } from "buffer"
 
 import { parseClap } from "./core/clap/parseClap.mts"
 import { ClapProject } from "./core/clap/types.mts"
+import { clapToTmpVideoFilePath } from "./main.mts"
+import { deleteFile } from "./core/files/deleteFile.mts"
 
 const app = express()
 const port = 7860
@@ -49,10 +51,23 @@ app.post("/", async (req, res) => {
   req.on("end", async () => {
     let clapProject: ClapProject
     try {
-      let fileData = Buffer.concat(data);
-      const clapBlob = new Blob([fileData]);
-      clapProject = await parseClap(clapBlob);
-      console.log("got a clap project!:", clapProject)
+      let fileData = Buffer.concat(data)
+
+      const clap = await parseClap(new Blob([fileData]));
+      console.log("got a clap project:", clapProject)
+
+      const {
+        tmpWorkDir,
+        outputFilePath,
+      } = await clapToTmpVideoFilePath({ clap })
+      console.log("got an output file at:", outputFilePath)
+
+      res.download(outputFilePath, async () => {
+        // clean-up after ourselves (we clear the whole tmp directory)
+        await deleteFile(tmpWorkDir)
+        console.log("cleared the temporary folder")
+      })
+      return
     } catch (err) {
       console.error(`failed to parse the request: ${err}`)
       res.status(500)
@@ -60,10 +75,6 @@ app.post("/", async (req, res) => {
       res.end()
       return
     }
-    // TODO read the mp4 file and convert it to 
-    res.status(200)
-    res.write("TODO")
-    res.end()
   });
 })
 

src/main.mts

@@ -1,7 +1,4 @@
-import { tmpdir } from "node:os"
 import { join } from "node:path"
-import { mkdtemp } from "node:fs/promises"
-import { v4 as uuidv4 } from "uuid"
 
 import { ClapProject } from "./core/clap/types.mts";
 import { concatenateAudio } from "./core/ffmpeg/concatenateAudio.mts";
@@ -9,6 +6,11 @@ import { concatenateVideosWithAudio } from "./core/ffmpeg/concatenateVideosWithA
 import { writeBase64ToFile } from "./core/files/writeBase64ToFile.mts";
 import { concatenateVideos } from "./core/ffmpeg/concatenateVideos.mts"
 import { deleteFilesWithName } from "./core/files/deleteFileWithName.mts"
+import { getRandomDirectory } from "./core/files/getRandomDirectory.mts";
+import { addTextToVideo } from "./core/ffmpeg/addTextToVideo.mts";
+import { startOfSegment1IsWithinSegment2 } from "./core/utils/startOfSegment1IsWithinSegment2.mts";
+import { deleteFile } from "./core/files/deleteFile.mts";
+import { extractBase64 } from "./core/base64/extractBase64.mts";
 
 /**
  * Generate a .mp4 video inside a direcory (if none is provided, it will be created in /tmp)
@@ -16,56 +18,139 @@ import { deleteFilesWithName } from "./core/files/deleteFileWithName.mts"
  * @param clap 
  * @returns file path to the final .mp4
  */
-export async function clapToTmpVideoFilePath(clap: ClapProject, dir = ""): Promise<string> {
+export async function clapToTmpVideoFilePath({
+  clap,
+  outputDir = "",
+  clearTmpFilesAtEnd = false
+}: {
+  clap: ClapProject
 
-  dir = dir || (await mkdtemp(join(tmpdir(), uuidv4())))
+  outputDir?: string
+
+  // if you leave this to false, you will have to clear files yourself
+  // (eg. after sending the final video file over)
+  clearTmpFilesAtEnd?: boolean
+}): Promise<{
+  tmpWorkDir: string
+  outputFilePath: string
+}> {
+
+  outputDir = outputDir || (await getRandomDirectory())
 
   const videoFilePaths: string[] = []
   const videoSegments = clap.segments.filter(s => s.category === "video" && s.assetUrl.startsWith("data:video/"))
 
   for (const segment of videoSegments) {
-    videoFilePaths.push(
-      await writeBase64ToFile(
-        segment.assetUrl,
-        join(dir, `tmp_asset_${segment.id}.mp4`)
-      )
+
+    const base64Info = extractBase64(segment.assetUrl)
+ 
+    // we write it to the disk *unconverted* (it might be a mp4, a webm or something else)
+    let videoSegmentFilePath = await writeBase64ToFile(
+      segment.assetUrl,
+      join(outputDir, `tmp_asset_${segment.id}.${base64Info.extension}`)
+    )
+
+    const interfaceSegments = clap.segments.filter(s =>
+      s.assetUrl.startsWith("data:text/") &&
+      s.category === "interface" &&
+      startOfSegment1IsWithinSegment2(s, segment)
+    )
+    const interfaceSegment = interfaceSegments.at(0)
+    if (interfaceSegment) {
+      // here we are free to use mp4, since this is an internal intermediary format
+      const videoSegmentWithOverlayFilePath = join(outputDir, `tmp_asset_${segment.id}_with_interface.mp4`)
+
+      await addTextToVideo({
+        inputVideoPath: videoSegmentFilePath,
+        outputVideoPath: videoSegmentWithOverlayFilePath,
+        text: atob(extractBase64(interfaceSegment.assetUrl).data),
+        width: clap.meta.width,
+        height: clap.meta.height,
+      })
+
+      // we overwrite
+      await deleteFile(videoSegmentFilePath)
+      videoSegmentFilePath = videoSegmentWithOverlayFilePath
+    }
+
+    const dialogueSegments = clap.segments.filter(s =>
+      s.assetUrl.startsWith("data:audio/") &&
+      s.category === "dialogue" &&
+      startOfSegment1IsWithinSegment2(s, segment)
     )
+    const dialogueSegment = dialogueSegments.at(0)
+    if (dialogueSegment) {
+      extractBase64(dialogueSegment.assetUrl)
+      const base64Info = extractBase64(segment.assetUrl)
+
+      const dialogueSegmentFilePath = await writeBase64ToFile(
+        dialogueSegment.assetUrl,
+        join(outputDir, `tmp_asset_${segment.id}_dialogue.${base64Info.extension}`)
+      )
+  
+      const finalFilePathOfVideoWithSound = await concatenateVideosWithAudio({
+        output: join(outputDir, `${segment.id}_video_with_audio.mp4`),
+        audioFilePath: dialogueSegmentFilePath,
+        videoFilePaths: [videoSegmentFilePath],
+        // videos are silent, so they can stay at 0
+        videoTracksVolume: 0.0,
+        audioTrackVolume: 1.0,
+      })
+
+      // we delete the temporary dialogue file
+      await deleteFile(dialogueSegmentFilePath)
+
+      // we overwrite the video segment
+      await deleteFile(videoSegmentFilePath)
+
+      videoSegmentFilePath = finalFilePathOfVideoWithSound
+    }
+
+    videoFilePaths.push(videoSegmentFilePath)
   }
 
-  const concatenatedVideosNoSound = await concatenateVideos({
+  const concatenatedVideosNoMusic = await concatenateVideos({
     videoFilePaths,
-    output: join(dir, `tmp_asset_concatenated_videos.mp4`)
+    output: join(outputDir, `tmp_asset_concatenated_videos.mp4`)
   })
 
   const audioTracks: string[] = []
 
-  const musicSegments = clap.segments.filter(s => s.category === "music" && s.assetUrl.startsWith("data:audio/"))
+  const musicSegments = clap.segments.filter(s =>
+      s.category === "music" &&
+      s.assetUrl.startsWith("data:audio/")
+  )
   for (const segment of musicSegments) {
     audioTracks.push(
       await writeBase64ToFile(
         segment.assetUrl,
-        join(dir, `tmp_asset_${segment.id}.wav`)
+        join(outputDir, `tmp_asset_${segment.id}.wav`)
       )
     )
   }
 
   const concatenatedAudio = await concatenateAudio({
-    output: join(dir, `tmp_asset_concatenated_audio.wav`),
+    output: join(outputDir, `tmp_asset_concatenated_audio.wav`),
     audioTracks,
     crossfadeDurationInSec: 2 // 2 seconds
   })
 
-  const finalFilePathOfVideoWithSound = await concatenateVideosWithAudio({
-    output: join(dir, `final_video.mp4`),
+  const finalFilePathOfVideoWithMusic = await concatenateVideosWithAudio({
+    output: join(outputDir, `final_video.mp4`),
     audioFilePath: concatenatedAudio.filepath,
-    videoFilePaths: [concatenatedVideosNoSound.filepath],
+    videoFilePaths: [concatenatedVideosNoMusic.filepath],
     // videos are silent, so they can stay at 0
-    videoTracksVolume: 0.0,
-    audioTrackVolume: 1.0,
+    videoTracksVolume: 0.85,
+    audioTrackVolume: 0.15, // let's keep the music volume low
   })
 
-  // we delete all the temporary assets
-  await deleteFilesWithName(dir, `tmp_asset_`)
+  if (clearTmpFilesAtEnd) {
+    // we delete all the temporary assets
+    await deleteFilesWithName(outputDir, `tmp_asset_`)
+  }
 
-  return finalFilePathOfVideoWithSound
+  return {
+    tmpWorkDir: outputDir,
+    outputFilePath: finalFilePathOfVideoWithMusic
+  }
 }