it works

package-lock.json

@@ -23,6 +23,8 @@
         "node-fetch": "^3.3.1",
         "openai": "^3.3.0",
         "puppeteer": "^20.8.0",
+        "resize-base64": "^1.0.12",
+        "sharp": "^0.32.4",
         "temp-dir": "^3.0.0",
         "ts-node": "^10.9.1",
         "uuid": "^9.0.0",
@@ -766,6 +768,11 @@
         "readable-stream": "^3.4.0"
       }
     },
+    "node_modules/bluebird": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
+    },
     "node_modules/body-parser": {
       "version": "1.20.1",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@@ -1144,6 +1151,43 @@
         "url": "https://github.com/sponsors/d-fischer"
       }
     },
+    "node_modules/create-readme": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/create-readme/-/create-readme-1.2.0.tgz",
+      "integrity": "sha512-HK3Jpr+5qIYHJHFBeZs0SaLdu7gGHdqS3KORfu5Ivp64Pw+SRSPZ9e0p++NP8JlYeqycxFdVg5yTix0VM5kHFA==",
+      "dependencies": {
+        "bluebird": "^3.1.1",
+        "commander": "^2.9.0",
+        "debug": "^4.1.0",
+        "github-url-to-object": "^4.0.4",
+        "mustache": "^3.0.0",
+        "require-all": "^3.0.0"
+      },
+      "bin": {
+        "create-readme": "bin/create-readme.js"
+      }
+    },
+    "node_modules/create-readme/node_modules/debug": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "dependencies": {
+        "ms": "2.1.2"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/create-readme/node_modules/ms": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+    },
     "node_modules/create-require": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
@@ -1718,6 +1762,11 @@
         "node": ">=10.13.0"
       }
     },
+    "node_modules/ffmpeg-concat/node_modules/chownr": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
+    },
     "node_modules/ffmpeg-concat/node_modules/fs-extra": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz",
@@ -1739,6 +1788,84 @@
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/ffmpeg-concat/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/ffmpeg-concat/node_modules/node-addon-api": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
+      "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
+    },
+    "node_modules/ffmpeg-concat/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/ffmpeg-concat/node_modules/sharp": {
+      "version": "0.31.3",
+      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.3.tgz",
+      "integrity": "sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "color": "^4.2.3",
+        "detect-libc": "^2.0.1",
+        "node-addon-api": "^5.0.0",
+        "prebuild-install": "^7.1.1",
+        "semver": "^7.3.8",
+        "simple-get": "^4.0.1",
+        "tar-fs": "^2.1.1",
+        "tunnel-agent": "^0.6.0"
+      },
+      "engines": {
+        "node": ">=14.15.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/ffmpeg-concat/node_modules/tar-fs": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
+      "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
+      "dependencies": {
+        "chownr": "^1.1.1",
+        "mkdirp-classic": "^0.5.2",
+        "pump": "^3.0.0",
+        "tar-stream": "^2.1.4"
+      }
+    },
+    "node_modules/ffmpeg-concat/node_modules/tar-stream": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+      "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+      "dependencies": {
+        "bl": "^4.0.3",
+        "end-of-stream": "^1.4.1",
+        "fs-constants": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.1.1"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/ffmpeg-concat/node_modules/universalify": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
@@ -2145,6 +2272,14 @@
       "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
       "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="
     },
+    "node_modules/github-url-to-object": {
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/github-url-to-object/-/github-url-to-object-4.0.6.tgz",
+      "integrity": "sha512-NaqbYHMUAlPcmWFdrAB7bcxrNIiiJWJe8s/2+iOc9vlcHlwHqSGrPk+Yi3nu6ebTwgsZEa7igz+NH2vEq3gYwQ==",
+      "dependencies": {
+        "is-url": "^1.1.0"
+      }
+    },
     "node_modules/gl": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/gl/-/gl-6.0.2.tgz",
@@ -2626,6 +2761,11 @@
       "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
       "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA=="
     },
+    "node_modules/is-url": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
+      "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww=="
+    },
     "node_modules/isarray": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
@@ -3118,6 +3258,17 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
     },
+    "node_modules/mustache": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/mustache/-/mustache-3.2.1.tgz",
+      "integrity": "sha512-RERvMFdLpaFfSRIEe632yDm5nsd0SDKn8hGmcUwswnyiE5mtdZLDybtHAz6hjJhawokF0hXvGLtx9mrQfm6FkA==",
+      "bin": {
+        "mustache": "bin/mustache"
+      },
+      "engines": {
+        "npm": ">=1.4.0"
+      }
+    },
     "node_modules/nan": {
       "version": "2.17.0",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz",
@@ -3212,9 +3363,9 @@
       }
     },
     "node_modules/node-addon-api": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
-      "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
+      "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
     },
     "node_modules/node-bitmap": {
       "version": "0.0.1",
@@ -4007,6 +4158,14 @@
         "uuid": "bin/uuid"
       }
     },
+    "node_modules/require-all": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/require-all/-/require-all-3.0.0.tgz",
+      "integrity": "sha512-jPGN876lc5exWYrMcgZSd7U42P0PmVQzxnQB13fCSzmyGnqQWW4WUz5DosZ/qe24hz+5o9lSvW2epBNZ1xa6Fw==",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -4020,6 +4179,14 @@
       "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
       "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
     },
+    "node_modules/resize-base64": {
+      "version": "1.0.12",
+      "resolved": "https://registry.npmjs.org/resize-base64/-/resize-base64-1.0.12.tgz",
+      "integrity": "sha512-OIiowGyyrwzrwbufDLPpBfofuUAkPSA4OPzT5rTGxfU4iHszKYHNICd1uhnRNDBFnU08WU8mt94+OaBXgN+WMg==",
+      "dependencies": {
+        "create-readme": "^1.1.0"
+      }
+    },
     "node_modules/resolve-from": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
@@ -4166,18 +4333,18 @@
       "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
     },
     "node_modules/sharp": {
-      "version": "0.31.3",
-      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.3.tgz",
-      "integrity": "sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==",
+      "version": "0.32.4",
+      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.4.tgz",
+      "integrity": "sha512-exUnZewqVZC6UXqXuQ8fyJJv0M968feBi04jb9GcUHrWtkRoAKnbJt8IfwT4NJs7FskArbJ14JAFGVuooszoGg==",
       "hasInstallScript": true,
       "dependencies": {
         "color": "^4.2.3",
-        "detect-libc": "^2.0.1",
-        "node-addon-api": "^5.0.0",
+        "detect-libc": "^2.0.2",
+        "node-addon-api": "^6.1.0",
         "prebuild-install": "^7.1.1",
-        "semver": "^7.3.8",
+        "semver": "^7.5.4",
         "simple-get": "^4.0.1",
-        "tar-fs": "^2.1.1",
+        "tar-fs": "^3.0.4",
         "tunnel-agent": "^0.6.0"
       },
       "engines": {
@@ -4187,11 +4354,6 @@
         "url": "https://opencollective.com/libvips"
       }
     },
-    "node_modules/sharp/node_modules/chownr": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
-    },
     "node_modules/sharp/node_modules/lru-cache": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@@ -4218,29 +4380,13 @@
       }
     },
     "node_modules/sharp/node_modules/tar-fs": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
-      "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz",
+      "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==",
       "dependencies": {
-        "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
         "pump": "^3.0.0",
-        "tar-stream": "^2.1.4"
-      }
-    },
-    "node_modules/sharp/node_modules/tar-stream": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
-      "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
-      "dependencies": {
-        "bl": "^4.0.3",
-        "end-of-stream": "^1.4.1",
-        "fs-constants": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^3.1.1"
-      },
-      "engines": {
-        "node": ">=6"
+        "tar-stream": "^3.1.5"
       }
     },
     "node_modules/shebang-command": {

package.json

@@ -30,6 +30,8 @@
     "node-fetch": "^3.3.1",
     "openai": "^3.3.0",
     "puppeteer": "^20.8.0",
+    "resize-base64": "^1.0.12",
+    "sharp": "^0.32.4",
     "temp-dir": "^3.0.0",
     "ts-node": "^10.9.1",
     "uuid": "^9.0.0",

src/production/renderStaticScene.mts

@@ -14,20 +14,23 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
   let imageBase64 = ""
   let error = ""
 
+  const width = 1024
+  const height = 512
+
   try {
     console.log(`calling generateImageSDXLAsBase64 with: `, JSON.stringify({
       positivePrompt: scene.prompt,
       seed: scene.seed || undefined,
       nbSteps: scene.nbSteps || undefined,
-      width: 1024,
-      height: 512
+      width,
+      height,
     }, null, 2))
     imageBase64 = await generateImageSDXLAsBase64({
       positivePrompt: scene.prompt,
       seed: scene.seed || undefined,
       nbSteps: scene.nbSteps || undefined,
-      width: 1024,
-      height: 512
+      width,
+      height
     })
     console.log("successful generation!", imageBase64.slice(0, 30))
     error = ""
@@ -64,7 +67,7 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
       error = "failed to segment the image"
     } else {
       console.log("got the first frame! segmenting..")
-      const result = await segmentImage(tmpImageFilePath, actionnables)
+      const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
       mask = result.pngInBase64
       segments = result.segments
       console.log("success!", {  segments })

src/production/renderVideoScene.mts

@@ -13,6 +13,9 @@ export async function renderVideoScene(scene: RenderRequest): Promise<RenderedSc
   let url = ""
   let error = ""
 
+  const width = 576
+  const height = 320
+
   try {
     url = await generateVideo(scene.prompt, {
       seed: getValidNumber(scene.seed, 0, 2147483647, generateSeed()),
@@ -50,7 +53,7 @@ export async function renderVideoScene(scene: RenderRequest): Promise<RenderedSc
         error = "failed to segment the image"
       } else {
         console.log("got the first frame! segmenting..")
-        const result = await segmentImage(firstFrameFilePath, actionnables)
+        const result = await segmentImage(firstFrameFilePath, actionnables, width, height)
         mask = result.pngInBase64
         segments = result.segments
         // console.log("success!", {  segments })

src/utils/resizeBase64Image.mts

@@ -0,0 +1,16 @@
+import sharp from "sharp"
+
+export async function resizeBase64Image(imgBase64: string, targetWidth: number, targetHeight: number): Promise<string> {
+  // Convert base64 to buffer
+  const buffer = Buffer.from(imgBase64, 'base64');
+
+  // Resize the buffer to the target size
+  const resizedBuffer = await sharp(buffer)
+      .resize(targetWidth, targetHeight)
+      .toBuffer();
+
+  // Convert the buffer back to base64
+  const resizedImageBase64 = resizedBuffer.toString('base64');
+
+  return resizedImageBase64;
+}

src/utils/segmentImage.mts

@@ -3,6 +3,7 @@ import puppeteer from "puppeteer"
 import { sleep } from "./sleep.mts"
 import { ImageSegment } from "../types.mts"
 import { downloadImageAsBase64 } from "./downloadFileAsBase64.mts"
+import { resizeBase64Image } from "./resizeBase64Image.mts"
 
 // we don't use replicas yet, because it ain't easy to get their hostname
 const instances: string[] = [
@@ -16,7 +17,9 @@ const instances: string[] = [
 // it takes about 30 seconds to compute
 export async function segmentImage(
   inputImageFilePath: string,
-  actionnables: string[]
+  actionnables: string[],
+  width: number,
+  height: number,
 ): Promise<{
   pngInBase64: string
   segments: ImageSegment[]
@@ -73,7 +76,10 @@ export async function segmentImage(
   // const tmpMaskFileName = `${uuidv4()}.png`
   // await downloadFileToTmp(maskUrl, tmpMaskFileName)
 
-  const pngInBase64 = await downloadImageAsBase64(maskUrl)
+  const rawPngInBase64 = await downloadImageAsBase64(maskUrl)
+
+  const pngInBase64 = await resizeBase64Image(rawPngInBase64, width, height)
+
   return {
     pngInBase64,
     segments,