test with whisper cpp (warning: might break the prod server)

Dockerfile

@@ -31,6 +31,7 @@ COPY --chown=user package*.json $HOME/app
 
 RUN npm install
 
+
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user
 COPY --chown=user . $HOME/app
 

package-lock.json

@@ -23,12 +23,14 @@
         "fs-extra": "^11.1.1",
         "gpt-tokens": "^1.1.1",
         "node-fetch": "^3.3.1",
+        "nodejs-whisper": "^0.1.4",
         "openai": "^3.3.0",
         "puppeteer": "^20.8.0",
         "resize-base64": "^1.0.12",
         "sharp": "^0.32.4",
         "temp-dir": "^3.0.0",
         "ts-node": "^10.9.1",
+        "tts-react": "^3.0.1",
         "uuid": "^9.0.0",
         "yaml": "^2.3.1"
       }
@@ -76,6 +78,20 @@
         "node": ">=12"
       }
     },
+    "node_modules/@derhuerst/http-basic": {
+      "version": "8.2.4",
+      "resolved": "https://registry.npmjs.org/@derhuerst/http-basic/-/http-basic-8.2.4.tgz",
+      "integrity": "sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw==",
+      "dependencies": {
+        "caseless": "^0.12.0",
+        "concat-stream": "^2.0.0",
+        "http-response-object": "^3.0.1",
+        "parse-cache-control": "^1.0.1"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
     "node_modules/@gorgonjs/file-provider": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/@gorgonjs/file-provider/-/file-provider-1.4.1.tgz",
@@ -897,6 +913,11 @@
         "node": "*"
       }
     },
+    "node_modules/buffer-from": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
+    },
     "node_modules/bufferutil": {
       "version": "4.0.7",
       "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.7.tgz",
@@ -1153,6 +1174,20 @@
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="
     },
+    "node_modules/concat-stream": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
+      "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
+      "engines": [
+        "node >= 6.0"
+      ],
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.0.2",
+        "typedarray": "^0.0.6"
+      }
+    },
     "node_modules/console-control-strings": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
@@ -1974,6 +2009,65 @@
         "execa": "^0.10.0"
       }
     },
+    "node_modules/ffmpeg-static": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/ffmpeg-static/-/ffmpeg-static-5.2.0.tgz",
+      "integrity": "sha512-WrM7kLW+do9HLr+H6tk7LzQ7kPqbAgLjdzNE32+u3Ff11gXt9Kkkd2nusGFrlWMIe+XaA97t+I8JS7sZIrvRgA==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "@derhuerst/http-basic": "^8.2.0",
+        "env-paths": "^2.2.0",
+        "https-proxy-agent": "^5.0.0",
+        "progress": "^2.0.3"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
+    "node_modules/ffmpeg-static/node_modules/agent-base": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+      "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+      "dependencies": {
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6.0.0"
+      }
+    },
+    "node_modules/ffmpeg-static/node_modules/debug": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "dependencies": {
+        "ms": "2.1.2"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/ffmpeg-static/node_modules/https-proxy-agent": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+      "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+      "dependencies": {
+        "agent-base": "6",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/ffmpeg-static/node_modules/ms": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+    },
     "node_modules/file-uri-to-path": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
@@ -2646,6 +2740,19 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
+    "node_modules/http-response-object": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.2.tgz",
+      "integrity": "sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==",
+      "dependencies": {
+        "@types/node": "^10.0.3"
+      }
+    },
+    "node_modules/http-response-object/node_modules/@types/node": {
+      "version": "10.17.60",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz",
+      "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw=="
+    },
     "node_modules/http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -2797,6 +2904,14 @@
         "kind-of": "^6.0.2"
       }
     },
+    "node_modules/interpret": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
+      "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/iota-array": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz",
@@ -2825,6 +2940,17 @@
       "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
       "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
     },
+    "node_modules/is-core-module": {
+      "version": "2.13.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz",
+      "integrity": "sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==",
+      "dependencies": {
+        "has": "^1.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/is-empty-iterable": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/is-empty-iterable/-/is-empty-iterable-2.1.0.tgz",
@@ -3041,6 +3167,18 @@
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
     },
+    "node_modules/loose-envify": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
+      "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
+      "peer": true,
+      "dependencies": {
+        "js-tokens": "^3.0.0 || ^4.0.0"
+      },
+      "bin": {
+        "loose-envify": "cli.js"
+      }
+    },
     "node_modules/lru-cache": {
       "version": "7.18.3",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
@@ -3636,6 +3774,19 @@
         "node": ">= 8"
       }
     },
+    "node_modules/nodejs-whisper": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/nodejs-whisper/-/nodejs-whisper-0.1.4.tgz",
+      "integrity": "sha512-V3cRV+K4hA4TGTgGxAcz3+zDPfPgD2e82KXojgS3Tgbw4RLnirzE1DdiW9hFrcuR3/6pyr7sXvctYhZUV4asvQ==",
+      "dependencies": {
+        "ffmpeg-static": "^5.2.0",
+        "readline-sync": "^1.4.10",
+        "shelljs": "^0.8.5"
+      },
+      "bin": {
+        "download": "dist/downloadModel.js"
+      }
+    },
     "node_modules/nopt": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz",
@@ -3853,6 +4004,11 @@
         "node": ">=6"
       }
     },
+    "node_modules/parse-cache-control": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz",
+      "integrity": "sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg=="
+    },
     "node_modules/parse-data-uri": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/parse-data-uri/-/parse-data-uri-0.2.0.tgz",
@@ -3907,6 +4063,11 @@
         "node": ">=4"
       }
     },
+    "node_modules/path-parse": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
+    },
     "node_modules/path-scurry": {
       "version": "1.10.1",
       "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz",
@@ -4272,6 +4433,31 @@
         "rc": "cli.js"
       }
     },
+    "node_modules/react": {
+      "version": "18.2.0",
+      "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
+      "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
+      "peer": true,
+      "dependencies": {
+        "loose-envify": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/react-dom": {
+      "version": "18.2.0",
+      "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz",
+      "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==",
+      "peer": true,
+      "dependencies": {
+        "loose-envify": "^1.1.0",
+        "scheduler": "^0.23.0"
+      },
+      "peerDependencies": {
+        "react": "^18.2.0"
+      }
+    },
     "node_modules/readable-stream": {
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -4285,6 +4471,25 @@
         "node": ">= 6"
       }
     },
+    "node_modules/readline-sync": {
+      "version": "1.4.10",
+      "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz",
+      "integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==",
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/rechoir": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
+      "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==",
+      "dependencies": {
+        "resolve": "^1.1.6"
+      },
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/repeat-string": {
       "version": "1.6.1",
       "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
@@ -4370,6 +4575,22 @@
         "create-readme": "^1.1.0"
       }
     },
+    "node_modules/resolve": {
+      "version": "1.22.4",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz",
+      "integrity": "sha512-PXNdCiPqDqeUou+w1C2eTQbNfxKSuMxqTCuvlmmMsk1NWHL5fRrhY6Pl0qEYYc6+QqGClco1Qj8XnjPego4wfg==",
+      "dependencies": {
+        "is-core-module": "^2.13.0",
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
+      },
+      "bin": {
+        "resolve": "bin/resolve"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/resolve-from": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
@@ -4478,6 +4699,15 @@
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
+    "node_modules/scheduler": {
+      "version": "0.23.0",
+      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz",
+      "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==",
+      "peer": true,
+      "dependencies": {
+        "loose-envify": "^1.1.0"
+      }
+    },
     "node_modules/semiver": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/semiver/-/semiver-1.1.0.tgz",
@@ -4622,6 +4852,22 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/shelljs": {
+      "version": "0.8.5",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
+      "dependencies": {
+        "glob": "^7.0.0",
+        "interpret": "^1.0.0",
+        "rechoir": "^0.6.2"
+      },
+      "bin": {
+        "shjs": "bin/shjs"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/side-channel": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
@@ -4908,6 +5154,17 @@
         "node": ">=4"
       }
     },
+    "node_modules/supports-preserve-symlinks-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
+      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/tar": {
       "version": "6.1.15",
       "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.15.tgz",
@@ -5107,6 +5364,19 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.0.tgz",
       "integrity": "sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA=="
     },
+    "node_modules/tts-react": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/tts-react/-/tts-react-3.0.1.tgz",
+      "integrity": "sha512-v/kUYPyLHB9xAUtI+EF/dU1fEqzP6BByg8fLI9+BcZGJyT+tfT4Yv/cTE7bzhPBS9W90a79XujyikMrG1QqR9g==",
+      "engines": {
+        "node": ">=18.16.0",
+        "npm": ">=9.5.1"
+      },
+      "peerDependencies": {
+        "react": ">=17.0.0",
+        "react-dom": ">=17.0.0"
+      }
+    },
     "node_modules/tunnel-agent": {
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
@@ -5146,6 +5416,11 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/typedarray": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+      "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="
+    },
     "node_modules/typedarray-pool": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/typedarray-pool/-/typedarray-pool-1.2.0.tgz",

package.json

@@ -30,12 +30,14 @@
     "fs-extra": "^11.1.1",
     "gpt-tokens": "^1.1.1",
     "node-fetch": "^3.3.1",
+    "nodejs-whisper": "^0.1.4",
     "openai": "^3.3.0",
     "puppeteer": "^20.8.0",
     "resize-base64": "^1.0.12",
     "sharp": "^0.32.4",
     "temp-dir": "^3.0.0",
     "ts-node": "^10.9.1",
+    "tts-react": "^3.0.1",
     "uuid": "^9.0.0",
     "yaml": "^2.3.1"
   }

src/index.mts

@@ -3,7 +3,7 @@ import path from "node:path"
 
 import { validate as uuidValidate } from "uuid"
 import express from "express"
-import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene, ImageAnalysisRequest, ImageAnalysisResponse } from "./types.mts"
+import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene, ImageAnalysisRequest, ImageAnalysisResponse, SoundAnalysisResponse, SoundAnalysisRequest } from "./types.mts"
 
 import { parseVideoRequest } from "./utils/parseVideoRequest.mts"
 import { savePendingVideo } from "./scheduler/savePendingVideo.mts"
@@ -23,6 +23,7 @@ import { getRenderedScene, renderScene } from "./production/renderScene.mts"
 import { parseRenderRequest } from "./utils/parseRenderRequest.mts"
 import { loadRenderedSceneFromCache } from "./utils/loadRenderedSceneFromCache.mts"
 import { analyzeImage } from "./analysis/analyzeImageWithIDEFICSAndNastyHack.mts"
+import { speechToText } from "./speechToText/speechToTextWithWhisperLib.mts"
 
 initFolders()
 // to disable all processing (eg. to debug)
@@ -75,9 +76,55 @@ app.post("/analyze", async (req, res) => {
   try {
     response.result = await analyzeImage(request.image, request.prompt)
   } catch (err) {
-    console.log("failed to render scene!")
+    console.log("failed to analyze the image!")
     console.log(err)
-    response.error = `failed to render scene: ${err}`
+    response.error = `failed to analyze the image: ${err}`
+  }
+
+  if (response.error.length > 0) {
+    // console.log("server error")
+    res.status(500)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  } else {
+    // console.log("all good")
+    res.status(200)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  }
+})
+
+
+// a sound recognition pipeline
+app.post("/listen", async (req, res) => {
+
+  const request = req.body as SoundAnalysisRequest
+
+  if (!request.sound) {
+    console.log("Invalid sound")
+    res.status(400)
+    res.write(JSON.stringify({ result: "", error: "invalid sound" }))
+    res.end()
+    return
+  }
+
+  console.log("/listen called with: ", {
+    sound: request.sound.slice(0, 50)
+  })
+
+  const response: SoundAnalysisResponse = {
+    result: "",
+    error: ""
+  }
+
+  try {
+    response.result = await speechToText(request.sound)
+  } catch (err) {
+    console.log("failed to listen to the sound!")
+    console.log(err)
+    response.error = `failed to listen to the sound: ${err}`
   }
 
   if (response.error.length > 0) {

src/speechToText/speechToTextWithWhisperLib.mts

@@ -0,0 +1,21 @@
+
+import { nodewhisper } from "nodejs-whisper"
+
+import { convertMp3ToWavFilePath } from "../utils/convertMp3ToWavFilePath.mts"
+
+export async function speechToText(sound: string): Promise<string> {
+
+  console.log("/speechToText: calling whisper binding..")
+
+  // TODO try a wav? audio/wav
+  const wavFilePath = await convertMp3ToWavFilePath(sound)
+
+  const result = await nodewhisper(wavFilePath, {
+    modelName: 'base.en', //Downloaded models name
+  })
+
+  console.log(result)
+
+  return "TODO"
+
+}

src/speechToText/speechToTextWithWhisperSpace.mts

@@ -0,0 +1,37 @@
+
+import { client } from "@gradio/client"
+
+import { convertMp3ToWavBase64 } from "../utils/convertMp3ToWavBase64.mts"
+
+const instances: string[] = [
+  `${process.env.VC_SPEECH_TO_TEXT_SPACE_API_URL_1 || ""}`,
+].filter(instance => instance?.length > 0)
+
+export async function speechToText(sound: string): Promise<string> {
+
+  const instance = instances.shift()
+  instances.push(instance)
+
+  const api = await client(instance, {
+    hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+  })
+
+  console.log("/speechToText: calling Space..")
+
+  // TODO try a wav? audio/wav
+  const wav = await convertMp3ToWavBase64(sound)
+
+  // const input = sound
+  // const input = Buffer.from(sound, "base64")
+  // const input = new Blob([sound], { type: 'audio/mpeg' })
+  const input = new Blob([wav], { type: 'audio/wav' })
+
+  const result = await api.predict("/transcribe", [
+    input,
+  ])
+
+  console.log(result)
+
+  return "TODO"
+
+}

src/types.mts

@@ -320,6 +320,16 @@ export interface ImageAnalysisResponse {
   error?: string
 }
 
+export interface SoundAnalysisRequest {
+  sound: string // in base64
+  prompt: string
+}
+
+export interface SoundAnalysisResponse {
+  result: string
+  error?: string
+}
+
 
 export interface ImageSegmentationRequest {
   image: string // in base64

src/utils/computeSecretFingerprint.mts

@@ -0,0 +1,7 @@
+import { computeSha256 } from "./computeSha256.mts"
+
+const secretFingerprint = `${process.env.VC_SECRET_FINGERPRINT || ""}`
+
+export function computeSecretFingerprint(input: string) {
+  return computeSha256(`${secretFingerprint}_${input}`)
+}

src/utils/convertMp3ToWavBase64.mts

@@ -0,0 +1,42 @@
+
+import path from "node:path"
+import * as fs from "node:fs"
+
+import { v4 as uuidv4 } from "uuid"
+import tmpDir from "temp-dir"
+import ffmpeg from "fluent-ffmpeg"
+
+export function convertMp3ToWavBase64(base64: string) : Promise<string> {
+  return new Promise((resolve, reject) => {
+
+    const inputPath = path.join(tmpDir, `${uuidv4()}.mp3`)
+    const outputPath = path.join(tmpDir, `${uuidv4()}.wav`)
+
+    const base64Header = 'data:audio/mpeg;base64,';
+
+    fs.writeFile(inputPath, base64.replace(base64Header, ''), { encoding: 'base64' }, (writeError) => {
+      if (writeError) {
+        reject(writeError);
+        return;
+      }
+
+      ffmpeg(inputPath)
+        .output(outputPath)
+        .on('end', () => {
+          fs.readFile(outputPath, { encoding: 'base64' }, (readError, data) => {
+            if (readError) {
+              reject(readError);
+              return;
+            }
+            resolve(`data:audio/wav;base64,${data}`);
+            fs.promises.unlink(inputPath);
+            fs.promises.unlink(outputPath);
+          });
+        })
+        .on('error', (conversionError) => {
+          reject(conversionError);
+        })
+        .run();
+    });
+  });
+}

src/utils/convertMp3ToWavFilePath.mts

@@ -0,0 +1,38 @@
+
+import path from "node:path"
+import * as fs from "node:fs"
+
+import { v4 as uuidv4 } from "uuid"
+import tmpDir from "temp-dir"
+import ffmpeg from "fluent-ffmpeg"
+
+export function convertMp3ToWavFilePath(base64: string) : Promise<string> {
+  return new Promise((resolve, reject) => {
+
+    // this one will be deleted at the end
+    const tmpInputPath = path.join(tmpDir, `${uuidv4()}.mp3`)
+
+    const tmpOutputPath = path.join(tmpDir, `${uuidv4()}.wav`)
+
+    const base64Header = 'data:audio/mpeg;base64,';
+
+    fs.writeFile(tmpInputPath, base64.replace(base64Header, ''), { encoding: 'base64' }, (writeError) => {
+      if (writeError) {
+        reject(writeError);
+        return;
+      }
+
+      ffmpeg(tmpInputPath)
+        .output(tmpOutputPath)
+        .on('end', () => {
+          fs.promises.unlink(tmpInputPath)
+          resolve(tmpOutputPath)
+        
+        })
+        .on('error', (conversionError) => {
+          reject(conversionError);
+        })
+        .run();
+    });
+  });
+}