Spaces:
Sleeping
Sleeping
Julian Bilcke
commited on
Commit
·
7b5fc2b
1
Parent(s):
c1e4aec
added images
Browse files- .env +3 -0
- package-lock.json +13 -0
- package.json +1 -0
- src/app/interface/generate/index.tsx +196 -72
- src/app/server/actions/generateImage.ts +106 -0
- src/lib/getValidNumber.ts +10 -0
- src/lib/useAudio.ts +88 -16
.env
CHANGED
|
@@ -2,6 +2,9 @@
|
|
| 2 |
AI_BEDTIME_STORY_API_GRADIO_URL="https://jbilcke-hf-ai-bedtime-story-server.hf.space"
|
| 3 |
AI_BEDTIME_STORY_API_SECRET_TOKEN=
|
| 4 |
|
|
|
|
|
|
|
|
|
|
| 5 |
# ----------- CENSORSHIP -------
|
| 6 |
ENABLE_CENSORSHIP=
|
| 7 |
FINGERPRINT_KEY=
|
|
|
|
| 2 |
AI_BEDTIME_STORY_API_GRADIO_URL="https://jbilcke-hf-ai-bedtime-story-server.hf.space"
|
| 3 |
AI_BEDTIME_STORY_API_SECRET_TOKEN=
|
| 4 |
|
| 5 |
+
FAST_IMAGE_SERVER_API_GRADIO_URL="https://jbilcke-hf-fast-image-server.hf.space"
|
| 6 |
+
FAST_IMAGE_SERVER_API_SECRET_TOKEN=
|
| 7 |
+
|
| 8 |
# ----------- CENSORSHIP -------
|
| 9 |
ENABLE_CENSORSHIP=
|
| 10 |
FINGERPRINT_KEY=
|
package-lock.json
CHANGED
|
@@ -46,6 +46,7 @@
|
|
| 46 |
"react": "18.2.0",
|
| 47 |
"react-circular-progressbar": "^2.1.0",
|
| 48 |
"react-dom": "18.2.0",
|
|
|
|
| 49 |
"react-snowfall": "^1.2.1",
|
| 50 |
"react-virtualized-auto-sizer": "^1.0.20",
|
| 51 |
"replicate": "^0.17.0",
|
|
@@ -5531,6 +5532,18 @@
|
|
| 5531 |
}
|
| 5532 |
}
|
| 5533 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5534 |
"node_modules/react-snowfall": {
|
| 5535 |
"version": "1.2.1",
|
| 5536 |
"resolved": "https://registry.npmjs.org/react-snowfall/-/react-snowfall-1.2.1.tgz",
|
|
|
|
| 46 |
"react": "18.2.0",
|
| 47 |
"react-circular-progressbar": "^2.1.0",
|
| 48 |
"react-dom": "18.2.0",
|
| 49 |
+
"react-smooth-scroll-hook": "^1.3.4",
|
| 50 |
"react-snowfall": "^1.2.1",
|
| 51 |
"react-virtualized-auto-sizer": "^1.0.20",
|
| 52 |
"replicate": "^0.17.0",
|
|
|
|
| 5532 |
}
|
| 5533 |
}
|
| 5534 |
},
|
| 5535 |
+
"node_modules/react-smooth-scroll-hook": {
|
| 5536 |
+
"version": "1.3.4",
|
| 5537 |
+
"resolved": "https://registry.npmjs.org/react-smooth-scroll-hook/-/react-smooth-scroll-hook-1.3.4.tgz",
|
| 5538 |
+
"integrity": "sha512-nPNSQStr8gz1ogQbWTmeXiOEValKKr7hImucipoUlP7mK1n54qOBoFpO1aGE9yZEB7vSnkRx3mTH9zwO4nj8MQ==",
|
| 5539 |
+
"engines": {
|
| 5540 |
+
"node": ">=10"
|
| 5541 |
+
},
|
| 5542 |
+
"peerDependencies": {
|
| 5543 |
+
"react": ">=16.8.0",
|
| 5544 |
+
"react-dom": ">=16.8.0"
|
| 5545 |
+
}
|
| 5546 |
+
},
|
| 5547 |
"node_modules/react-snowfall": {
|
| 5548 |
"version": "1.2.1",
|
| 5549 |
"resolved": "https://registry.npmjs.org/react-snowfall/-/react-snowfall-1.2.1.tgz",
|
package.json
CHANGED
|
@@ -47,6 +47,7 @@
|
|
| 47 |
"react": "18.2.0",
|
| 48 |
"react-circular-progressbar": "^2.1.0",
|
| 49 |
"react-dom": "18.2.0",
|
|
|
|
| 50 |
"react-snowfall": "^1.2.1",
|
| 51 |
"react-virtualized-auto-sizer": "^1.0.20",
|
| 52 |
"replicate": "^0.17.0",
|
|
|
|
| 47 |
"react": "18.2.0",
|
| 48 |
"react-circular-progressbar": "^2.1.0",
|
| 49 |
"react-dom": "18.2.0",
|
| 50 |
+
"react-smooth-scroll-hook": "^1.3.4",
|
| 51 |
"react-snowfall": "^1.2.1",
|
| 52 |
"react-virtualized-auto-sizer": "^1.0.20",
|
| 53 |
"replicate": "^0.17.0",
|
src/app/interface/generate/index.tsx
CHANGED
|
@@ -3,6 +3,7 @@
|
|
| 3 |
import { useEffect, useRef, useState, useTransition } from "react"
|
| 4 |
import { useSpring, animated } from "@react-spring/web"
|
| 5 |
import { usePathname, useRouter, useSearchParams } from "next/navigation"
|
|
|
|
| 6 |
import { split } from "sentence-splitter"
|
| 7 |
|
| 8 |
import { useToast } from "@/components/ui/use-toast"
|
|
@@ -16,6 +17,7 @@ import { useCountdown } from "@/lib/useCountdown"
|
|
| 16 |
import { useAudio } from "@/lib/useAudio"
|
| 17 |
|
| 18 |
import { Countdown } from "../countdown"
|
|
|
|
| 19 |
|
| 20 |
type Stage = "generate" | "finished"
|
| 21 |
|
|
@@ -32,6 +34,7 @@ export function Generate() {
|
|
| 32 |
const [promptDraft, setPromptDraft] = useState("")
|
| 33 |
const [assetUrl, setAssetUrl] = useState("")
|
| 34 |
const [isOverSubmitButton, setOverSubmitButton] = useState(false)
|
|
|
|
| 35 |
|
| 36 |
const [runs, setRuns] = useState(0)
|
| 37 |
const runsRef = useRef(0)
|
|
@@ -39,13 +42,30 @@ export function Generate() {
|
|
| 39 |
const currentLineIndexRef = useRef(0)
|
| 40 |
const [currentLineIndex, setCurrentLineIndex] = useState(0)
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
useEffect(() => {
|
| 43 |
currentLineIndexRef.current = currentLineIndex
|
| 44 |
}, [currentLineIndex])
|
| 45 |
-
|
| 46 |
const [storyLines, setStoryLines] = useState<StoryLine[]>([])
|
| 47 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
// computing those is cheap
|
|
|
|
| 49 |
const wholeStory = storyLines.map(line => line.text).join("\n")
|
| 50 |
const currentLine = storyLines.at(currentLineIndex)
|
| 51 |
const currentLineText = currentLine?.text || ""
|
|
@@ -60,30 +80,12 @@ export function Generate() {
|
|
| 60 |
|
| 61 |
const { toast } = useToast()
|
| 62 |
|
| 63 |
-
const
|
| 64 |
-
|
| 65 |
-
/*
|
| 66 |
-
// to simulate a "typing" effect
|
| 67 |
-
however.. we don't need this as we already have an audio player!
|
| 68 |
|
| 69 |
-
const [typedStoryText, setTypedStoryText] = useState("")
|
| 70 |
-
const [typedStoryCharacterIndex, setTypedStoryCharacterIndex] = useState(0)
|
| 71 |
-
|
| 72 |
-
useEffect(() => {
|
| 73 |
-
if (storyText && typedStoryCharacterIndex < storyText.length) {
|
| 74 |
-
setTimeout(() => {
|
| 75 |
-
setTypedStoryText(typedStoryText + story.text[typedStoryCharacterIndex])
|
| 76 |
-
setTypedStoryCharacterIndex(typedStoryCharacterIndex + 1)
|
| 77 |
-
console.log("boom")
|
| 78 |
-
}, 40)
|
| 79 |
-
}
|
| 80 |
-
}, [storyText, typedStoryCharacterIndex])
|
| 81 |
-
*/
|
| 82 |
-
|
| 83 |
const { progressPercent, remainingTimeInSec } = useCountdown({
|
| 84 |
isActive: isLocked,
|
| 85 |
timerId: runs, // everytime we change this, the timer will reset
|
| 86 |
-
durationInSec: /*stage === "interpolate" ? 30 :*/
|
| 87 |
onEnd: () => {}
|
| 88 |
})
|
| 89 |
|
|
@@ -107,6 +109,20 @@ export function Generate() {
|
|
| 107 |
},
|
| 108 |
})
|
| 109 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
const handleSubmit = () => {
|
| 111 |
if (isLocked) { return }
|
| 112 |
if (!promptDraft) { return }
|
|
@@ -187,11 +203,47 @@ export function Generate() {
|
|
| 187 |
console.log("story audio changed!")
|
| 188 |
|
| 189 |
try {
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
} catch (err) {
|
| 196 |
console.error(err)
|
| 197 |
}
|
|
@@ -199,9 +251,9 @@ export function Generate() {
|
|
| 199 |
fn()
|
| 200 |
|
| 201 |
return () => {
|
| 202 |
-
|
| 203 |
}
|
| 204 |
-
}, [currentLineAudio])
|
| 205 |
|
| 206 |
return (
|
| 207 |
<div
|
|
@@ -254,7 +306,7 @@ export function Generate() {
|
|
| 254 |
|
| 255 |
<div className={cn(
|
| 256 |
`flex flex-col md:flex-row`,
|
| 257 |
-
`space-y-
|
| 258 |
` w-full md:max-w-[1024px]`,
|
| 259 |
`items-center justify-between`
|
| 260 |
)}>
|
|
@@ -270,13 +322,16 @@ export function Generate() {
|
|
| 270 |
`input input-bordered rounded-full`,
|
| 271 |
`transition-all duration-300 ease-in-out`,
|
| 272 |
`backdrop-blur-md `,
|
| 273 |
-
`placeholder:text-gray-400`,
|
| 274 |
-
`disabled:bg-
|
| 275 |
isLocked
|
| 276 |
-
? `bg-
|
| 277 |
: `bg-white/10 text-yellow-400/100 selection:bg-yellow-200/100 selection:text-yellow-200/100`,
|
| 278 |
`text-left`,
|
| 279 |
-
|
|
|
|
|
|
|
|
|
|
| 280 |
)}
|
| 281 |
value={promptDraft}
|
| 282 |
onChange={e => setPromptDraft(e.target.value)}
|
|
@@ -306,39 +361,82 @@ export function Generate() {
|
|
| 306 |
<span>{nbCharsLimits}</span>
|
| 307 |
</div>
|
| 308 |
</div>
|
| 309 |
-
<div className="flex flex-row w-
|
| 310 |
-
<
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 342 |
</div>
|
| 343 |
</div>
|
| 344 |
|
|
@@ -350,6 +448,7 @@ export function Generate() {
|
|
| 350 |
`space-y-8`,
|
| 351 |
// `transition-all duration-300 ease-in-out`,
|
| 352 |
)}>
|
|
|
|
| 353 |
|
| 354 |
<div
|
| 355 |
className={cn(
|
|
@@ -379,6 +478,7 @@ export function Generate() {
|
|
| 379 |
/>}
|
| 380 |
</div> : null}
|
| 381 |
|
|
|
|
| 382 |
<div className={cn(
|
| 383 |
`flex flex-col md:flex-row`,
|
| 384 |
`space-y-3 md:space-y-0 md:space-x-3`,
|
|
@@ -386,17 +486,41 @@ export function Generate() {
|
|
| 386 |
`items-center justify-between`
|
| 387 |
)}>
|
| 388 |
<div className={cn(
|
| 389 |
-
`flex flex-col flex-grow w-full space-y-2 text-2xl text-blue-200/
|
| 390 |
)}>
|
| 391 |
{storyLines.map((line, i) =>
|
| 392 |
<div
|
| 393 |
-
|
|
|
|
|
|
|
| 394 |
|
| 395 |
// TODO change a color if we have progressed at the current index (i)
|
| 396 |
-
className={cn(
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 400 |
</div>
|
| 401 |
</div>
|
| 402 |
</div>
|
|
|
|
| 3 |
import { useEffect, useRef, useState, useTransition } from "react"
|
| 4 |
import { useSpring, animated } from "@react-spring/web"
|
| 5 |
import { usePathname, useRouter, useSearchParams } from "next/navigation"
|
| 6 |
+
import useSmoothScroll from "react-smooth-scroll-hook"
|
| 7 |
import { split } from "sentence-splitter"
|
| 8 |
|
| 9 |
import { useToast } from "@/components/ui/use-toast"
|
|
|
|
| 17 |
import { useAudio } from "@/lib/useAudio"
|
| 18 |
|
| 19 |
import { Countdown } from "../countdown"
|
| 20 |
+
import { generateImage } from "@/app/server/actions/generateImage"
|
| 21 |
|
| 22 |
type Stage = "generate" | "finished"
|
| 23 |
|
|
|
|
| 34 |
const [promptDraft, setPromptDraft] = useState("")
|
| 35 |
const [assetUrl, setAssetUrl] = useState("")
|
| 36 |
const [isOverSubmitButton, setOverSubmitButton] = useState(false)
|
| 37 |
+
const [isOverPauseButton, setOverPauseButton] = useState(false)
|
| 38 |
|
| 39 |
const [runs, setRuns] = useState(0)
|
| 40 |
const runsRef = useRef(0)
|
|
|
|
| 42 |
const currentLineIndexRef = useRef(0)
|
| 43 |
const [currentLineIndex, setCurrentLineIndex] = useState(0)
|
| 44 |
|
| 45 |
+
const voices: TTSVoice[] = ["Cloée", "Julian"]
|
| 46 |
+
const [voice, setVoice] = useState<TTSVoice>("Cloée")
|
| 47 |
+
|
| 48 |
+
const { scrollTo } = useSmoothScroll({
|
| 49 |
+
ref: scrollRef,
|
| 50 |
+
speed: 2000,
|
| 51 |
+
direction: 'y',
|
| 52 |
+
});
|
| 53 |
+
|
| 54 |
useEffect(() => {
|
| 55 |
currentLineIndexRef.current = currentLineIndex
|
| 56 |
}, [currentLineIndex])
|
| 57 |
+
|
| 58 |
const [storyLines, setStoryLines] = useState<StoryLine[]>([])
|
| 59 |
|
| 60 |
+
const [images, setImages] = useState<string[]>([])
|
| 61 |
+
const imagesRef = useRef<string[]>([])
|
| 62 |
+
const imageListKey = images.join("")
|
| 63 |
+
useEffect(() => {
|
| 64 |
+
imagesRef.current = images
|
| 65 |
+
}, [imageListKey])
|
| 66 |
+
|
| 67 |
// computing those is cheap
|
| 68 |
+
|
| 69 |
const wholeStory = storyLines.map(line => line.text).join("\n")
|
| 70 |
const currentLine = storyLines.at(currentLineIndex)
|
| 71 |
const currentLineText = currentLine?.text || ""
|
|
|
|
| 80 |
|
| 81 |
const { toast } = useToast()
|
| 82 |
|
| 83 |
+
const { playback, isPlaying, isSwitchingTracks, isLoaded, progress, togglePause } = useAudio()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
const { progressPercent, remainingTimeInSec } = useCountdown({
|
| 86 |
isActive: isLocked,
|
| 87 |
timerId: runs, // everytime we change this, the timer will reset
|
| 88 |
+
durationInSec: /*stage === "interpolate" ? 30 :*/ 50, // it usually takes 40 seconds, but there might be lag
|
| 89 |
onEnd: () => {}
|
| 90 |
})
|
| 91 |
|
|
|
|
| 109 |
},
|
| 110 |
})
|
| 111 |
|
| 112 |
+
const pauseButtonBouncer = useSpring({
|
| 113 |
+
transform: isOverPauseButton
|
| 114 |
+
? 'scale(1.05)'
|
| 115 |
+
: 'scale(1.0)',
|
| 116 |
+
boxShadow: isOverPauseButton
|
| 117 |
+
? `0px 5px 15px 0px rgba(0, 0, 0, 0.05)`
|
| 118 |
+
: `0px 0px 0px 0px rgba(0, 0, 0, 0.05)`,
|
| 119 |
+
loop: true,
|
| 120 |
+
config: {
|
| 121 |
+
tension: 300,
|
| 122 |
+
friction: 10,
|
| 123 |
+
},
|
| 124 |
+
})
|
| 125 |
+
|
| 126 |
const handleSubmit = () => {
|
| 127 |
if (isLocked) { return }
|
| 128 |
if (!promptDraft) { return }
|
|
|
|
| 203 |
console.log("story audio changed!")
|
| 204 |
|
| 205 |
try {
|
| 206 |
+
const isLastLine =
|
| 207 |
+
(storyLines.length === 0) ||
|
| 208 |
+
(currentLineIndexRef.current === (storyLines.length - 1))
|
| 209 |
+
|
| 210 |
+
scrollTo(`#story-line-${currentLineIndexRef.current}`)
|
| 211 |
+
|
| 212 |
+
const nextLineIndex = (currentLineIndexRef.current += 1)
|
| 213 |
+
const nextLineText = storyLines[nextLineIndex]?.text || ""
|
| 214 |
+
|
| 215 |
+
if (nextLineText) {
|
| 216 |
+
setTimeout(() => {
|
| 217 |
+
startTransition(async () => {
|
| 218 |
+
try {
|
| 219 |
+
const newImage = await generateImage({
|
| 220 |
+
positivePrompt: [
|
| 221 |
+
"bedtime story illustration",
|
| 222 |
+
"painting illustration",
|
| 223 |
+
promptDraft,
|
| 224 |
+
nextLineText,
|
| 225 |
+
].join(", "),
|
| 226 |
+
width: 1024,
|
| 227 |
+
height: 800
|
| 228 |
+
})
|
| 229 |
+
// console.log("newImage:", newImage.slice(0, 50))
|
| 230 |
+
setImages(imagesRef.current.concat(newImage))
|
| 231 |
+
} catch (err) {
|
| 232 |
+
setImages(imagesRef.current.concat(""))
|
| 233 |
+
}
|
| 234 |
+
})
|
| 235 |
+
}, 100)
|
| 236 |
+
} else {
|
| 237 |
+
setImages(imagesRef.current.concat(""))
|
| 238 |
+
}
|
| 239 |
+
|
| 240 |
+
await playback(currentLineAudio, isLastLine) // play
|
| 241 |
+
|
| 242 |
+
if (!isLastLine && nextLineText) {
|
| 243 |
+
setTimeout(() => {
|
| 244 |
+
setCurrentLineIndex(nextLineIndex)
|
| 245 |
+
}, 1000)
|
| 246 |
+
}
|
| 247 |
} catch (err) {
|
| 248 |
console.error(err)
|
| 249 |
}
|
|
|
|
| 251 |
fn()
|
| 252 |
|
| 253 |
return () => {
|
| 254 |
+
playback() // stop
|
| 255 |
}
|
| 256 |
+
}, [currentLineText, currentLineAudio])
|
| 257 |
|
| 258 |
return (
|
| 259 |
<div
|
|
|
|
| 306 |
|
| 307 |
<div className={cn(
|
| 308 |
`flex flex-col md:flex-row`,
|
| 309 |
+
`space-y-4 md:space-y-0 md:space-x-4`,
|
| 310 |
` w-full md:max-w-[1024px]`,
|
| 311 |
`items-center justify-between`
|
| 312 |
)}>
|
|
|
|
| 322 |
`input input-bordered rounded-full`,
|
| 323 |
`transition-all duration-300 ease-in-out`,
|
| 324 |
`backdrop-blur-md `,
|
| 325 |
+
`placeholder:text-gray-400/90`,
|
| 326 |
+
`disabled:bg-blue-900/70 disabled:text-blue-300/60 disabled:border-transparent`,
|
| 327 |
isLocked
|
| 328 |
+
? `bg-blue-100/80 text-yellow-400/60 selection:bg-yellow-200/60 selection:text-yellow-200/60 border-transparent`
|
| 329 |
: `bg-white/10 text-yellow-400/100 selection:bg-yellow-200/100 selection:text-yellow-200/100`,
|
| 330 |
`text-left`,
|
| 331 |
+
``,
|
| 332 |
+
storyLines?.length
|
| 333 |
+
? `text-2xl leading-10 px-6 h-16 pt-1`
|
| 334 |
+
: `text-3xl leading-14 px-8 h-[70px] pt-1`
|
| 335 |
)}
|
| 336 |
value={promptDraft}
|
| 337 |
onChange={e => setPromptDraft(e.target.value)}
|
|
|
|
| 361 |
<span>{nbCharsLimits}</span>
|
| 362 |
</div>
|
| 363 |
</div>
|
| 364 |
+
<div className="flex flex-row w-full md:w-auto justify-center">
|
| 365 |
+
<div className="flex flex-row w-1/2 md:w-52">
|
| 366 |
+
<animated.button
|
| 367 |
+
style={{
|
| 368 |
+
textShadow: "0px 0px 1px #000000ab",
|
| 369 |
+
...submitButtonBouncer
|
| 370 |
+
}}
|
| 371 |
+
onMouseEnter={() => setOverSubmitButton(true)}
|
| 372 |
+
onMouseLeave={() => setOverSubmitButton(false)}
|
| 373 |
+
className={cn(
|
| 374 |
+
storyLines?.length
|
| 375 |
+
? `text-2xl leading-10 px-4 h-16`
|
| 376 |
+
: `text-3xl leading-14 px-6 h-[70px]`,
|
| 377 |
+
`rounded-full`,
|
| 378 |
+
`transition-all duration-300 ease-in-out`,
|
| 379 |
+
`backdrop-blur-sm`,
|
| 380 |
+
isLocked
|
| 381 |
+
? `bg-blue-900/70 text-sky-50/80 border-yellow-600/10`
|
| 382 |
+
: `bg-yellow-400/70 text-sky-50 border-yellow-800/20 hover:bg-yellow-400/80`,
|
| 383 |
+
`text-center`,
|
| 384 |
+
`w-full`,
|
| 385 |
+
`border`,
|
| 386 |
+
headingFont.className,
|
| 387 |
+
// `transition-all duration-300`,
|
| 388 |
+
// `hover:animate-bounce`
|
| 389 |
+
)}
|
| 390 |
+
disabled={isLocked}
|
| 391 |
+
onClick={handleSubmit}
|
| 392 |
+
>
|
| 393 |
+
{isLocked
|
| 394 |
+
? `Dreaming..`
|
| 395 |
+
: "Dream 🌙"
|
| 396 |
+
}
|
| 397 |
+
</animated.button>
|
| 398 |
+
</div>
|
| 399 |
+
{
|
| 400 |
+
/*
|
| 401 |
+
!!storyLines.length && <div className={cn(
|
| 402 |
+
`flex flex-row w-1/2 md:w-44`,
|
| 403 |
+
`transition-all duration-300 ease-in-out`,
|
| 404 |
+
isLoaded ? 'scale-100' : 'scale-0'
|
| 405 |
+
)}>
|
| 406 |
+
<animated.button
|
| 407 |
+
style={{
|
| 408 |
+
textShadow: "0px 0px 1px #000000ab",
|
| 409 |
+
...pauseButtonBouncer
|
| 410 |
+
}}
|
| 411 |
+
onMouseEnter={() => setOverPauseButton(true)}
|
| 412 |
+
onMouseLeave={() => setOverPauseButton(false)}
|
| 413 |
+
className={cn(
|
| 414 |
+
`px-4 h-16`,
|
| 415 |
+
`rounded-full`,
|
| 416 |
+
`transition-all duration-300 ease-in-out`,
|
| 417 |
+
`backdrop-blur-sm`,
|
| 418 |
+
isLocked
|
| 419 |
+
? `bg-orange-200/30 text-sky-50/60 border-yellow-600/10`
|
| 420 |
+
: `bg-yellow-400/50 text-sky-50 border-yellow-800/20 hover:bg-yellow-400/60`,
|
| 421 |
+
`text-center`,
|
| 422 |
+
`w-full`,
|
| 423 |
+
`text-2xl `,
|
| 424 |
+
`border`,
|
| 425 |
+
headingFont.className,
|
| 426 |
+
// `transition-all duration-300`,
|
| 427 |
+
// `hover:animate-bounce`
|
| 428 |
+
)}
|
| 429 |
+
disabled={isLocked}
|
| 430 |
+
onClick={togglePause}
|
| 431 |
+
>
|
| 432 |
+
{isPlaying || isSwitchingTracks
|
| 433 |
+
? "Pause 🔊"
|
| 434 |
+
: "Play 🔊"
|
| 435 |
+
}
|
| 436 |
+
</animated.button>
|
| 437 |
+
</div>
|
| 438 |
+
*/
|
| 439 |
+
}</div>
|
| 440 |
</div>
|
| 441 |
</div>
|
| 442 |
|
|
|
|
| 448 |
`space-y-8`,
|
| 449 |
// `transition-all duration-300 ease-in-out`,
|
| 450 |
)}>
|
| 451 |
+
|
| 452 |
|
| 453 |
<div
|
| 454 |
className={cn(
|
|
|
|
| 478 |
/>}
|
| 479 |
</div> : null}
|
| 480 |
|
| 481 |
+
|
| 482 |
<div className={cn(
|
| 483 |
`flex flex-col md:flex-row`,
|
| 484 |
`space-y-3 md:space-y-0 md:space-x-3`,
|
|
|
|
| 486 |
`items-center justify-between`
|
| 487 |
)}>
|
| 488 |
<div className={cn(
|
| 489 |
+
`flex flex-col flex-grow w-full items-center space-y-2 text-2xl text-blue-200/60`
|
| 490 |
)}>
|
| 491 |
{storyLines.map((line, i) =>
|
| 492 |
<div
|
| 493 |
+
id={`story-line-${i}`}
|
| 494 |
+
key={`${line.text}_${i}`}>
|
| 495 |
+
<div
|
| 496 |
|
| 497 |
// TODO change a color if we have progressed at the current index (i)
|
| 498 |
+
className={cn(
|
| 499 |
+
"flex flex-col items-center w-full "
|
| 500 |
+
//i < currentLineIndex
|
| 501 |
+
//? 'text-yellow-200'
|
| 502 |
+
//: 'text-blue-200/80'
|
| 503 |
+
)}
|
| 504 |
+
style={{}}
|
| 505 |
+
>
|
| 506 |
+
<div className="w-full md:w-2/3 text-center"> {
|
| 507 |
+
line.text.split("").map((c, j, arr) => <span
|
| 508 |
+
key={`${c}_${j}`}
|
| 509 |
+
className={cn(
|
| 510 |
+
`transition-all duration-100 ease-in-out`,
|
| 511 |
+
i < currentLineIndex || (isLoaded && i === currentLineIndex && j <= (progress * 1.3 * arr.length))
|
| 512 |
+
? 'text-yellow-400/90'
|
| 513 |
+
: ''
|
| 514 |
+
)}>{c || " "}</span>)
|
| 515 |
+
}</div>
|
| 516 |
+
<div className="flex flex-col items-center justify-center w-full p-8">
|
| 517 |
+
{images.at(i) ? <img
|
| 518 |
+
className="h-[400px] rounded-lg overflow-hidden"
|
| 519 |
+
src={images.at(i)}
|
| 520 |
+
/> : null}
|
| 521 |
+
</div>
|
| 522 |
+
</div>
|
| 523 |
+
</div>)}
|
| 524 |
</div>
|
| 525 |
</div>
|
| 526 |
</div>
|
src/app/server/actions/generateImage.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"use server"
|
| 2 |
+
|
| 3 |
+
// TODO add a system to mark failed instances as "unavailable" for a couple of minutes
|
| 4 |
+
// console.log("process.env:", process.env)
|
| 5 |
+
|
| 6 |
+
import { generateSeed } from "@/lib/generateSeed";
|
| 7 |
+
import { getValidNumber } from "@/lib/getValidNumber";
|
| 8 |
+
|
| 9 |
+
// note: to reduce costs I use the small A10s (not the large)
|
| 10 |
+
// anyway, we will soon not need to use this cloud anymore
|
| 11 |
+
// since we will be able to leverage the Inference API
|
| 12 |
+
const instance = `${process.env.FAST_IMAGE_SERVER_API_GRADIO_URL || ""}`
|
| 13 |
+
const secretToken = `${process.env.FAST_IMAGE_SERVER_API_SECRET_TOKEN || ""}`
|
| 14 |
+
|
| 15 |
+
// console.log("DEBUG:", JSON.stringify({ instances, secretToken }, null, 2))
|
| 16 |
+
|
| 17 |
+
export async function generateImage(options: {
|
| 18 |
+
positivePrompt: string;
|
| 19 |
+
negativePrompt?: string;
|
| 20 |
+
seed?: number;
|
| 21 |
+
width?: number;
|
| 22 |
+
height?: number;
|
| 23 |
+
nbSteps?: number;
|
| 24 |
+
}): Promise<string> {
|
| 25 |
+
|
| 26 |
+
// console.log("querying " + instance)
|
| 27 |
+
const positivePrompt = options?.positivePrompt || ""
|
| 28 |
+
if (!positivePrompt) {
|
| 29 |
+
throw new Error("missing prompt")
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
// the negative prompt CAN be missing, since we use a trick
|
| 33 |
+
// where we make the interface mandatory in the TS doc,
|
| 34 |
+
// but browsers might send something partial
|
| 35 |
+
const negativePrompt = options?.negativePrompt || ""
|
| 36 |
+
|
| 37 |
+
// we treat 0 as meaning "random seed"
|
| 38 |
+
const seed = (options?.seed ? options.seed : 0) || generateSeed()
|
| 39 |
+
|
| 40 |
+
const width = getValidNumber(options?.width, 256, 1024, 512)
|
| 41 |
+
const height = getValidNumber(options?.height, 256, 1024, 512)
|
| 42 |
+
const nbSteps = getValidNumber(options?.nbSteps, 1, 8, 4)
|
| 43 |
+
// console.log("SEED:", seed)
|
| 44 |
+
|
| 45 |
+
const positive = [
|
| 46 |
+
|
| 47 |
+
// oh well.. is it too late to move this to the bottom?
|
| 48 |
+
"beautiful",
|
| 49 |
+
|
| 50 |
+
// too opinionated, so let's remove it
|
| 51 |
+
// "intricate details",
|
| 52 |
+
|
| 53 |
+
positivePrompt,
|
| 54 |
+
|
| 55 |
+
"award winning",
|
| 56 |
+
"high resolution"
|
| 57 |
+
].filter(word => word)
|
| 58 |
+
.join(", ")
|
| 59 |
+
|
| 60 |
+
const negative = [
|
| 61 |
+
negativePrompt,
|
| 62 |
+
"watermark",
|
| 63 |
+
"copyright",
|
| 64 |
+
"blurry",
|
| 65 |
+
// "artificial",
|
| 66 |
+
// "cropped",
|
| 67 |
+
"low quality",
|
| 68 |
+
"ugly"
|
| 69 |
+
].filter(word => word)
|
| 70 |
+
.join(", ")
|
| 71 |
+
|
| 72 |
+
const res = await fetch(instance + (instance.endsWith("/") ? "" : "/") + "api/predict", {
|
| 73 |
+
method: "POST",
|
| 74 |
+
headers: {
|
| 75 |
+
"Content-Type": "application/json",
|
| 76 |
+
// Authorization: `Bearer ${token}`,
|
| 77 |
+
},
|
| 78 |
+
body: JSON.stringify({
|
| 79 |
+
fn_index: 0, // <- important!
|
| 80 |
+
data: [
|
| 81 |
+
positive, // string in 'Prompt' Textbox component
|
| 82 |
+
negative, // string in 'Negative prompt' Textbox component
|
| 83 |
+
seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
|
| 84 |
+
width, // number (numeric value between 256 and 1024) in 'Width' Slider component
|
| 85 |
+
height, // number (numeric value between 256 and 1024) in 'Height' Slider component
|
| 86 |
+
0.0, // can be disabled for LCM SDXL
|
| 87 |
+
nbSteps, // number (numeric value between 2 and 8) in 'Number of inference steps for base' Slider component
|
| 88 |
+
secretToken
|
| 89 |
+
]
|
| 90 |
+
}),
|
| 91 |
+
cache: "no-store",
|
| 92 |
+
})
|
| 93 |
+
|
| 94 |
+
const { data } = await res.json()
|
| 95 |
+
|
| 96 |
+
if (res.status !== 200 || !Array.isArray(data)) {
|
| 97 |
+
// This will activate the closest `error.js` Error Boundary
|
| 98 |
+
throw new Error(`Failed to fetch data (status: ${res.status})`)
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
if (!data[0]) {
|
| 102 |
+
throw new Error(`the returned image was empty`)
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
return data[0] as string
|
| 106 |
+
}
|
src/lib/getValidNumber.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
export const getValidNumber = (something: any, minValue: number, maxValue: number, defaultValue: number) => {
|
| 2 |
+
const strValue = `${something || defaultValue}`
|
| 3 |
+
const numValue = Number(strValue)
|
| 4 |
+
const isValid = !isNaN(numValue) && isFinite(numValue)
|
| 5 |
+
if (!isValid) {
|
| 6 |
+
return defaultValue
|
| 7 |
+
}
|
| 8 |
+
return Math.max(minValue, Math.min(maxValue, numValue))
|
| 9 |
+
|
| 10 |
+
}
|
src/lib/useAudio.ts
CHANGED
|
@@ -1,11 +1,37 @@
|
|
| 1 |
-
import { useCallback, useEffect, useRef } from 'react';
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
const stopAudio = useCallback(() => {
|
| 7 |
-
|
| 8 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
}, []);
|
| 10 |
|
| 11 |
// Helper function to handle conversion from Base64 to an ArrayBuffer
|
|
@@ -14,13 +40,13 @@ export function useAudio() {
|
|
| 14 |
return response.arrayBuffer();
|
| 15 |
}
|
| 16 |
|
| 17 |
-
const
|
| 18 |
-
async (base64Data?: string) => {
|
| 19 |
stopAudio(); // Stop any playing audio first
|
| 20 |
|
| 21 |
// If no base64 data provided, we don't attempt to play any audio
|
| 22 |
if (!base64Data) {
|
| 23 |
-
return;
|
| 24 |
}
|
| 25 |
|
| 26 |
// Initialize AudioContext
|
|
@@ -52,13 +78,41 @@ export function useAudio() {
|
|
| 52 |
source.connect(gainNode);
|
| 53 |
gainNode.connect(audioContext.destination);
|
| 54 |
|
| 55 |
-
//
|
| 56 |
-
source
|
|
|
|
|
|
|
| 57 |
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
}, (error) => {
|
| 63 |
console.error('Error decoding audio data:', error);
|
| 64 |
reject(error);
|
|
@@ -68,6 +122,25 @@ export function useAudio() {
|
|
| 68 |
[stopAudio]
|
| 69 |
);
|
| 70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
// Effect to handle cleanup on component unmount
|
| 72 |
useEffect(() => {
|
| 73 |
return () => {
|
|
@@ -75,6 +148,5 @@ export function useAudio() {
|
|
| 75 |
};
|
| 76 |
}, [stopAudio]);
|
| 77 |
|
| 78 |
-
|
| 79 |
-
return playAudio;
|
| 80 |
}
|
|
|
|
| 1 |
+
import { useCallback, useEffect, useRef, useState } from 'react';
|
| 2 |
|
| 3 |
+
// Helper Types
|
| 4 |
+
type UseAudioResponse = {
|
| 5 |
+
playback: (base64Data?: string, isLastTrackOfPlaylist?: boolean) => Promise<boolean>;
|
| 6 |
+
progress: number;
|
| 7 |
+
isLoaded: boolean;
|
| 8 |
+
isPlaying: boolean;
|
| 9 |
+
isSwitchingTracks: boolean; // when audio is temporary cut (but it's not a real pause)
|
| 10 |
+
togglePause: () => void;
|
| 11 |
+
};
|
| 12 |
|
| 13 |
+
export function useAudio(): UseAudioResponse {
|
| 14 |
+
const audioContextRef = useRef<AudioContext | null>(null);
|
| 15 |
+
const sourceNodeRef = useRef<AudioBufferSourceNode | null>(null);
|
| 16 |
+
const [progress, setProgress] = useState(0.0);
|
| 17 |
+
const [isPlaying, setIsPlaying] = useState(false);
|
| 18 |
+
const [isLoaded, setIsLoaded] = useState(false);
|
| 19 |
+
const [isSwitchingTracks, setSwitchingTracks] = useState(false);
|
| 20 |
+
const startTimeRef = useRef(0);
|
| 21 |
+
const pauseTimeRef = useRef(0);
|
| 22 |
+
|
| 23 |
const stopAudio = useCallback(() => {
|
| 24 |
+
try {
|
| 25 |
+
audioContextRef.current?.close();
|
| 26 |
+
} catch (err) {
|
| 27 |
+
// already closed probably
|
| 28 |
+
}
|
| 29 |
+
setSwitchingTracks(false);
|
| 30 |
+
|
| 31 |
+
sourceNodeRef.current = null;
|
| 32 |
+
sourceNodeRef.current = null;
|
| 33 |
+
|
| 34 |
+
// setProgress(0); // Reset progress
|
| 35 |
}, []);
|
| 36 |
|
| 37 |
// Helper function to handle conversion from Base64 to an ArrayBuffer
|
|
|
|
| 40 |
return response.arrayBuffer();
|
| 41 |
}
|
| 42 |
|
| 43 |
+
const playback = useCallback(
|
| 44 |
+
async (base64Data?: string, isLastTrackOfPlaylist?: boolean): Promise<boolean> => {
|
| 45 |
stopAudio(); // Stop any playing audio first
|
| 46 |
|
| 47 |
// If no base64 data provided, we don't attempt to play any audio
|
| 48 |
if (!base64Data) {
|
| 49 |
+
return false;
|
| 50 |
}
|
| 51 |
|
| 52 |
// Initialize AudioContext
|
|
|
|
| 78 |
source.connect(gainNode);
|
| 79 |
gainNode.connect(audioContext.destination);
|
| 80 |
|
| 81 |
+
// Assign source node to ref for progress tracking
|
| 82 |
+
sourceNodeRef.current = source;
|
| 83 |
+
source.start(0, pauseTimeRef.current % audioBuffer.duration); // Start at the correct offset if paused previously
|
| 84 |
+
startTimeRef.current = audioContextRef.current!.currentTime - pauseTimeRef.current;
|
| 85 |
|
| 86 |
+
setSwitchingTracks(false);
|
| 87 |
+
setProgress(0);
|
| 88 |
+
setIsLoaded(true);
|
| 89 |
+
setIsPlaying(true);
|
| 90 |
+
|
| 91 |
+
// Set up progress interval
|
| 92 |
+
const totalDuration = audioBuffer.duration;
|
| 93 |
+
const updateProgressInterval = setInterval(() => {
|
| 94 |
+
if (sourceNodeRef.current && audioContextRef.current) {
|
| 95 |
+
const currentTime = audioContextRef.current.currentTime;
|
| 96 |
+
const currentProgress = currentTime / totalDuration;
|
| 97 |
+
setProgress(currentProgress);
|
| 98 |
+
if (currentProgress >= 1.0) {
|
| 99 |
+
clearInterval(updateProgressInterval);
|
| 100 |
+
}
|
| 101 |
+
}
|
| 102 |
+
}, 50); // Update every 50ms
|
| 103 |
+
|
| 104 |
+
if (source) {
|
| 105 |
+
source.onended = () => {
|
| 106 |
+
// used to indicate a temporary stop, while we switch tracks
|
| 107 |
+
if (!isLastTrackOfPlaylist) {
|
| 108 |
+
setSwitchingTracks(true);
|
| 109 |
+
}
|
| 110 |
+
setIsPlaying(false);
|
| 111 |
+
clearInterval(updateProgressInterval);
|
| 112 |
+
stopAudio();
|
| 113 |
+
resolve(true);
|
| 114 |
+
};
|
| 115 |
+
}
|
| 116 |
}, (error) => {
|
| 117 |
console.error('Error decoding audio data:', error);
|
| 118 |
reject(error);
|
|
|
|
| 122 |
[stopAudio]
|
| 123 |
);
|
| 124 |
|
| 125 |
+
const togglePause = useCallback(() => {
|
| 126 |
+
if (!audioContextRef.current || !sourceNodeRef.current) {
|
| 127 |
+
return; // Do nothing if audio is not initialized
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
if (isPlaying) {
|
| 131 |
+
// Pause the audio
|
| 132 |
+
pauseTimeRef.current += audioContextRef.current.currentTime - startTimeRef.current;
|
| 133 |
+
sourceNodeRef.current.stop(); // This effectively "pauses" the audio, but it also means the sourceNode will be unusable
|
| 134 |
+
sourceNodeRef.current = null; // As the node is now unusable, we nullify it
|
| 135 |
+
setIsPlaying(false);
|
| 136 |
+
} else {
|
| 137 |
+
// Resume playing
|
| 138 |
+
audioContextRef.current.resume().then(() => {
|
| 139 |
+
playback(); // This will pick up where we left off due to pauseTimeRef
|
| 140 |
+
});
|
| 141 |
+
}
|
| 142 |
+
}, [audioContextRef, sourceNodeRef, isPlaying, playback]);
|
| 143 |
+
|
| 144 |
// Effect to handle cleanup on component unmount
|
| 145 |
useEffect(() => {
|
| 146 |
return () => {
|
|
|
|
| 148 |
};
|
| 149 |
}, [stopAudio]);
|
| 150 |
|
| 151 |
+
return { playback, isPlaying, isSwitchingTracks, isLoaded, progress, togglePause };
|
|
|
|
| 152 |
}
|