feat: paste long context as plaintext files (#1549)

* feat: paste long context as a file

* fix: anim sources

* fix: animations

* feat: actually inject plaintext files in the prompt

* filter out files that are plain text

* feat: use custom MIME type for clipboard content

* feat: add better UI affordance for pasting

* fix: cleanup animations

src/lib/components/chat/ChatMessage.svelte

@@ -260,7 +260,7 @@
 			{#if message.files?.length}
 				<div class="flex h-fit flex-wrap gap-x-5 gap-y-2">
 					{#each message.files as file}
-						<UploadedFile {file} canClose={false} isPreview={false} />
+						<UploadedFile {file} canClose={false} />
 					{/each}
 				</div>
 			{/if}
@@ -410,7 +410,7 @@
 			{#if message.files?.length}
 				<div class="flex w-fit gap-4 px-5">
 					{#each message.files as file}
-						<UploadedFile {file} canClose={false} isPreview={false} />
+						<UploadedFile {file} canClose={false} />
 					{/each}
 				</div>
 			{/if}

src/lib/components/chat/ChatWindow.svelte

@@ -38,6 +38,9 @@
 	import type { ToolFront } from "$lib/types/Tool";
 	import ModelSwitch from "./ModelSwitch.svelte";
 
+	import { fly } from "svelte/transition";
+	import { cubicInOut } from "svelte/easing";
+
 	export let messages: Message[] = [];
 	export let loading = false;
 	export let pending = false;
@@ -55,6 +58,7 @@
 	let message: string;
 	let timeout: ReturnType<typeof setTimeout>;
 	let isSharedRecently = false;
+	$: pastedLongContent = false;
 	$: $page.params.id && (isSharedRecently = false);
 
 	const dispatch = createEventDispatcher<{
@@ -86,6 +90,21 @@
 	};
 
 	const onPaste = (e: ClipboardEvent) => {
+		const textContent = e.clipboardData?.getData("text");
+
+		if (textContent && textContent.length > 256) {
+			e.preventDefault();
+			pastedLongContent = true;
+			setTimeout(() => {
+				pastedLongContent = false;
+			}, 1000);
+			const pastedFile = new File([textContent], "Pasted Content", {
+				type: "application/vnd.chatui.clipboard",
+			});
+
+			files = [...files, pastedFile];
+		}
+
 		if (!e.clipboardData) {
 			return;
 		}
@@ -344,7 +363,10 @@
 		class="dark:via-gray-80 pointer-events-none absolute inset-x-0 bottom-0 z-0 mx-auto flex w-full max-w-3xl flex-col items-center justify-center bg-gradient-to-t from-white via-white/80 to-white/0 px-3.5 py-4 dark:border-gray-800 dark:from-gray-900 dark:to-gray-900/0 max-md:border-t max-md:bg-white max-md:dark:bg-gray-900 sm:px-5 md:py-8 xl:max-w-4xl [&>*]:pointer-events-auto"
 	>
 		{#if sources?.length && !loading}
-			<div class="flex flex-row flex-wrap justify-center gap-2.5 max-md:pb-3">
+			<div
+				in:fly|local={sources.length === 1 ? { y: -20, easing: cubicInOut } : undefined}
+				class="flex flex-row flex-wrap justify-center gap-2.5 rounded-xl max-md:pb-3"
+			>
 				{#each sources as source, index}
 					{#await source then src}
 						<UploadedFile
@@ -409,7 +431,10 @@
 				{#if onDrag && isFileUploadEnabled}
 					<FileDropzone bind:files bind:onDrag mimeTypes={activeMimeTypes} />
 				{:else}
-					<div class="flex w-full flex-1 border-none bg-transparent">
+					<div
+						class="flex w-full flex-1 rounded-xl border-none bg-transparent"
+						class:paste-glow={pastedLongContent}
+					>
 						{#if lastIsError}
 							<ChatInput value="Sorry, something went wrong. Please try again." disabled={true} />
 						{:else}
@@ -508,3 +533,22 @@
 		</div>
 	</div>
 </div>
+
+<style lang="postcss">
+	.paste-glow {
+		animation: glow 1s cubic-bezier(0.4, 0, 0.2, 1) forwards;
+		will-change: box-shadow;
+	}
+
+	@keyframes glow {
+		0% {
+			box-shadow: 0 0 0 0 rgba(59, 130, 246, 0.8);
+		}
+		50% {
+			box-shadow: 0 0 20px 4px rgba(59, 130, 246, 0.6);
+		}
+		100% {
+			box-shadow: 0 0 0 0 rgba(59, 130, 246, 0);
+		}
+	}
+</style>

src/lib/components/chat/UploadedFile.svelte

@@ -5,13 +5,13 @@
 	import CarbonClose from "~icons/carbon/close";
 	import CarbonDocumentBlank from "~icons/carbon/document-blank";
 	import CarbonDownload from "~icons/carbon/download";
-
+	import CarbonDocument from "~icons/carbon/document";
 	import Modal from "../Modal.svelte";
 	import AudioPlayer from "../players/AudioPlayer.svelte";
+	import EosIconsLoading from "~icons/eos-icons/loading";
 
 	export let file: MessageFile;
 	export let canClose = true;
-	export let isPreview = false;
 
 	$: showModal = false;
 	$: urlNotTrailing = $page.url.pathname.replace(/\/$/, "");
@@ -38,33 +38,74 @@
 	const isVideo = (mime: string) =>
 		mime.startsWith("video/") || mime === "mp4" || mime === "x-mpeg";
 
-	$: isClickable = isImage(file.mime) && !isPreview;
+	const isPlainText = (mime: string) =>
+		mime === "text/plain" ||
+		mime === "text/csv" ||
+		mime === "text/markdown" ||
+		mime === "application/json" ||
+		mime === "application/xml" ||
+		mime === "application/vnd.chatui.clipboard";
+
+	$: isClickable = isImage(file.mime) || isPlainText(file.mime);
 </script>
 
 {#if showModal && isClickable}
 	<!-- show the image file full screen, click outside to exit -->
-	<Modal width="sm:max-w-[500px]" on:close={() => (showModal = false)}>
-		{#if file.type === "hash"}
-			<img
-				src={urlNotTrailing + "/output/" + file.value}
-				alt="input from user"
-				class="aspect-auto"
-			/>
-		{:else}
-			<!-- handle the case where this is a base64 encoded image -->
-			<img
-				src={`data:${file.mime};base64,${file.value}`}
-				alt="input from user"
-				class="aspect-auto"
-			/>
+	<Modal width="sm:max-w-[800px]" on:close={() => (showModal = false)}>
+		{#if isImage(file.mime)}
+			{#if file.type === "hash"}
+				<img
+					src={urlNotTrailing + "/output/" + file.value}
+					alt="input from user"
+					class="aspect-auto"
+				/>
+			{:else}
+				<!-- handle the case where this is a base64 encoded image -->
+				<img
+					src={`data:${file.mime};base64,${file.value}`}
+					alt="input from user"
+					class="aspect-auto"
+				/>
+			{/if}
+		{:else if isPlainText(file.mime)}
+			<div class="relative flex h-full w-full flex-col gap-4 p-4">
+				<h3 class="-mb-2 pt-2 text-xl font-bold">{file.name}</h3>
+				<button
+					class="absolute right-4 top-4 text-xl text-gray-500 hover:text-gray-800"
+					on:click={() => (showModal = false)}
+				>
+					<CarbonClose class="text-xl" />
+				</button>
+				{#if file.type === "hash"}
+					{#await fetch(urlNotTrailing + "/output/" + file.value).then((res) => res.text())}
+						<div class="flex h-full w-full items-center justify-center">
+							<EosIconsLoading class="text-xl" />
+						</div>
+					{:then result}
+						<pre
+							class="w-full whitespace-pre-wrap break-words pt-0 text-sm"
+							class:font-sans={file.mime === "text/plain" ||
+								file.mime === "application/vnd.chatui.clipboard"}
+							class:font-mono={file.mime !== "text/plain" &&
+								file.mime !== "application/vnd.chatui.clipboard"}>{result}</pre>
+					{/await}
+				{:else}
+					<pre
+						class="w-full whitespace-pre-wrap break-words pt-0 text-sm"
+						class:font-sans={file.mime === "text/plain" ||
+							file.mime === "application/vnd.chatui.clipboard"}
+						class:font-mono={file.mime !== "text/plain" &&
+							file.mime !== "application/vnd.chatui.clipboard"}>{atob(file.value)}</pre>
+				{/if}
+			</div>
 		{/if}
 	</Modal>
 {/if}
 
-<button on:click={() => (showModal = true)} disabled={!isClickable}>
+<button on:click={() => (showModal = true)} disabled={!isClickable} class:clickable={isClickable}>
 	<div class="group relative flex items-center rounded-xl shadow-sm">
 		{#if isImage(file.mime)}
-			<div class=" overflow-hidden rounded-xl" class:size-24={isPreview} class:size-48={!isPreview}>
+			<div class="size-48 overflow-hidden rounded-xl">
 				<img
 					src={file.type === "base64"
 						? `data:${file.mime};base64,${file.value}`
@@ -92,9 +133,31 @@
 					controls
 				/>
 			</div>
+		{:else if isPlainText(file.mime)}
+			<div
+				class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
+				class:hoverable={isClickable}
+			>
+				<div
+					class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
+				>
+					<CarbonDocument class="text-base text-gray-700 dark:text-gray-300" />
+				</div>
+				<dl class="flex flex-col items-start truncate leading-tight">
+					<dd class="text-sm">
+						{truncateMiddle(file.name, 28)}
+					</dd>
+					{#if file.mime === "application/vnd.chatui.clipboard"}
+						<dt class="text-xs text-gray-400">Clipboard source</dt>
+					{:else}
+						<dt class="text-xs text-gray-400">{file.mime}</dt>
+					{/if}
+				</dl>
+			</div>
 		{:else if file.mime === "octet-stream"}
 			<div
 				class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
+				class:hoverable={isClickable}
 			>
 				<div
 					class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
@@ -120,13 +183,14 @@
 		{:else}
 			<div
 				class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
+				class:hoverable={isClickable}
 			>
 				<div
 					class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
 				>
 					<CarbonDocumentBlank class="text-base text-gray-700 dark:text-gray-300" />
 				</div>
-				<dl class="flex flex-col truncate leading-tight">
+				<dl class="flex flex-col items-start truncate leading-tight">
 					<dd class="text-sm">
 						{truncateMiddle(file.name, 28)}
 					</dd>
@@ -137,11 +201,18 @@
 		<!-- add a button on top that removes the image -->
 		{#if canClose}
 			<button
-				class="invisible absolute -right-2 -top-2 grid size-6 place-items-center rounded-full border bg-black group-hover:visible dark:border-gray-700"
-				on:click={() => dispatch("close")}
+				class="absolute -right-2 -top-2 z-10 grid size-6 place-items-center rounded-full border bg-black group-hover:visible dark:border-gray-700"
+				class:invisible={navigator.maxTouchPoints === 0}
+				on:click|stopPropagation|preventDefault={() => dispatch("close")}
 			>
 				<CarbonClose class=" text-xs  text-white" />
 			</button>
 		{/if}
 	</div>
 </button>
+
+<style lang="postcss">
+	.hoverable {
+		@apply hover:bg-gray-500/10;
+	}
+</style>

src/lib/server/endpoints/preprocessMessages.ts

@@ -11,7 +11,8 @@ export async function preprocessMessages(
 ): Promise<EndpointMessage[]> {
 	return Promise.resolve(messages)
 		.then((msgs) => addWebSearchContext(msgs, webSearch))
-		.then((msgs) => downloadFiles(msgs, convId));
+		.then((msgs) => downloadFiles(msgs, convId))
+		.then((msgs) => injectClipboardFiles(msgs));
 }
 
 function addWebSearchContext(messages: Message[], webSearch: Message["webSearch"]) {
@@ -54,3 +55,21 @@ async function downloadFiles(messages: Message[], convId: ObjectId): Promise<End
 		)
 	);
 }
+
+async function injectClipboardFiles(messages: EndpointMessage[]) {
+	return Promise.all(
+		messages.map((message) => {
+			const plaintextFiles = message.files
+				?.filter((file) => file.mime === "application/vnd.chatui.clipboard")
+				.map((file) => Buffer.from(file.value, "base64").toString("utf-8"));
+
+			if (!plaintextFiles || plaintextFiles.length === 0) return message;
+
+			return {
+				...message,
+				content: `${plaintextFiles.join("\n\n")}\n\n${message.content}`,
+				files: message.files?.filter((file) => file.mime !== "application/vnd.chatui.clipboard"),
+			};
+		})
+	);
+}