thinking toggle

PROMPTS.md

@@ -70,3 +70,9 @@ System: {{preprompt}}\nUser:{{#each messages}}{{#ifUser}}{{content}}\nFalcon:{{/
 ```env
 {{#each messages}}{{#ifUser}}<start_of_turn>user\n{{#if @first}}{{#if @root.preprompt}}{{@root.preprompt}}\n{{/if}}{{/if}}{{content}}<end_of_turn>\n<start_of_turn>model\n{{/ifUser}}{{#ifAssistant}}{{content}}<end_of_turn>\n{{/ifAssistant}}{{/each}}
 ```
+
+## Qwen3
+
+```env
+{{#if @root.preprompt}}<|im_start|>system\n{{@root.preprompt}}<|im_end|>\n{{/if}}{{#each messages}}{{#ifUser}}<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n{{/ifUser}}{{#ifAssistant}}{{content}}<|im_end|>\n{{/ifAssistant}}{{/each}}
+```

src/lib/components/chat/ChatInput.svelte

@@ -6,6 +6,7 @@
 	import IconInternet from "$lib/components/icons/IconInternet.svelte";
 	import IconImageGen from "$lib/components/icons/IconImageGen.svelte";
 	import IconPaperclip from "$lib/components/icons/IconPaperclip.svelte";
+	import IconThinking from "$lib/components/icons/IconThinking.svelte";
 	import { useSettingsStore } from "$lib/stores/settings";
 	import { webSearchParameters } from "$lib/stores/webSearchParameters";
 	import {
@@ -37,6 +38,7 @@
 		modelIsMultimodal?: boolean;
 		children?: import("svelte").Snippet;
 		onPaste?: (e: ClipboardEvent) => void;
+		showThinking?: boolean;
 	}
 
 	let {
@@ -51,6 +53,7 @@
 		modelIsMultimodal = false,
 		children,
 		onPaste,
+		showThinking = false,
 	}: Props = $props();
 
 	const onFileChange = async (e: Event) => {
@@ -68,7 +71,7 @@
 	let textareaElement: HTMLTextAreaElement | undefined = $state();
 	let isCompositionOn = $state(false);
 
-	const dispatch = createEventDispatcher<{ submit: void }>();
+	const dispatch = createEventDispatcher<{ submit: { text: string } }>();
 
 	onMount(() => {
 		if (!isVirtualKeyboard()) {
@@ -121,8 +124,11 @@
 			!isVirtualKeyboard() &&
 			value.trim() !== ""
 		) {
-			event.preventDefault();
-			dispatch("submit");
+			event.preventDefault();	
+            const textToSend = thinkingIsOn
+               ? `${value.trim()} /think`
+               : `${value.trim()} /no_think`;
+	        dispatch("submit", { text: textToSend });
 		}
 	}
 
@@ -158,6 +164,9 @@
 	let showExtraTools = $derived(modelHasTools && !assistant);
 
 	let showNoTools = $derived(!showWebSearch && !showImageGen && !showFileUpload && !showExtraTools);
+
+	let thinkingIsOn = $state(false);
+
 </script>
 
 <div class="flex min-h-full flex-1 flex-col" onpaste={onPaste}>
@@ -227,6 +236,33 @@
 					</button>
 				</HoverTooltip>
 			{/if}
+			{#if showThinking}
+				<HoverTooltip
+					label="Thinking"
+					position="top"
+					TooltipClassNames="text-xs !text-left !w-auto whitespace-nowrap !py-1 !mb-0 max-sm:hidden {thinkingIsOn
+						? 'hidden'
+						: ''}"
+				>
+					<button
+						class="base-tool"
+						class:active-tool={thinkingIsOn}
+						disabled={loading}
+						onclick={(e) => {
+							e.preventDefault();
+							thinkingIsOn = !thinkingIsOn;
+						}}
+					>
+					<IconThinking classNames="text-xl" />
+					{#if thinkingIsOn}
+							Thinking
+						{:else}
+							Not Thinking
+						{/if}
+					
+					</button>
+				</HoverTooltip>
+			{/if}
 			{#if showImageGen}
 				<HoverTooltip
 					label="Generate	images"

src/lib/components/chat/ChatMessage.svelte

@@ -313,7 +313,7 @@
 					<p
 						class="disabled w-full appearance-none whitespace-break-spaces text-wrap break-words bg-inherit px-5 py-3.5 text-gray-500 dark:text-gray-400"
 					>
-						{message.content.trim()}
+						{message.content.trim().replace(/\/(no_)?think$/, '')}
 					</p>
 				{:else}
 					<form
@@ -329,7 +329,7 @@
 							class="w-full whitespace-break-spaces break-words rounded-xl bg-gray-100 px-5 py-3.5 text-gray-500 *:h-max dark:bg-gray-800 dark:text-gray-400"
 							rows="5"
 							bind:this={editContentEl}
-							value={message.content.trim()}
+							value={message.content.trim().replace(/\/(no_)?think$/, '')}
 							onkeydown={handleKeyDown}
 							required
 						></textarea>

src/lib/components/chat/ChatWindow.svelte

@@ -73,6 +73,9 @@
 	let isSharedRecently = $state(false);
 	let editMsdgId: Message["id"] | null = $state(null);
 	let pastedLongContent = $state(false);
+	let showThinking = $derived(
+		currentModel.name === "Qwen/Qwen3_8B"
+		);
 
 	beforeNavigate(() => {
 		if (page.params.id) {
@@ -88,11 +91,19 @@
 		continue: { id: Message["id"] };
 	}>();
 
-	const handleSubmit = () => {
+	// const handleSubmit = () => {
+	// 	if (loading) return;
+	// 	dispatch("message", message);
+	// 	message = "";
+	// };
+
+	function handleSubmit(ev?: CustomEvent<{ text: string }>) {
 		if (loading) return;
-		dispatch("message", message);
+
+		const content = ev?.detail?.text ?? message;
+		dispatch("message", content);
 		message = "";
-	};
+	}
 
 	let lastTarget: EventTarget | null = null;
 
@@ -438,6 +449,7 @@
 								disabled={isReadOnly || lastIsError}
 								modelHasTools={currentModel.tools}
 								modelIsMultimodal={currentModel.multimodal}
+								showThinking={showThinking}
 							/>
 						{/if}
 

src/lib/components/icons/IconThinking.svelte

@@ -0,0 +1,27 @@
+<script lang="ts">
+	interface Props {
+		classNames?: string;
+	}
+	let { classNames = "" }: Props = $props();
+</script>
+
+<svg
+    class={classNames}
+    xmlns="http://www.w3.org/2000/svg"  
+    width="20"  
+    height="20"  
+    viewBox="0 0 24 24"  
+    fill="none"  
+    stroke="currentColor"  
+    stroke-width="1.5"  
+    stroke-linecap="round"  
+    stroke-linejoin="round"  
+    >
+
+<path 
+    stroke="none" 
+    d="M0 0h24v24H0z" 
+    fill="none"
+    />
+<path
+d="M15.5 13a3.5 3.5 0 0 0 -3.5 3.5v1a3.5 3.5 0 0 0 7 0v-1.8" /><path d="M8.5 13a3.5 3.5 0 0 1 3.5 3.5v1a3.5 3.5 0 0 1 -7 0v-1.8" /><path d="M17.5 16a3.5 3.5 0 0 0 0 -7h-.5" /><path d="M19 9.3v-2.8a3.5 3.5 0 0 0 -7 0" /><path d="M6.5 16a3.5 3.5 0 0 1 0 -7h.5" /><path d="M5 9.3v-2.8a3.5 3.5 0 0 1 7 0v10" /></svg>

tgi_deploy.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+cd text-generation-inference
+conda create -n tgi python=3.11
+eval "$(/home/user/miniconda3/bin/conda shell.bash hook)"
+conda install -c conda-forge pkg-config openssl
+
+
+conda activate tgi
+export OPENSSL_DIR=$CONDA_PREFIX && \
+export OPENSSL_INCLUDE_DIR=$CONDA_PREFIX/include && \
+export OPENSSL_LIB_DIR=$CONDA_PREFIX/lib && \
+export PKG_CONFIG_PATH=$CONDA_PREFIX/lib/pkgconfig
+export PYTHONPATH=/home/user/miniconda3/envs/tgi/lib/python3.11/site-packages
+export LD_LIBRARY_PATH=/home/user/:$LD_LIBRARY_PATH
+ln -s /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1 /home/user/libnvidia-ml.so
+
+nohup text-generation-launcher   --model-id HuggingFaceH4/zephyr-7b-beta   -p 7860 &> qwen2.log &
+
+PYTHONPATH=/home/user/:$PYTHONPATH \
+LD_LIBRARY_PATH=/home/user/:$LD_LIBRARY_PATH \
+text-generation-launcher \
+  --model-id HuggingFaceH4/zephyr-7b-beta \
+  --disable-custom-kernels \
+  -p 7860
+
+LD_LIBRARY_PATH=/home/user/:$LD_LIBRARY_PATH \
+text-generation-launcher \
+  --model-id HuggingFaceH4/zephyr-7b-beta \
+  -p 7860
+
+LD_LIBRARY_PATH=/home/user/:$LD_LIBRARY_PATH text-generation-launcher --model-id Qwen/Qwen2.5-VL-7B-Instruct -p 7860
+LD_LIBRARY_PATH=/home/user/:$LD_LIBRARY_PATH text-generation-launcher --model-id Qwen/Qwen3-8B -p 7860
+
+
+text-generation-launcher \
+  --model-id HuggingFaceH4/zephyr-7b-beta \
+  --disable-custom-kernels
+
+
+# To run the server in the background, use:
+nohup text-generation-launcher \
+    --model-id mistralai/Mistral-7B-v0.1 \
+    --port 8080 \
+    --max-batch-prefill-tokens 2048 \
+    --max-batch-total-tokens 4096 \
+    --max-input-length 4096 \
+    --max-total-tokens 8192 \
+    --max-batch-size 32 \
+    --max-waiting-tokens 20 \
+    --hostname 0.0.0.0 \
+    --cuda-memory-fraction 0.95 \
+    --max-concurrent-requests 128 \
+    --trust-remote-code \
+    --json-output > tgi.log 2>&1 &
+
+
+# To stop the server, use:
+ps aux | grep text-generation-launcher
+pkill -f text-generation-launcher
+kill -9 $(nvidia-smi | grep python | awk '{ print $5 }')
+
+
+
+curl https://jdelavande-dev-tgi.hf.space/generate \
+  -X POST \
+  -H "Content-Type: application/json" \
+  -d '{"inputs":"Bonjour !", "parameters":{"max_new_tokens":20}}'
+
+  curl https://jdelavande-dev-tgi2.hf.space/ \
+  -X POST \
+  -H "Content-Type: application/json" \
+  -d '{"inputs":"Bonjour !", "parameters":{"max_new_tokens":20}}'
+
+    curl localhost:7860/generate \
+  -X POST \
+  -H "Content-Type: application/json" \
+  -d '{"inputs":"Bonjour !", "parameters":{"max_new_tokens":20}}'