Upload folder using huggingface_hub

README.md

@@ -11,6 +11,19 @@ app_port: 3000
 ---
 
 
+---
+title: Responses.js
+emoji: 😻
+colorFrom: red
+colorTo: red
+sdk: docker
+pinned: false
+license: mit
+short_description: Check out https://github.com/huggingface/responses.js
+app_port: 3000
+---
+
+
 # responses.js
 
 A lightweight Express.js server that implements OpenAI's Responses API, built on top of Chat Completions and powered by Hugging Face Inference Providers.

src/routes/responses.ts

@@ -17,7 +17,10 @@ import type {
 	ResponseFunctionToolCall,
 	ResponseOutputItem,
 } from "openai/resources/responses/responses";
-import type { ChatCompletionInputTool } from "@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.js";
+import type {
+	ChatCompletionInputFunctionDefinition,
+	ChatCompletionInputTool,
+} from "@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.js";
 import { callMcpTool, connectMcpServer } from "../mcp.js";
 
 class StreamingError extends Error {
@@ -136,6 +139,7 @@ async function* runCreateResponseStream(
 	}
 
 	// Response completed event
+	responseObject.status = "completed";
 	yield {
 		type: "response.completed",
 		response: responseObject as Response,
@@ -226,34 +230,7 @@ async function* innerRunStream(
 		tools = undefined;
 	}
 
-	// If MCP approval requests => execute them and return (no LLM call)
-	if (Array.isArray(req.body.input)) {
-		for (const item of req.body.input) {
-			// Note: currently supporting only 1 mcp_approval_response per request
-			let shouldStop = false;
-			if (item.type === "mcp_approval_response" && item.approve) {
-				const approvalRequest = req.body.input.find(
-					(i) => i.type === "mcp_approval_request" && i.id === item.approval_request_id
-				) as McpApprovalRequestParams | undefined;
-				for await (const event of callApprovedMCPToolStream(
-					item.approval_request_id,
-					approvalRequest,
-					mcpToolsMapping,
-					responseObject
-				)) {
-					yield event;
-				}
-				shouldStop = true;
-			}
-			if (shouldStop) {
-				// stop if at least one approval request is processed
-				break;
-			}
-		}
-	}
-
-	// At this point, we have all tools and we know we want to call the LLM
-	// Let's prepare the payload and make the call!
+	// Prepare payload for the LLM
 
 	// Resolve model and provider
 	const model = req.body.model.includes("@") ? req.body.model.split("@")[1] : req.body.model;
@@ -356,9 +333,9 @@ async function* innerRunStream(
 	// Prepare payload for the LLM
 	const payload: ChatCompletionInput = {
 		// main params
-		model: model,
-		provider: provider,
-		messages: messages,
+		model,
+		provider,
+		messages,
 		stream: req.body.stream,
 		// options
 		max_tokens: req.body.max_output_tokens === null ? undefined : req.body.max_output_tokens,
@@ -392,21 +369,72 @@ async function* innerRunStream(
 		top_p: req.body.top_p,
 	};
 
-	// Call LLM
-	for await (const event of callLLMStream(apiKey, payload, responseObject, mcpToolsMapping)) {
-		yield event;
-	}
+	// If MCP approval requests => execute them and return (no LLM call)
+	if (Array.isArray(req.body.input)) {
+		for (const item of req.body.input) {
+			if (item.type === "mcp_approval_response" && item.approve) {
+				const approvalRequest = req.body.input.find(
+					(i) => i.type === "mcp_approval_request" && i.id === item.approval_request_id
+				) as McpApprovalRequestParams | undefined;
+				const mcpCallId = "mcp_" + item.approval_request_id.split("_")[1];
+				const mcpCall = req.body.input.find((i) => i.type === "mcp_call" && i.id === mcpCallId);
+				if (mcpCall) {
+					// MCP call for that approval request has already been made, so we can skip it
+					continue;
+				}
 
-	// Handle MCP tool calls if any
-	for await (const event of handleMCPToolCallsAfterLLM(responseObject, mcpToolsMapping)) {
-		yield event;
+				for await (const event of callApprovedMCPToolStream(
+					item.approval_request_id,
+					mcpCallId,
+					approvalRequest,
+					mcpToolsMapping,
+					responseObject,
+					payload
+				)) {
+					yield event;
+				}
+			}
+		}
 	}
+
+	// Call the LLM until no new message is added to the payload.
+	// New messages can be added if the LLM calls an MCP tool that is automatically run.
+	// A maximum number of iterations is set to avoid infinite loops.
+	let previousMessageCount: number;
+	let currentMessageCount = payload.messages.length;
+	const MAX_ITERATIONS = 5; // hard-coded
+	let iterations = 0;
+	do {
+		previousMessageCount = currentMessageCount;
+
+		for await (const event of handleOneTurnStream(apiKey, payload, responseObject, mcpToolsMapping)) {
+			yield event;
+		}
+
+		currentMessageCount = payload.messages.length;
+		iterations++;
+	} while (currentMessageCount > previousMessageCount && iterations < MAX_ITERATIONS);
 }
 
 async function* listMcpToolsStream(
 	tool: McpServerParams,
 	responseObject: IncompleteResponse
 ): AsyncGenerator<ResponseStreamEvent> {
+	const outputObject: ResponseOutputItem.McpListTools = {
+		id: generateUniqueId("mcpl"),
+		type: "mcp_list_tools",
+		server_label: tool.server_label,
+		tools: [],
+	};
+	responseObject.output.push(outputObject);
+
+	yield {
+		type: "response.output_item.added",
+		output_index: responseObject.output.length - 1,
+		item: outputObject,
+		sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
+	};
+
 	yield {
 		type: "response.mcp_list_tools.in_progress",
 		sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
@@ -419,17 +447,18 @@ async function* listMcpToolsStream(
 			type: "response.mcp_list_tools.completed",
 			sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
 		};
-		responseObject.output.push({
-			id: generateUniqueId("mcp_list_tools"),
-			type: "mcp_list_tools",
-			server_label: tool.server_label,
-			tools: mcpTools.tools.map((mcpTool) => ({
-				input_schema: mcpTool.inputSchema,
-				name: mcpTool.name,
-				annotations: mcpTool.annotations,
-				description: mcpTool.description,
-			})),
-		});
+		outputObject.tools = mcpTools.tools.map((mcpTool) => ({
+			input_schema: mcpTool.inputSchema,
+			name: mcpTool.name,
+			annotations: mcpTool.annotations,
+			description: mcpTool.description,
+		}));
+		yield {
+			type: "response.output_item.done",
+			output_index: responseObject.output.length - 1,
+			item: outputObject,
+			sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
+		};
 	} catch (error) {
 		const errorMessage = `Failed to list tools from MCP server '${tool.server_label}': ${error instanceof Error ? error.message : "Unknown error"}`;
 		console.error(errorMessage);
@@ -444,27 +473,31 @@ async function* listMcpToolsStream(
 /*
  * Call LLM and stream the response.
  */
-async function* callLLMStream(
+async function* handleOneTurnStream(
 	apiKey: string | undefined,
 	payload: ChatCompletionInput,
 	responseObject: IncompleteResponse,
 	mcpToolsMapping: Record<string, McpServerParams>
 ): AsyncGenerator<ResponseStreamEvent> {
 	const stream = new InferenceClient(apiKey).chatCompletionStream(payload);
+	let previousInputTokens = responseObject.usage?.input_tokens ?? 0;
+	let previousOutputTokens = responseObject.usage?.output_tokens ?? 0;
+	let previousTotalTokens = responseObject.usage?.total_tokens ?? 0;
 
 	for await (const chunk of stream) {
 		if (chunk.usage) {
 			// Overwrite usage with the latest chunk's usage
 			responseObject.usage = {
-				input_tokens: chunk.usage.prompt_tokens,
+				input_tokens: previousInputTokens + chunk.usage.prompt_tokens,
 				input_tokens_details: { cached_tokens: 0 },
-				output_tokens: chunk.usage.completion_tokens,
+				output_tokens: previousOutputTokens + chunk.usage.completion_tokens,
 				output_tokens_details: { reasoning_tokens: 0 },
-				total_tokens: chunk.usage.total_tokens,
+				total_tokens: previousTotalTokens + chunk.usage.total_tokens,
 			};
 		}
 
 		const delta = chunk.choices[0].delta;
+
 		if (delta.content) {
 			let currentOutputItem = responseObject.output.at(-1);
 
@@ -528,32 +561,45 @@ async function* callLLMStream(
 			};
 		} else if (delta.tool_calls && delta.tool_calls.length > 0) {
 			if (delta.tool_calls.length > 1) {
-				throw new StreamingError("Not implemented: multiple tool calls are not supported.");
+				console.log("Multiple tool calls are not supported. Only the first one will be processed.");
 			}
 
 			let currentOutputItem = responseObject.output.at(-1);
-			if (currentOutputItem?.type !== "mcp_call" && currentOutputItem?.type !== "function_call") {
-				if (!delta.tool_calls[0].function.name) {
-					throw new StreamingError("Tool call function name is required when starting a new tool call.");
+			if (delta.tool_calls[0].function.name) {
+				const functionName = delta.tool_calls[0].function.name;
+				// Tool call with a name => new tool call
+				let newOutputObject:
+					| ResponseOutputItem.McpCall
+					| ResponseFunctionToolCall
+					| ResponseOutputItem.McpApprovalRequest;
+				if (functionName in mcpToolsMapping) {
+					if (requiresApproval(functionName, mcpToolsMapping)) {
+						newOutputObject = {
+							id: generateUniqueId("mcpr"),
+							type: "mcp_approval_request",
+							name: functionName,
+							server_label: mcpToolsMapping[functionName].server_label,
+							arguments: "",
+						};
+					} else {
+						newOutputObject = {
+							type: "mcp_call",
+							id: generateUniqueId("mcp"),
+							name: functionName,
+							server_label: mcpToolsMapping[functionName].server_label,
+							arguments: "",
+						};
+					}
+				} else {
+					newOutputObject = {
+						type: "function_call",
+						id: generateUniqueId("fc"),
+						call_id: delta.tool_calls[0].id,
+						name: functionName,
+						arguments: "",
+					};
 				}
 
-				const newOutputObject: ResponseOutputItem.McpCall | ResponseFunctionToolCall =
-					delta.tool_calls[0].function.name in mcpToolsMapping
-						? {
-								type: "mcp_call",
-								id: generateUniqueId("mcp_call"),
-								name: delta.tool_calls[0].function.name,
-								server_label: mcpToolsMapping[delta.tool_calls[0].function.name].server_label,
-								arguments: "",
-							}
-						: {
-								type: "function_call",
-								id: generateUniqueId("fc"),
-								call_id: delta.tool_calls[0].id,
-								name: delta.tool_calls[0].function.name,
-								arguments: "",
-							};
-
 				// Response output item added event
 				responseObject.output.push(newOutputObject);
 				yield {
@@ -562,21 +608,36 @@ async function* callLLMStream(
 					item: newOutputObject,
 					sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
 				};
+				if (newOutputObject.type === "mcp_call") {
+					yield {
+						type: "response.mcp_call.in_progress",
+						sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
+						item_id: newOutputObject.id,
+						output_index: responseObject.output.length - 1,
+					};
+				}
 			}
 
-			// Current item is necessarily a tool call
-			currentOutputItem = responseObject.output.at(-1) as ResponseOutputItem.McpCall | ResponseFunctionToolCall;
-			currentOutputItem.arguments += delta.tool_calls[0].function.arguments;
-			yield {
-				type:
-					currentOutputItem.type === "mcp_call"
-						? "response.mcp_call.arguments_delta"
-						: "response.function_call_arguments.delta",
-				item_id: currentOutputItem.id as string,
-				output_index: responseObject.output.length - 1,
-				delta: delta.tool_calls[0].function.arguments,
-				sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
-			};
+			if (delta.tool_calls[0].function.arguments) {
+				// Current item is necessarily a tool call
+				currentOutputItem = responseObject.output.at(-1) as
+					| ResponseOutputItem.McpCall
+					| ResponseFunctionToolCall
+					| ResponseOutputItem.McpApprovalRequest;
+				currentOutputItem.arguments += delta.tool_calls[0].function.arguments;
+				if (currentOutputItem.type === "mcp_call" || currentOutputItem.type === "function_call") {
+					yield {
+						type:
+							currentOutputItem.type === "mcp_call"
+								? ("response.mcp_call_arguments.delta" as "response.mcp_call.arguments_delta") // bug workaround (see https://github.com/openai/openai-node/issues/1562)
+								: "response.function_call_arguments.delta",
+						item_id: currentOutputItem.id as string,
+						output_index: responseObject.output.length - 1,
+						delta: delta.tool_calls[0].function.arguments,
+						sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
+					};
+				}
+			}
 		}
 	}
 
@@ -632,12 +693,65 @@ async function* callLLMStream(
 			};
 		} else if (lastOutputItem?.type === "mcp_call") {
 			yield {
-				type: "response.mcp_call.arguments_done",
+				type: "response.mcp_call_arguments.done" as "response.mcp_call.arguments_done", // bug workaround (see https://github.com/openai/openai-node/issues/1562)
 				item_id: lastOutputItem.id as string,
 				output_index: responseObject.output.length - 1,
 				arguments: lastOutputItem.arguments,
 				sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
 			};
+
+			// Call MCP tool
+			const toolParams = mcpToolsMapping[lastOutputItem.name];
+			const toolResult = await callMcpTool(toolParams, lastOutputItem.name, lastOutputItem.arguments);
+			if (toolResult.error) {
+				lastOutputItem.error = toolResult.error;
+				yield {
+					type: "response.mcp_call.failed",
+					sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
+				};
+			} else {
+				lastOutputItem.output = toolResult.output;
+				yield {
+					type: "response.mcp_call.completed",
+					sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
+				};
+			}
+
+			yield {
+				type: "response.output_item.done",
+				output_index: responseObject.output.length - 1,
+				item: lastOutputItem,
+				sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
+			};
+
+			// Updating the payload for next LLM call
+			payload.messages.push(
+				{
+					role: "assistant",
+					tool_calls: [
+						{
+							id: lastOutputItem.id,
+							type: "function",
+							function: {
+								name: lastOutputItem.name,
+								arguments: lastOutputItem.arguments,
+								// Hacky: type is not correct in inference.js. Will fix it but in the meantime we need to cast it.
+								// TODO: fix it in the inference.js package. Should be "arguments" and not "parameters".
+							} as unknown as ChatCompletionInputFunctionDefinition,
+						},
+					],
+				},
+				{
+					role: "tool",
+					tool_call_id: lastOutputItem.id,
+					content: lastOutputItem.output
+						? lastOutputItem.output
+						: lastOutputItem.error
+							? `Error: ${lastOutputItem.error}`
+							: "",
+				}
+			);
+		} else if (lastOutputItem?.type === "mcp_approval_request") {
 			yield {
 				type: "response.output_item.done",
 				output_index: responseObject.output.length - 1,
@@ -657,9 +771,11 @@ async function* callLLMStream(
  */
 async function* callApprovedMCPToolStream(
 	approval_request_id: string,
+	mcpCallId: string,
 	approvalRequest: McpApprovalRequestParams | undefined,
 	mcpToolsMapping: Record<string, McpServerParams>,
-	responseObject: IncompleteResponse
+	responseObject: IncompleteResponse,
+	payload: ChatCompletionInput
 ): AsyncGenerator<ResponseStreamEvent> {
 	if (!approvalRequest) {
 		throw new Error(`MCP approval request '${approval_request_id}' not found`);
@@ -667,7 +783,7 @@ async function* callApprovedMCPToolStream(
 
 	const outputObject: ResponseOutputItem.McpCall = {
 		type: "mcp_call",
-		id: generateUniqueId("mcp_call"),
+		id: mcpCallId,
 		name: approvalRequest.name,
 		server_label: approvalRequest.server_label,
 		arguments: approvalRequest.arguments,
@@ -698,83 +814,55 @@ async function* callApprovedMCPToolStream(
 			type: "response.mcp_call.failed",
 			sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
 		};
-		throw new Error(outputObject.error);
+	} else {
+		outputObject.output = toolResult.output;
+		yield {
+			type: "response.mcp_call.completed",
+			sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
+		};
 	}
 
-	outputObject.output = toolResult.output;
-	yield {
-		type: "response.mcp_call.completed",
-		sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
-	};
 	yield {
 		type: "response.output_item.done",
 		output_index: responseObject.output.length - 1,
 		item: outputObject,
 		sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
 	};
-}
 
-async function* handleMCPToolCallsAfterLLM(
-	responseObject: IncompleteResponse,
-	mcpToolsMapping: Record<string, McpServerParams>
-): AsyncGenerator<ResponseStreamEvent> {
-	for (let output_index = 0; output_index < responseObject.output.length; output_index++) {
-		const outputItem = responseObject.output[output_index];
-		if (outputItem.type === "mcp_call") {
-			const toolCall = outputItem as ResponseOutputItem.McpCall;
-			const toolParams = mcpToolsMapping[toolCall.name];
-			if (toolParams) {
-				const approvalRequired =
-					toolParams.require_approval === "always"
-						? true
-						: toolParams.require_approval === "never"
-							? false
-							: toolParams.require_approval.always?.tool_names?.includes(toolCall.name)
-								? true
-								: toolParams.require_approval.never?.tool_names?.includes(toolCall.name)
-									? false
-									: true; // behavior is undefined in specs, let's default to
-
-				if (approvalRequired) {
-					const approvalRequest: ResponseOutputItem.McpApprovalRequest = {
-						type: "mcp_approval_request",
-						id: generateUniqueId("mcp_approval_request"),
-						name: toolCall.name,
-						server_label: toolParams.server_label,
-						arguments: toolCall.arguments,
-					};
-					responseObject.output.push(approvalRequest);
-					yield {
-						type: "response.output_item.added",
-						output_index: responseObject.output.length,
-						item: approvalRequest,
-						sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
-					};
-				} else {
-					responseObject.output.push;
-					yield {
-						type: "response.mcp_call.in_progress",
-						item_id: toolCall.id,
-						sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
-						output_index,
-					};
-					const toolResult = await callMcpTool(toolParams, toolCall.name, toolCall.arguments);
-					if (toolResult.error) {
-						toolCall.error = toolResult.error;
-						yield {
-							type: "response.mcp_call.failed",
-							sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
-						};
-						throw new Error(toolCall.error);
-					} else {
-						toolCall.output = toolResult.output;
-						yield {
-							type: "response.mcp_call.completed",
-							sequence_number: SEQUENCE_NUMBER_PLACEHOLDER,
-						};
-					}
-				}
-			}
+	// Updating the payload for next LLM call
+	payload.messages.push(
+		{
+			role: "assistant",
+			tool_calls: [
+				{
+					id: outputObject.id,
+					type: "function",
+					function: {
+						name: outputObject.name,
+						arguments: outputObject.arguments,
+						// Hacky: type is not correct in inference.js. Will fix it but in the meantime we need to cast it.
+						// TODO: fix it in the inference.js package. Should be "arguments" and not "parameters".
+					} as unknown as ChatCompletionInputFunctionDefinition,
+				},
+			],
+		},
+		{
+			role: "tool",
+			tool_call_id: outputObject.id,
+			content: outputObject.output ? outputObject.output : outputObject.error ? `Error: ${outputObject.error}` : "",
 		}
-	}
+	);
+}
+
+function requiresApproval(toolName: string, mcpToolsMapping: Record<string, McpServerParams>): boolean {
+	const toolParams = mcpToolsMapping[toolName];
+	return toolParams.require_approval === "always"
+		? true
+		: toolParams.require_approval === "never"
+			? false
+			: toolParams.require_approval.always?.tool_names?.includes(toolName)
+				? true
+				: toolParams.require_approval.never?.tool_names?.includes(toolName)
+					? false
+					: true; // behavior is undefined in specs, let's default to true
 }

src/schemas.ts

@@ -125,7 +125,7 @@ export const createResponseParamsSchema = z.object({
 					output: z.string(),
 					type: z.literal("function_call_output"),
 					id: z.string().optional(),
-					status: z.enum(["in_progress", "completed", "incomplete"]),
+					status: z.enum(["in_progress", "completed", "incomplete"]).optional(),
 				}),
 				z.object({
 					type: z.literal("mcp_list_tools"),