Spaces:
Runtime error
Runtime error
Ashhar
commited on
Commit
·
9dcdd49
1
Parent(s):
afcd851
clarified cache hit and miss msgs
Browse files
app/components/chat/AssistantMessage.tsx
CHANGED
@@ -16,13 +16,20 @@ export const AssistantMessage = memo(({ content, annotations }: AssistantMessage
|
|
16 |
completionTokens: number;
|
17 |
promptTokens: number;
|
18 |
totalTokens: number;
|
19 |
-
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
return (
|
22 |
<div className="overflow-hidden w-full">
|
23 |
{usage && (
|
24 |
<div className="text-sm text-bolt-elements-textSecondary mb-2">
|
25 |
Tokens: {usage.totalTokens} (prompt: {usage.promptTokens}, completion: {usage.completionTokens})
|
|
|
|
|
26 |
</div>
|
27 |
)}
|
28 |
<Markdown html>{content}</Markdown>
|
|
|
16 |
completionTokens: number;
|
17 |
promptTokens: number;
|
18 |
totalTokens: number;
|
19 |
+
isCacheHit?: boolean;
|
20 |
+
isCacheMiss?: boolean;
|
21 |
+
} = filteredAnnotations.find((annotation) => annotation.type === 'usage')?.value ?? undefined;
|
22 |
+
|
23 |
+
const cacheHitMsg = usage?.isCacheHit ? ' [Cache Hit]' : '';
|
24 |
+
const cacheMissMsg = usage?.isCacheMiss ? ' [Cache Miss]' : '';
|
25 |
|
26 |
return (
|
27 |
<div className="overflow-hidden w-full">
|
28 |
{usage && (
|
29 |
<div className="text-sm text-bolt-elements-textSecondary mb-2">
|
30 |
Tokens: {usage.totalTokens} (prompt: {usage.promptTokens}, completion: {usage.completionTokens})
|
31 |
+
<span className="text-sm text-green-500 ml-1">{cacheHitMsg}</span>
|
32 |
+
<span className="text-sm text-red-500 ml-1">{cacheMissMsg}</span>
|
33 |
</div>
|
34 |
)}
|
35 |
<Markdown html>{content}</Markdown>
|
app/routes/api.chat.ts
CHANGED
@@ -64,6 +64,9 @@ async function chatAction({ context, request }: ActionFunctionArgs) {
|
|
64 |
const cacheUsage = experimental_providerMetadata?.anthropic;
|
65 |
console.debug({ cacheUsage });
|
66 |
|
|
|
|
|
|
|
67 |
if (usage) {
|
68 |
cumulativeUsage.completionTokens += Math.round(usage.completionTokens || 0);
|
69 |
cumulativeUsage.promptTokens += Math.round(
|
@@ -84,6 +87,8 @@ async function chatAction({ context, request }: ActionFunctionArgs) {
|
|
84 |
completionTokens: cumulativeUsage.completionTokens,
|
85 |
promptTokens: cumulativeUsage.promptTokens,
|
86 |
totalTokens: cumulativeUsage.totalTokens,
|
|
|
|
|
87 |
},
|
88 |
});
|
89 |
},
|
|
|
64 |
const cacheUsage = experimental_providerMetadata?.anthropic;
|
65 |
console.debug({ cacheUsage });
|
66 |
|
67 |
+
const isCacheHit = !!cacheUsage?.cacheReadInputTokens;
|
68 |
+
const isCacheMiss = !!cacheUsage && !isCacheHit;
|
69 |
+
|
70 |
if (usage) {
|
71 |
cumulativeUsage.completionTokens += Math.round(usage.completionTokens || 0);
|
72 |
cumulativeUsage.promptTokens += Math.round(
|
|
|
87 |
completionTokens: cumulativeUsage.completionTokens,
|
88 |
promptTokens: cumulativeUsage.promptTokens,
|
89 |
totalTokens: cumulativeUsage.totalTokens,
|
90 |
+
isCacheHit,
|
91 |
+
isCacheMiss,
|
92 |
},
|
93 |
});
|
94 |
},
|