Skip to content

Commit 6ac355e

Browse files
lgrammellgrammel
authoredAug 16, 2024··
feat (provider/anthropic): add cache control support (#2697)
·
@ai-sdk/xai@1.2.1ai@3.3.9
1 parent a47cb9c commit 6ac355e

40 files changed

+1580
-350
lines changed
 

‎.changeset/nine-paws-flash.md

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@ai-sdk/anthropic': patch
3+
'@ai-sdk/provider': patch
4+
'ai': patch
5+
---
6+
7+
feat (provider/anthropic): add cache control support

‎content/docs/07-reference/ai-sdk-core/01-generate-text.mdx

+6
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,12 @@ To see `generateText` in action, check out [these examples](#examples).
481481
description:
482482
'Warnings from the model provider (e.g. unsupported settings).',
483483
},
484+
{
485+
name: 'experimental_providerMetadata',
486+
type: 'Record<string,Record<string,JSONValue>> | undefined',
487+
description:
488+
'Optional metadata from the provider. The outer key is the provider name. The inner values are the metadata. Details depend on the provider.',
489+
},
484490
{
485491
name: 'responseMessages',
486492
type: 'Array<CoreAssistantMessage | CoreToolMessage>',

‎content/docs/07-reference/ai-sdk-core/02-stream-text.mdx

+12
Original file line numberDiff line numberDiff line change
@@ -595,6 +595,12 @@ To see `streamText` in action, check out [these examples](#examples).
595595
},
596596
],
597597
},
598+
{
599+
name: 'experimental_providerMetadata',
600+
type: 'Record<string,Record<string,JSONValue>> | undefined',
601+
description:
602+
'Optional metadata from the provider. The outer key is the provider name. The inner values are the metadata. Details depend on the provider.',
603+
},
598604
{
599605
name: 'text',
600606
type: 'string',
@@ -679,6 +685,12 @@ To see `streamText` in action, check out [these examples](#examples).
679685
},
680686
],
681687
},
688+
{
689+
name: 'experimental_providerMetadata',
690+
type: 'Promise<Record<string,Record<string,JSONValue>> | undefined>',
691+
description:
692+
'Optional metadata from the provider. Resolved whe the response is finished. The outer key is the provider name. The inner values are the metadata. Details depend on the provider.',
693+
},
682694
{
683695
name: 'text',
684696
type: 'Promise<string>',

‎content/docs/07-reference/ai-sdk-core/03-generate-object.mdx

+6
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,12 @@ To see `generateObject` in action, check out [these examples](#examples).
449449
description:
450450
'Warnings from the model provider (e.g. unsupported settings).',
451451
},
452+
{
453+
name: 'experimental_providerMetadata',
454+
type: 'Record<string,Record<string,JSONValue>> | undefined',
455+
description:
456+
'Optional metadata from the provider. The outer key is the provider name. The inner values are the metadata. Details depend on the provider.',
457+
},
452458
{
453459
name: 'toJsonResponse',
454460
type: '(init?: ResponseInit) => Response',

‎content/docs/07-reference/ai-sdk-core/04-stream-object.mdx

+12
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,12 @@ To see `streamObject` in action, check out [these examples](#examples).
420420
},
421421
],
422422
},
423+
{
424+
name: 'experimental_providerMetadata',
425+
type: 'Record<string,Record<string,JSONValue>> | undefined',
426+
description:
427+
'Optional metadata from the provider. The outer key is the provider name. The inner values are the metadata. Details depend on the provider.',
428+
},
423429
{
424430
name: 'object',
425431
type: 'T | undefined',
@@ -495,6 +501,12 @@ To see `streamObject` in action, check out [these examples](#examples).
495501
},
496502
],
497503
},
504+
{
505+
name: 'experimental_providerMetadata',
506+
type: 'Promise<Record<string,Record<string,JSONValue>> | undefined>',
507+
description:
508+
'Optional metadata from the provider. Resolved whe the response is finished. The outer key is the provider name. The inner values are the metadata. Details depend on the provider.',
509+
},
498510
{
499511
name: 'object',
500512
type: 'Promise<T>',

‎content/providers/01-ai-sdk-providers/05-anthropic.mdx

+56-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
22
title: Anthropic
3-
description: Learn how to use Anthropic.
3+
description: Learn how to use the Anthropic provider for the Vercel AI SDK.
44
---
55

66
# Anthropic Provider
@@ -74,7 +74,15 @@ Some models have multi-modal capabilities.
7474
const model = anthropic('claude-3-haiku-20240307');
7575
```
7676

77-
### Example
77+
The following optional settings are available for Anthropic models:
78+
79+
- **cacheControl** _boolean_
80+
81+
Enable the Anthropic cache control beta.
82+
83+
You can then use provider metadata to set cache control breakpoints ([example](#example-cache-control))
84+
85+
### Example: Generate Text
7886

7987
You can use Anthropic language models to generate text with the `generateText` function:
8088

@@ -91,6 +99,52 @@ const { text } = await generateText({
9199
Anthropic language models can also be used in the `streamText`, `generateObject`, `streamObject`, and `streamUI` functions
92100
(see [AI SDK Core](/docs/ai-sdk-core) and [AI SDK RSC](/docs/ai-sdk-rsc)).
93101

102+
### Example: Cache Control
103+
104+
You can enable the cache control beta by setting the `cacheControl` option to `true` when creating the model instance.
105+
106+
In the messages and message parts, you can then use the `experimental_providerMetadata` property to set cache control breakpoints.
107+
You need to set the `anthropic` property in the `experimental_providerMetadata` object to `{ cacheControl: { type: 'ephemeral' } }` to set a cache control breakpoint.
108+
109+
The cache creation input tokens are then returned in the `experimental_providerMetadata` object
110+
for `generateText` and `generateObject`, again under the `anthropic` property.
111+
When you use `streamText` or `streamObject`, the response contains a promise
112+
that resolves to the metadata. Alternatively you can receive it in the
113+
`onFinish` callback.
114+
115+
```ts highlight="8,18-20,29-30"
116+
import { anthropic } from '@ai-sdk/anthropic';
117+
import { generateText } from 'ai';
118+
119+
const errorMessage = '... long error message ...';
120+
121+
const result = await generateText({
122+
model: anthropic('claude-3-5-sonnet-20240620', {
123+
cacheControl: true,
124+
}),
125+
messages: [
126+
{
127+
role: 'user',
128+
content: [
129+
{ type: 'text', text: 'You are a JavaScript expert.' },
130+
{
131+
type: 'text',
132+
text: `Error message: ${errorMessage}`,
133+
experimental_providerMetadata: {
134+
anthropic: { cacheControl: { type: 'ephemeral' } },
135+
},
136+
},
137+
{ type: 'text', text: 'Explain the error message.' },
138+
],
139+
},
140+
],
141+
});
142+
143+
console.log(result.text);
144+
console.log(result.experimental_providerMetadata?.anthropic);
145+
// e.g. { cacheCreationInputTokens: 2118, cacheReadInputTokens: 0 }
146+
```
147+
94148
### Model Capabilities
95149

96150
| Model | Image Input | Object Generation | Tool Usage | Tool Streaming |

‎examples/ai-core/data/error-message.txt

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
❯ pnpm tsx src/generate-text/anthropic-cache-control.ts
2+
Body {
3+
"model": "claude-3-5-sonnet-20240620",
4+
"max_tokens": 4096,
5+
"temperature": 0,
6+
"messages": [
7+
{
8+
"role": "user",
9+
"content": [
10+
{
11+
"type": "text",
12+
"text": "You are a JavaScript expert."
13+
},
14+
{
15+
"type": "text",
16+
"text": "Error messages: \nAPICallError [AI_APICallError]: Failed to process error response\n at postToApi (/Users/larsgrammel/repositories/ai/packages/provider-utils/dist/index.js:382:15)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n ... 4 lines matching cause stack trace ...\n at async fn (/Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:2723:36)\n at async /Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:339:22\n at async main (/Users/larsgrammel/repositories/ai/examples/ai-core/src/generate-text/anthropic-cache-control.ts:2:1351) {\n cause: TypeError: Body is unusable\n at consumeBody (node:internal/deps/undici/undici:4281:15)\n at _Response.text (node:internal/deps/undici/undici:4236:18)\n at /Users/larsgrammel/repositories/ai/packages/provider-utils/dist/index.js:443:39\n at postToApi (/Users/larsgrammel/repositories/ai/packages/provider-utils/dist/index.js:373:34)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async AnthropicMessagesLanguageModel.doGenerate (/Users/larsgrammel/repositories/ai/packages/anthropic/dist/index.js:316:50)\n at async fn (/Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:2748:34)\n at async /Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:339:22\n at async _retryWithExponentialBackoff (/Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:170:12)\n at async fn (/Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:2723:36),\n url: 'https://api.anthropic.com/v1/messages',\n requestBodyValues: {\n model: 'claude-3-5-sonnet-20240620',\n top_k: undefined,\n max_tokens: 4096,\n temperature: 0,\n top_p: undefined,\n stop_sequences: undefined,\n system: undefined,\n messages: [ [Object] ],\n tools: undefined,\n tool_choice: undefined\n },\n statusCode: 400,\n responseHeaders: {\n 'cf-cache-status': 'DYNAMIC',\n 'cf-ray': '8b39b60ab8734516-TXL',\n connection: 'keep-alive',\n 'content-length': '171',\n 'content-type': 'application/json',\n date: 'Thu, 15 Aug 2024 14:00:28 GMT',\n 'request-id': 'req_01PLrS159iiihG7kS9PFQiqx',\n server: 'cloudflare',\n via: '1.1 google',\n 'x-cloud-trace-context': '1371f8e6d358102b79d109db3829d62e',\n 'x-robots-tag': 'none',\n 'x-should-retry': 'false'\n },\n responseBody: undefined,\n isRetryable: false,\n data: undefined,\n [Symbol(vercel.ai.error)]: true,\n [Symbol(vercel.ai.error.AI_APICallError)]: true\n}",
17+
"cache_control": {
18+
"type": "ephemeral"
19+
}
20+
},
21+
{
22+
"type": "text",
23+
"text": "Explain the error message."
24+
}
25+
]
26+
}
27+
]
28+
}
29+
Fetched {"type":"error","error":{"type":"invalid_request_error","message":"The message up to and including the first cache-control block must be at least 1024 tokens. Found: 939."}}
30+
31+
APICallError [AI_APICallError]: Failed to process error response
32+
at postToApi (/Users/larsgrammel/repositories/ai/packages/provider-utils/dist/index.js:382:15)
33+
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
34+
... 4 lines matching cause stack trace ...
35+
at async fn (/Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:2723:36)
36+
at async /Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:339:22
37+
at async main (/Users/larsgrammel/repositories/ai/examples/ai-core/src/generate-text/anthropic-cache-control.ts:54:361) {
38+
cause: TypeError: Body is unusable
39+
at consumeBody (node:internal/deps/undici/undici:4281:15)
40+
at _Response.text (node:internal/deps/undici/undici:4236:18)
41+
at /Users/larsgrammel/repositories/ai/packages/provider-utils/dist/index.js:443:39
42+
at postToApi (/Users/larsgrammel/repositories/ai/packages/provider-utils/dist/index.js:373:34)
43+
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
44+
at async AnthropicMessagesLanguageModel.doGenerate (/Users/larsgrammel/repositories/ai/packages/anthropic/dist/index.js:316:50)
45+
at async fn (/Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:2748:34)
46+
at async /Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:339:22
47+
at async _retryWithExponentialBackoff (/Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:170:12)
48+
at async fn (/Users/larsgrammel/repositories/ai/packages/ai/dist/index.js:2723:36),
49+
url: 'https://api.anthropic.com/v1/messages',
50+
requestBodyValues: {
51+
model: 'claude-3-5-sonnet-20240620',
52+
top_k: undefined,
53+
max_tokens: 4096,
54+
temperature: 0,
55+
top_p: undefined,
56+
stop_sequences: undefined,
57+
system: undefined,
58+
messages: [ [Object] ],
59+
tools: undefined,
60+
tool_choice: undefined
61+
},
62+
statusCode: 400,
63+
responseHeaders: {
64+
'cf-cache-status': 'DYNAMIC',
65+
'cf-ray': '8b39b87a8f684541-TXL',
66+
connection: 'keep-alive',
67+
'content-length': '173',
68+
'content-type': 'application/json',
69+
date: 'Thu, 15 Aug 2024 14:02:08 GMT',
70+
'request-id': 'req_01YZqjpifTdvLZqfwBieLs44',
71+
server: 'cloudflare',
72+
via: '1.1 google',
73+
'x-cloud-trace-context': '00f2b1629d0dc8c6a4714db1dbdb4c2c',
74+
'x-robots-tag': 'none',
75+
'x-should-retry': 'false'
76+
},
77+
responseBody: undefined,
78+
isRetryable: false,
79+
data: undefined,
80+
[Symbol(vercel.ai.error)]: true,
81+
[Symbol(vercel.ai.error.AI_APICallError)]: true
82+
}
83+
}

‎examples/ai-core/src/generate-text/anthropic-cache-control.ts

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import { anthropic } from '@ai-sdk/anthropic';
2+
import { generateText } from 'ai';
3+
import dotenv from 'dotenv';
4+
import fs from 'node:fs';
5+
6+
dotenv.config();
7+
8+
const errorMessage = fs.readFileSync('data/error-message.txt', 'utf8');
9+
10+
async function main() {
11+
const result = await generateText({
12+
model: anthropic('claude-3-5-sonnet-20240620', {
13+
cacheControl: true,
14+
}),
15+
messages: [
16+
{
17+
role: 'user',
18+
content: [
19+
{
20+
type: 'text',
21+
text: 'You are a JavaScript expert.',
22+
},
23+
{
24+
type: 'text',
25+
text: `Error message: ${errorMessage}`,
26+
experimental_providerMetadata: {
27+
anthropic: {
28+
cacheControl: { type: 'ephemeral' },
29+
},
30+
},
31+
},
32+
{
33+
type: 'text',
34+
text: 'Explain the error message.',
35+
},
36+
],
37+
},
38+
],
39+
});
40+
41+
console.log(result.text);
42+
console.log(result.experimental_providerMetadata?.anthropic);
43+
// e.g. { cacheCreationInputTokens: 2118, cacheReadInputTokens: 0 }
44+
}
45+
46+
main().catch(console.error);

‎examples/ai-core/src/stream-text/anthropic-cache-control.ts

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import { anthropic } from '@ai-sdk/anthropic';
2+
import { streamText } from 'ai';
3+
import dotenv from 'dotenv';
4+
import fs from 'node:fs';
5+
6+
dotenv.config();
7+
8+
const errorMessage = fs.readFileSync('data/error-message.txt', 'utf8');
9+
10+
async function main() {
11+
const result = await streamText({
12+
model: anthropic('claude-3-5-sonnet-20240620', {
13+
cacheControl: true,
14+
}),
15+
messages: [
16+
{
17+
role: 'user',
18+
content: [
19+
{
20+
type: 'text',
21+
text: 'You are a JavaScript expert.',
22+
},
23+
{
24+
type: 'text',
25+
text: `Error message: ${errorMessage}`,
26+
experimental_providerMetadata: {
27+
anthropic: {
28+
cacheControl: { type: 'ephemeral' },
29+
},
30+
},
31+
},
32+
{
33+
type: 'text',
34+
text: 'Explain the error message.',
35+
},
36+
],
37+
},
38+
],
39+
onFinish({ experimental_providerMetadata }) {
40+
console.log();
41+
console.log('=== onFinish ===');
42+
console.log(experimental_providerMetadata?.anthropic);
43+
},
44+
});
45+
46+
for await (const textPart of result.textStream) {
47+
process.stdout.write(textPart);
48+
}
49+
50+
console.log('=== providerMetadata Promise ===');
51+
console.log((await result.experimental_providerMetadata)?.anthropic);
52+
// e.g. { cacheCreationInputTokens: 2118, cacheReadInputTokens: 0 }
53+
}
54+
55+
main().catch(console.error);

‎packages/ai/core/generate-object/generate-object-result.ts

+13-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
import { CallWarning, FinishReason, LogProbs } from '../types';
1+
import {
2+
CallWarning,
3+
FinishReason,
4+
LogProbs,
5+
ProviderMetadata,
6+
} from '../types';
27
import { CompletionTokenUsage } from '../types/token-usage';
38

49
/**
@@ -41,6 +46,13 @@ export interface GenerateObjectResult<T> {
4146
*/
4247
readonly logprobs: LogProbs | undefined;
4348

49+
/**
50+
Additional provider-specific metadata. They are passed through
51+
from the provider to the AI SDK and enable provider-specific
52+
results that can be fully encapsulated in the provider.
53+
*/
54+
readonly experimental_providerMetadata: ProviderMetadata | undefined;
55+
4456
/**
4557
Converts the object to a JSON response.
4658
The response will have a status code of 200 and a content type of `application/json; charset=utf-8`.

0 commit comments

Comments
 (0)
Please sign in to comment.