Skip to content

Commit d0c4659

Browse files
lgrammellgrammel
authoredMar 20, 2025··
feat (provider/groq): reasoning format support (#5273)
·
@ai-sdk/xai@1.2.1ai@4.1.64
1 parent 7e6f900 commit d0c4659

File tree

12 files changed

+206
-54
lines changed

12 files changed

+206
-54
lines changed
 

‎.changeset/perfect-frogs-notice.md

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@ai-sdk/provider-utils': patch
3+
'@ai-sdk/google': patch
4+
'@ai-sdk/groq': patch
5+
---
6+
7+
feat (provider-utils): parseProviderOptions function

‎.changeset/selfish-berries-think.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@ai-sdk/groq': patch
3+
---
4+
5+
fix (provider/groq): skip empty text deltas

‎.changeset/silent-bees-smell.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@ai-sdk/groq': patch
3+
---
4+
5+
feat (provider/groq): reasoning format support

‎content/providers/01-ai-sdk-providers/09-groq.mdx

+20-11
Original file line numberDiff line numberDiff line change
@@ -77,20 +77,24 @@ const model = groq('gemma2-9b-it');
7777

7878
### Reasoning Models
7979

80-
Groq exposes the thinking of `deepseek-r1-distill-llama-70b` in the generated text using the `<think>` tag.
81-
You can use the `extractReasoningMiddleware` to extract this reasoning and expose it as a `reasoning` property on the result:
80+
Groq offers several reasoning models such as `qwen-qwq-32b` and `deepseek-r1-distill-llama-70b`.
81+
You can configure how the reasoning is exposed in the generated text by using the `reasoningFormat` option.
82+
It supports the options `parsed`, `hidden`, and `raw`.
8283

8384
```ts
8485
import { groq } from '@ai-sdk/groq';
85-
import { wrapLanguageModel, extractReasoningMiddleware } from 'ai';
86+
import { generateText } from 'ai';
8687

87-
const enhancedModel = wrapLanguageModel({
88-
model: groq('deepseek-r1-distill-llama-70b'),
89-
middleware: extractReasoningMiddleware({ tagName: 'think' }),
88+
const result = await generateText({
89+
model: groq('qwen-qwq-32b'),
90+
providerOptions: {
91+
groq: { reasoningFormat: 'parsed' },
92+
},
93+
prompt: 'How many "r"s are in the word "strawberry"?',
9094
});
9195
```
9296

93-
You can then use that enhanced model in functions like `generateText` and `streamText`.
97+
<Note>Only Groq reasoning models support the `reasoningFormat` option.</Note>
9498

9599
### Example
96100

@@ -110,13 +114,18 @@ const { text } = await generateText({
110114

111115
| Model | Image Input | Object Generation | Tool Usage | Tool Streaming |
112116
| ------------------------------- | ------------------- | ------------------- | ------------------- | ------------------- |
113-
| `deepseek-r1-distill-llama-70b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
117+
| `gemma2-9b-it` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
114118
| `llama-3.3-70b-versatile` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
115119
| `llama-3.1-8b-instant` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
116-
| `mistral-saba-24b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
117-
| `qwen-qwq-32b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
120+
| `llama-guard-3-8b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
121+
| `llama3-70b-8192` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
122+
| `llama3-8b-8192` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
118123
| `mixtral-8x7b-32768` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
119-
| `gemma2-9b-it` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
124+
| `qwen-qwq-32b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
125+
| `mistral-saba-24b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
126+
| `qwen-2.5-32b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
127+
| `deepseek-r1-distill-qwen-32b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
128+
| `deepseek-r1-distill-llama-70b` | <Cross size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |
120129

121130
<Note>
122131
The table above lists popular models. Please see the [Groq

‎examples/ai-core/src/generate-text/groq-reasoning.ts

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import { groq } from '@ai-sdk/groq';
2+
import { generateText } from 'ai';
3+
import 'dotenv/config';
4+
5+
async function main() {
6+
const result = await generateText({
7+
model: groq('qwen-qwq-32b'),
8+
providerOptions: {
9+
groq: { reasoningFormat: 'parsed' },
10+
},
11+
prompt: 'How many "r"s are in the word "strawberry"?',
12+
});
13+
14+
console.log('Reasoning:');
15+
console.log(result.reasoning);
16+
console.log();
17+
18+
console.log('Text:');
19+
console.log(result.text);
20+
console.log();
21+
22+
console.log('Token usage:', result.usage);
23+
console.log('Finish reason:', result.finishReason);
24+
}
25+
26+
main().catch(console.error);

‎examples/ai-core/src/stream-text/groq-reasoning-fullstream.ts

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
import { groq } from '@ai-sdk/groq';
2-
import { extractReasoningMiddleware, streamText, wrapLanguageModel } from 'ai';
2+
import { streamText } from 'ai';
33
import 'dotenv/config';
44

55
async function main() {
66
const result = streamText({
7-
model: wrapLanguageModel({
8-
model: groq('deepseek-r1-distill-llama-70b'),
9-
middleware: extractReasoningMiddleware({ tagName: 'think' }),
10-
}),
7+
model: groq('deepseek-r1-distill-llama-70b'),
8+
providerOptions: {
9+
groq: { reasoningFormat: 'parsed' },
10+
},
1111
prompt: 'How many "r"s are in the word "strawberry"?',
1212
});
1313

‎packages/google/src/google-generative-ai-language-model.ts

+8-26
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import {
2-
InvalidArgumentError,
32
LanguageModelV1,
43
LanguageModelV1CallWarning,
54
LanguageModelV1FinishReason,
@@ -14,9 +13,9 @@ import {
1413
combineHeaders,
1514
createEventSourceResponseHandler,
1615
createJsonResponseHandler,
16+
parseProviderOptions,
1717
postJsonToApi,
1818
resolve,
19-
safeValidateTypes,
2019
} from '@ai-sdk/provider-utils';
2120
import { z } from 'zod';
2221
import { convertJSONSchemaToOpenAPISchema } from './convert-json-schema-to-openapi-schema';
@@ -86,22 +85,13 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV1 {
8685

8786
const warnings: LanguageModelV1CallWarning[] = [];
8887

89-
// parse and validate provider options:
90-
const parsedProviderOptions =
91-
providerMetadata != null
92-
? safeValidateTypes({
93-
value: providerMetadata,
94-
schema: providerOptionsSchema,
95-
})
96-
: { success: true as const, value: undefined };
97-
if (!parsedProviderOptions.success) {
98-
throw new InvalidArgumentError({
99-
argument: 'providerOptions',
100-
message: 'invalid provider options',
101-
cause: parsedProviderOptions.error,
102-
});
103-
}
104-
const googleOptions = parsedProviderOptions.value?.google;
88+
const googleOptions = parseProviderOptions({
89+
provider: 'google',
90+
providerOptions: providerMetadata,
91+
schema: z.object({
92+
responseModalities: z.array(z.enum(['TEXT', 'IMAGE'])).nullish(),
93+
}),
94+
});
10595

10696
const generationConfig = {
10797
// standardized settings:
@@ -633,11 +623,3 @@ const chunkSchema = z.object({
633623
})
634624
.nullish(),
635625
});
636-
637-
const providerOptionsSchema = z.object({
638-
google: z
639-
.object({
640-
responseModalities: z.array(z.enum(['TEXT', 'IMAGE'])).nullish(),
641-
})
642-
.nullish(),
643-
});

‎packages/groq/src/groq-chat-language-model.test.ts

+65-5
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ describe('doGenerate', () => {
4646

4747
function prepareJsonResponse({
4848
content = '',
49+
reasoning,
4950
tool_calls,
5051
function_call,
5152
usage = {
@@ -59,6 +60,7 @@ describe('doGenerate', () => {
5960
model = 'gemma2-9b-it',
6061
}: {
6162
content?: string;
63+
reasoning?: string;
6264
tool_calls?: Array<{
6365
id: string;
6466
type: 'function';
@@ -92,6 +94,7 @@ describe('doGenerate', () => {
9294
message: {
9395
role: 'assistant',
9496
content,
97+
reasoning,
9598
tool_calls,
9699
function_call,
97100
},
@@ -115,6 +118,20 @@ describe('doGenerate', () => {
115118
expect(text).toStrictEqual('Hello, World!');
116119
});
117120

121+
it('should extract reasoning', async () => {
122+
prepareJsonResponse({
123+
reasoning: 'This is a test reasoning',
124+
});
125+
126+
const { reasoning } = await model.doGenerate({
127+
inputFormat: 'prompt',
128+
mode: { type: 'regular' },
129+
prompt: TEST_PROMPT,
130+
});
131+
132+
expect(reasoning).toStrictEqual('This is a test reasoning');
133+
});
134+
118135
it('should extract usage', async () => {
119136
prepareJsonResponse({
120137
content: '',
@@ -249,13 +266,17 @@ describe('doGenerate', () => {
249266
inputFormat: 'prompt',
250267
mode: { type: 'regular' },
251268
prompt: TEST_PROMPT,
269+
providerMetadata: {
270+
groq: { reasoningFormat: 'hidden' },
271+
},
252272
});
253273

254274
expect(await server.getRequestBodyJson()).toStrictEqual({
255275
model: 'gemma2-9b-it',
256276
messages: [{ role: 'user', content: 'Hello' }],
257277
parallel_tool_calls: false,
258278
user: 'test-user-id',
279+
reasoning_format: 'hidden',
259280
});
260281
});
261282

@@ -489,7 +510,6 @@ describe('doStream', () => {
489510
modelId: 'gemma2-9b-it',
490511
timestamp: new Date('2023-12-15T16:17:00.000Z'),
491512
},
492-
{ type: 'text-delta', textDelta: '' },
493513
{ type: 'text-delta', textDelta: 'Hello' },
494514
{ type: 'text-delta', textDelta: ', ' },
495515
{ type: 'text-delta', textDelta: 'World!' },
@@ -501,6 +521,50 @@ describe('doStream', () => {
501521
]);
502522
});
503523

524+
it('should stream reasoning deltas', async () => {
525+
server.responseChunks = [
526+
`data: {"id":"chatcmpl-e7f8e220-656c-4455-a132-dacfc1370798","object":"chat.completion.chunk","created":1702657020,"model":"gemma2-9b-it",` +
527+
`"system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}\n\n`,
528+
`data: {"id":"chatcmpl-e7f8e220-656c-4455-a132-dacfc1370798","object":"chat.completion.chunk","created":1702657020,"model":"gemma2-9b-it",` +
529+
`"system_fingerprint":null,"choices":[{"index":1,"delta":{"reasoning":"I think,"},"finish_reason":null}]}\n\n`,
530+
`data: {"id":"chatcmpl-e7f8e220-656c-4455-a132-dacfc1370798","object":"chat.completion.chunk","created":1702657020,"model":"gemma2-9b-it",` +
531+
`"system_fingerprint":null,"choices":[{"index":1,"delta":{"reasoning":"therefore I am."},"finish_reason":null}]}\n\n`,
532+
`data: {"id":"chatcmpl-e7f8e220-656c-4455-a132-dacfc1370798","object":"chat.completion.chunk","created":1702657020,"model":"gemma2-9b-it",` +
533+
`"system_fingerprint":null,"choices":[{"index":1,"delta":{"content":"Hello"},"finish_reason":null}]}\n\n`,
534+
`data: {"id":"chatcmpl-e7f8e220-656c-4455-a132-dacfc1370798","object":"chat.completion.chunk","created":1702657020,"model":"gemma2-9b-it",` +
535+
`"system_fingerprint":null,"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`,
536+
`data: {"id":"chatcmpl-e7f8e220-656c-4455-a132-dacfc1370798","object":"chat.completion.chunk","created":1729171479,"model":"gemma2-9b-it",` +
537+
`"system_fingerprint":"fp_10c08bf97d","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],` +
538+
`"x_groq":{"id":"req_01jadadp0femyae9kav1gpkhe8","usage":{"queue_time":0.061348671,"prompt_tokens":18,"prompt_time":0.000211569,` +
539+
`"completion_tokens":439,"completion_time":0.798181818,"total_tokens":457,"total_time":0.798393387}}}\n\n`,
540+
'data: [DONE]\n\n',
541+
];
542+
543+
const { stream } = await model.doStream({
544+
inputFormat: 'prompt',
545+
mode: { type: 'regular' },
546+
prompt: TEST_PROMPT,
547+
});
548+
549+
// note: space moved to last chunk bc of trimming
550+
expect(await convertReadableStreamToArray(stream)).toStrictEqual([
551+
{
552+
type: 'response-metadata',
553+
id: 'chatcmpl-e7f8e220-656c-4455-a132-dacfc1370798',
554+
modelId: 'gemma2-9b-it',
555+
timestamp: new Date('2023-12-15T16:17:00.000Z'),
556+
},
557+
{ type: 'reasoning', textDelta: 'I think,' },
558+
{ type: 'reasoning', textDelta: 'therefore I am.' },
559+
{ type: 'text-delta', textDelta: 'Hello' },
560+
{
561+
type: 'finish',
562+
finishReason: 'stop',
563+
usage: { promptTokens: 18, completionTokens: 439 },
564+
},
565+
]);
566+
});
567+
504568
it('should stream tool deltas', async () => {
505569
server.responseChunks = [
506570
`data: {"id":"chatcmpl-e7f8e220-656c-4455-a132-dacfc1370798","object":"chat.completion.chunk","created":1711357598,"model":"gemma2-9b-it",` +
@@ -828,10 +892,6 @@ describe('doStream', () => {
828892
modelId: 'meta/llama-3.1-8b-instruct:fp8',
829893
timestamp: new Date('2024-12-02T17:57:21.000Z'),
830894
},
831-
{
832-
type: 'text-delta',
833-
textDelta: '',
834-
},
835895
{
836896
type: 'tool-call-delta',
837897
toolCallId: 'chatcmpl-tool-b3b307239370432d9910d4b79b4dbbaa',

‎packages/groq/src/groq-chat-language-model.ts

+24-3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
createJsonResponseHandler,
1515
generateId,
1616
isParsableJson,
17+
parseProviderOptions,
1718
postJsonToApi,
1819
} from '@ai-sdk/provider-utils';
1920
import { z } from 'zod';
@@ -74,6 +75,7 @@ export class GroqChatLanguageModel implements LanguageModelV1 {
7475
responseFormat,
7576
seed,
7677
stream,
78+
providerMetadata,
7779
}: Parameters<LanguageModelV1['doGenerate']>[0] & {
7880
stream: boolean;
7981
}) {
@@ -100,6 +102,14 @@ export class GroqChatLanguageModel implements LanguageModelV1 {
100102
});
101103
}
102104

105+
const groqOptions = parseProviderOptions({
106+
provider: 'groq',
107+
providerOptions: providerMetadata,
108+
schema: z.object({
109+
reasoningFormat: z.enum(['parsed', 'raw', 'hidden']).nullish(),
110+
}),
111+
});
112+
103113
const baseArgs = {
104114
// model id:
105115
model: this.modelId,
@@ -124,6 +134,9 @@ export class GroqChatLanguageModel implements LanguageModelV1 {
124134
? { type: 'json_object' }
125135
: undefined,
126136

137+
// provider options:
138+
reasoning_format: groqOptions?.reasoningFormat,
139+
127140
// messages:
128141
messages: convertToGroqChatMessages(prompt),
129142
};
@@ -214,6 +227,7 @@ export class GroqChatLanguageModel implements LanguageModelV1 {
214227

215228
return {
216229
text: choice.message.content ?? undefined,
230+
reasoning: choice.message.reasoning ?? undefined,
217231
toolCalls: choice.message.tool_calls?.map(toolCall => ({
218232
toolCallType: 'function',
219233
toolCallId: toolCall.id ?? generateId(),
@@ -332,7 +346,14 @@ export class GroqChatLanguageModel implements LanguageModelV1 {
332346

333347
const delta = choice.delta;
334348

335-
if (delta.content != null) {
349+
if (delta.reasoning != null && delta.reasoning.length > 0) {
350+
controller.enqueue({
351+
type: 'reasoning',
352+
textDelta: delta.reasoning,
353+
});
354+
}
355+
356+
if (delta.content != null && delta.content.length > 0) {
336357
controller.enqueue({
337358
type: 'text-delta',
338359
textDelta: delta.content,
@@ -479,8 +500,8 @@ const groqChatResponseSchema = z.object({
479500
choices: z.array(
480501
z.object({
481502
message: z.object({
482-
role: z.literal('assistant').nullish(),
483503
content: z.string().nullish(),
504+
reasoning: z.string().nullish(),
484505
tool_calls: z
485506
.array(
486507
z.object({
@@ -517,8 +538,8 @@ const groqChatChunkSchema = z.union([
517538
z.object({
518539
delta: z
519540
.object({
520-
role: z.enum(['assistant']).nullish(),
521541
content: z.string().nullish(),
542+
reasoning: z.string().nullish(),
522543
tool_calls: z
523544
.array(
524545
z.object({

‎packages/groq/src/groq-chat-settings.ts

+7-3
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,19 @@
11
// https://console.groq.com/docs/models
2-
// production models
32
export type GroqChatModelId =
4-
| 'deepseek-r1-distill-llama-70b'
3+
// production models
54
| 'gemma2-9b-it'
6-
| 'gemma-7b-it'
75
| 'llama-3.3-70b-versatile'
86
| 'llama-3.1-8b-instant'
97
| 'llama-guard-3-8b'
108
| 'llama3-70b-8192'
119
| 'llama3-8b-8192'
1210
| 'mixtral-8x7b-32768'
11+
// preview models (selection)
12+
| 'qwen-qwq-32b'
13+
| 'mistral-saba-24b'
14+
| 'qwen-2.5-32b'
15+
| 'deepseek-r1-distill-qwen-32b'
16+
| 'deepseek-r1-distill-llama-70b'
1317
| (string & {});
1418

1519
export interface GroqChatSettings {

‎packages/provider-utils/src/index.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@ export { loadOptionalSetting } from './load-optional-setting';
1212
export { loadSetting } from './load-setting';
1313
export * from './parse-json';
1414
export * from './post-to-api';
15-
export * from './resolve';
1615
export * from './remove-undefined-entries';
16+
export * from './resolve';
1717
export * from './response-handler';
1818
export * from './uint8-utils';
1919
export * from './validate-types';
2020
export * from './validator';
2121
export * from './without-trailing-slash';
2222

2323
export type { IDGenerator } from './generate-id';
24+
export { parseProviderOptions } from './parse-provider-options';
2425
export type { CoreToolCall, ToolCall } from './types/tool-call';
2526
export type { CoreToolResult, ToolResult } from './types/tool-result';

‎packages/provider-utils/src/parse-provider-options.ts

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import { InvalidArgumentError } from '@ai-sdk/provider';
2+
import { safeValidateTypes } from './validate-types';
3+
import { z } from 'zod';
4+
5+
export function parseProviderOptions<T>({
6+
provider,
7+
providerOptions,
8+
schema,
9+
}: {
10+
provider: string;
11+
providerOptions: Record<string, unknown> | undefined;
12+
schema: z.ZodSchema<T>;
13+
}): T | undefined {
14+
if (providerOptions?.[provider] == null) {
15+
return undefined;
16+
}
17+
18+
const parsedProviderOptions = safeValidateTypes({
19+
value: providerOptions[provider],
20+
schema,
21+
});
22+
23+
if (!parsedProviderOptions.success) {
24+
throw new InvalidArgumentError({
25+
argument: 'providerOptions',
26+
message: `invalid ${provider} provider options`,
27+
cause: parsedProviderOptions.error,
28+
});
29+
}
30+
31+
return parsedProviderOptions.value;
32+
}

0 commit comments

Comments
 (0)
Please sign in to comment.