feat (provider/openai): pdf input support (#5194)

Author: lgrammel

.changeset/odd-squids-unite.md

@@ -0,0 +1,7 @@
+---
+'@ai-sdk/provider': patch
+'@ai-sdk/openai': patch
+'ai': patch
+---
+
+feat (provider/openai): pdf input support

content/docs/02-foundations/03-prompts.mdx

@@ -201,8 +201,8 @@ const result = await generateText({
   Generative AI](/providers/ai-sdk-providers/google-generative-ai), [Google
   Vertex AI](/providers/ai-sdk-providers/google-vertex),
   [OpenAI](/providers/ai-sdk-providers/openai) (for `wav` and `mp3` audio with
-  `gpt-4o-audio-preview`), [Anthropic](/providers/ai-sdk-providers/anthropic)
-  (for `pdf`).
+  `gpt-4o-audio-preview`), [Anthropic](/providers/ai-sdk-providers/anthropic),
+  [OpenAI responses](/providers/ai-sdk-providers/openai) (for `pdf`).
 </Note>
 
 User messages can include file parts. A file can be one of the following:
@@ -237,6 +237,7 @@ const result = await generateText({
           type: 'file',
           mimeType: 'application/pdf',
           data: fs.readFileSync('./data/example.pdf'),
+          filename: 'example.pdf', // optional, not used by all providers
         },
       ],
     },

examples/ai-core/src/generate-text/openai-responses-pdf-url.ts

@@ -0,0 +1,32 @@
+import { openai } from '@ai-sdk/openai';
+import { generateText } from 'ai';
+import 'dotenv/config';
+
+async function main() {
+  const result = await generateText({
+    model: openai.responses('gpt-4o'),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'What is an embedding model according to this document?',
+          },
+          {
+            type: 'file',
+            data: new URL(
+              'https://github.com/vercel/ai/blob/main/examples/ai-core/data/ai.pdf?raw=true',
+            ),
+            mimeType: 'application/pdf',
+            filename: 'ai.pdf',
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result.text);
+}
+
+main().catch(console.error);

examples/ai-core/src/generate-text/openai-responses-pdf.ts

@@ -0,0 +1,31 @@
+import { openai } from '@ai-sdk/openai';
+import { generateText } from 'ai';
+import 'dotenv/config';
+import fs from 'node:fs';
+
+async function main() {
+  const result = await generateText({
+    model: openai.responses('gpt-4o'),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'What is an embedding model according to this document?',
+          },
+          {
+            type: 'file',
+            data: fs.readFileSync('./data/ai.pdf'),
+            mimeType: 'application/pdf',
+            // filename: 'ai.pdf',
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result.text);
+}
+
+main().catch(console.error);

packages/ai/core/prompt/content-part.ts

@@ -101,6 +101,11 @@ File data. Can either be:
    */
   data: DataContent | URL;
 
+  /**
+Optional filename of the file.
+   */
+  filename?: string;
+
   /**
 Mime type of the file.
    */
@@ -125,6 +130,7 @@ functionality that can be fully encapsulated in the provider.
 export const filePartSchema: z.ZodType<FilePart> = z.object({
   type: z.literal('file'),
   data: z.union([dataContentSchema, z.instanceof(URL)]),
+  filename: z.string().optional(),
   mimeType: z.string(),
   providerOptions: providerMetadataSchema.optional(),
   experimental_providerMetadata: providerMetadataSchema.optional(),

packages/ai/core/prompt/convert-to-language-model-prompt.test.ts

@@ -444,6 +444,87 @@ describe('convertToLanguageModelPrompt', () => {
           },
         ]);
       });
+
+      it('should handle file parts with filename', async () => {
+        const result = await convertToLanguageModelPrompt({
+          prompt: {
+            type: 'messages',
+            messages: [
+              {
+                role: 'user',
+                content: [
+                  {
+                    type: 'file',
+                    data: 'SGVsbG8sIFdvcmxkIQ==', // "Hello, World!" in base64
+                    mimeType: 'text/plain',
+                    filename: 'hello.txt',
+                  },
+                ],
+              },
+            ],
+          },
+          modelSupportsImageUrls: true,
+          modelSupportsUrl: undefined,
+        });
+
+        expect(result).toEqual([
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'file',
+                data: 'SGVsbG8sIFdvcmxkIQ==',
+                mimeType: 'text/plain',
+                filename: 'hello.txt',
+              },
+            ],
+          },
+        ]);
+      });
+
+      it('should preserve filename when downloading file from URL', async () => {
+        const result = await convertToLanguageModelPrompt({
+          prompt: {
+            type: 'messages',
+            messages: [
+              {
+                role: 'user',
+                content: [
+                  {
+                    type: 'file',
+                    data: new URL('https://example.com/document.pdf'),
+                    mimeType: 'application/pdf',
+                    filename: 'important-document.pdf',
+                  },
+                ],
+              },
+            ],
+          },
+          modelSupportsImageUrls: false,
+          modelSupportsUrl: () => false,
+          downloadImplementation: async ({ url }) => {
+            expect(url).toEqual(new URL('https://example.com/document.pdf'));
+            return {
+              data: new Uint8Array([0, 1, 2, 3]),
+              mimeType: 'application/pdf',
+            };
+          },
+        });
+
+        expect(result).toEqual([
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'file',
+                mimeType: 'application/pdf',
+                data: convertUint8ArrayToBase64(new Uint8Array([0, 1, 2, 3])),
+                filename: 'important-document.pdf',
+              },
+            ],
+          },
+        ]);
+      });
     });
 
     describe('provider metadata', async () => {

packages/ai/core/prompt/convert-to-language-model-prompt.ts

@@ -323,6 +323,7 @@ function convertPartToLanguageModelPart(
           normalizedData instanceof Uint8Array
             ? convertDataContentToBase64String(normalizedData)
             : normalizedData,
+        filename: part.filename,
         mimeType,
         providerMetadata:
           part.providerOptions ?? part.experimental_providerMetadata,

packages/openai/src/responses/convert-to-openai-responses-messages.test.ts

@@ -174,6 +174,115 @@ describe('convertToOpenAIResponsesMessages', () => {
         },
       ]);
     });
+
+    it('should convert messages with PDF file parts', async () => {
+      const base64Data = 'AQIDBAU='; // Base64 encoding of pdfData
+
+      const result = convertToOpenAIResponsesMessages({
+        prompt: [
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'file',
+                mimeType: 'application/pdf',
+                data: base64Data,
+                filename: 'document.pdf',
+              },
+            ],
+          },
+        ],
+        systemMessageMode: 'system',
+      });
+
+      expect(result.messages).toEqual([
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'input_file',
+              filename: 'document.pdf',
+              file_data: 'data:application/pdf;base64,AQIDBAU=',
+            },
+          ],
+        },
+      ]);
+    });
+
+    it('should use default filename for PDF file parts when not provided', async () => {
+      const base64Data = 'AQIDBAU=';
+
+      const result = convertToOpenAIResponsesMessages({
+        prompt: [
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'file',
+                mimeType: 'application/pdf',
+                data: base64Data,
+              },
+            ],
+          },
+        ],
+        systemMessageMode: 'system',
+      });
+
+      expect(result.messages).toEqual([
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'input_file',
+              filename: 'part-0.pdf',
+              file_data: 'data:application/pdf;base64,AQIDBAU=',
+            },
+          ],
+        },
+      ]);
+    });
+
+    it('should throw error for unsupported file types', async () => {
+      const base64Data = 'AQIDBAU=';
+
+      expect(() => {
+        convertToOpenAIResponsesMessages({
+          prompt: [
+            {
+              role: 'user',
+              content: [
+                {
+                  type: 'file',
+                  mimeType: 'text/plain',
+                  data: base64Data,
+                },
+              ],
+            },
+          ],
+          systemMessageMode: 'system',
+        });
+      }).toThrow('Only PDF files are supported in user messages');
+    });
+
+    it('should throw error for file URLs', async () => {
+      expect(() => {
+        convertToOpenAIResponsesMessages({
+          prompt: [
+            {
+              role: 'user',
+              content: [
+                {
+                  type: 'file',
+                  mimeType: 'application/pdf',
+                  data: new URL('https://example.com/document.pdf'),
+                },
+              ],
+            },
+          ],
+          systemMessageMode: 'system',
+        });
+      }).toThrow('File URLs in user messages');
+    });
   });
 
   describe('assistant messages', () => {

packages/openai/src/responses/convert-to-openai-responses-messages.ts

@@ -51,7 +51,7 @@ export function convertToOpenAIResponsesMessages({
       case 'user': {
         messages.push({
           role: 'user',
-          content: content.map(part => {
+          content: content.map((part, index) => {
             switch (part.type) {
               case 'text': {
                 return { type: 'input_text', text: part.text };
@@ -71,9 +71,28 @@ export function convertToOpenAIResponsesMessages({
                 };
               }
               case 'file': {
-                throw new UnsupportedFunctionalityError({
-                  functionality: 'Image content parts in user messages',
-                });
+                if (part.data instanceof URL) {
+                  // The AI SDK automatically downloads files for user file parts with URLs
+                  throw new UnsupportedFunctionalityError({
+                    functionality: 'File URLs in user messages',
+                  });
+                }
+
+                switch (part.mimeType) {
+                  case 'application/pdf': {
+                    return {
+                      type: 'input_file',
+                      filename: part.filename ?? `part-${index}.pdf`,
+                      file_data: `data:application/pdf;base64,${part.data}`,
+                    };
+                  }
+                  default: {
+                    throw new UnsupportedFunctionalityError({
+                      functionality:
+                        'Only PDF files are supported in user messages',
+                    });
+                  }
+                }
               }
             }
           }),

packages/openai/src/responses/openai-responses-api-types.ts

@@ -19,6 +19,7 @@ export type OpenAIResponsesUserMessage = {
   content: Array<
     | { type: 'input_text'; text: string }
     | { type: 'input_image'; image_url: string }
+    | { type: 'input_file'; filename: string; file_data: string }
   >;
 };
 

packages/provider/src/language-model/v1/language-model-v1-prompt.ts

@@ -143,6 +143,11 @@ File content part of a prompt. It contains a file.
 export interface LanguageModelV1FilePart {
   type: 'file';
 
+  /**
+   * Optional filename of the file.
+   */
+  filename?: string;
+
   /**
 File data as base64 encoded string or as a URL.
    */