Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 2840c6d

Browse files
RobertCraigieRobertCraigie
committedOct 1, 2024·
feat(api): support storing chat completions, enabling evals and model distillation in the dashboard
Learn more at http://openai.com/devday2024
1 parent e1bcfe8 commit 2840c6d

File tree

9 files changed

+164
-22
lines changed

9 files changed

+164
-22
lines changed
 

‎.stats.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
configured_endpoints: 68
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-8ad878332083dd506a478a293db78dc9e7b1b2124f2682e1d991225bc5bbcc3b.yml
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-52b934aee6468039ec7f4ce046a282b5fbce114afc708e70f17121df654f71da.yml

‎src/openai/resources/beta/chat/completions.py

+16
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,14 @@ def parse(
6969
logprobs: Optional[bool] | NotGiven = NOT_GIVEN,
7070
max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN,
7171
max_tokens: Optional[int] | NotGiven = NOT_GIVEN,
72+
metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
7273
n: Optional[int] | NotGiven = NOT_GIVEN,
7374
parallel_tool_calls: bool | NotGiven = NOT_GIVEN,
7475
presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
7576
seed: Optional[int] | NotGiven = NOT_GIVEN,
7677
service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
7778
stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
79+
store: Optional[bool] | NotGiven = NOT_GIVEN,
7880
stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
7981
temperature: Optional[float] | NotGiven = NOT_GIVEN,
8082
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
@@ -158,13 +160,15 @@ def parser(raw_completion: ChatCompletion) -> ParsedChatCompletion[ResponseForma
158160
"logprobs": logprobs,
159161
"max_completion_tokens": max_completion_tokens,
160162
"max_tokens": max_tokens,
163+
"metadata": metadata,
161164
"n": n,
162165
"parallel_tool_calls": parallel_tool_calls,
163166
"presence_penalty": presence_penalty,
164167
"response_format": _type_to_response_format(response_format),
165168
"seed": seed,
166169
"service_tier": service_tier,
167170
"stop": stop,
171+
"store": store,
168172
"stream": False,
169173
"stream_options": stream_options,
170174
"temperature": temperature,
@@ -202,12 +206,14 @@ def stream(
202206
logprobs: Optional[bool] | NotGiven = NOT_GIVEN,
203207
max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN,
204208
max_tokens: Optional[int] | NotGiven = NOT_GIVEN,
209+
metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
205210
n: Optional[int] | NotGiven = NOT_GIVEN,
206211
parallel_tool_calls: bool | NotGiven = NOT_GIVEN,
207212
presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
208213
seed: Optional[int] | NotGiven = NOT_GIVEN,
209214
service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
210215
stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
216+
store: Optional[bool] | NotGiven = NOT_GIVEN,
211217
stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
212218
temperature: Optional[float] | NotGiven = NOT_GIVEN,
213219
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
@@ -262,11 +268,13 @@ def stream(
262268
logprobs=logprobs,
263269
max_completion_tokens=max_completion_tokens,
264270
max_tokens=max_tokens,
271+
metadata=metadata,
265272
n=n,
266273
parallel_tool_calls=parallel_tool_calls,
267274
presence_penalty=presence_penalty,
268275
seed=seed,
269276
service_tier=service_tier,
277+
store=store,
270278
stop=stop,
271279
stream_options=stream_options,
272280
temperature=temperature,
@@ -320,12 +328,14 @@ async def parse(
320328
logprobs: Optional[bool] | NotGiven = NOT_GIVEN,
321329
max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN,
322330
max_tokens: Optional[int] | NotGiven = NOT_GIVEN,
331+
metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
323332
n: Optional[int] | NotGiven = NOT_GIVEN,
324333
parallel_tool_calls: bool | NotGiven = NOT_GIVEN,
325334
presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
326335
seed: Optional[int] | NotGiven = NOT_GIVEN,
327336
service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
328337
stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
338+
store: Optional[bool] | NotGiven = NOT_GIVEN,
329339
stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
330340
temperature: Optional[float] | NotGiven = NOT_GIVEN,
331341
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
@@ -409,12 +419,14 @@ def parser(raw_completion: ChatCompletion) -> ParsedChatCompletion[ResponseForma
409419
"logprobs": logprobs,
410420
"max_completion_tokens": max_completion_tokens,
411421
"max_tokens": max_tokens,
422+
"metadata": metadata,
412423
"n": n,
413424
"parallel_tool_calls": parallel_tool_calls,
414425
"presence_penalty": presence_penalty,
415426
"response_format": _type_to_response_format(response_format),
416427
"seed": seed,
417428
"service_tier": service_tier,
429+
"store": store,
418430
"stop": stop,
419431
"stream": False,
420432
"stream_options": stream_options,
@@ -453,12 +465,14 @@ def stream(
453465
logprobs: Optional[bool] | NotGiven = NOT_GIVEN,
454466
max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN,
455467
max_tokens: Optional[int] | NotGiven = NOT_GIVEN,
468+
metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
456469
n: Optional[int] | NotGiven = NOT_GIVEN,
457470
parallel_tool_calls: bool | NotGiven = NOT_GIVEN,
458471
presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
459472
seed: Optional[int] | NotGiven = NOT_GIVEN,
460473
service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
461474
stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
475+
store: Optional[bool] | NotGiven = NOT_GIVEN,
462476
stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
463477
temperature: Optional[float] | NotGiven = NOT_GIVEN,
464478
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
@@ -514,12 +528,14 @@ def stream(
514528
logprobs=logprobs,
515529
max_completion_tokens=max_completion_tokens,
516530
max_tokens=max_tokens,
531+
metadata=metadata,
517532
n=n,
518533
parallel_tool_calls=parallel_tool_calls,
519534
presence_penalty=presence_penalty,
520535
seed=seed,
521536
service_tier=service_tier,
522537
stop=stop,
538+
store=store,
523539
stream_options=stream_options,
524540
temperature=temperature,
525541
tool_choice=tool_choice,

‎src/openai/resources/chat/completions.py

+92-12
Large diffs are not rendered by default.

‎src/openai/types/chat/completion_create_params.py

+17-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,11 @@ class CompletionCreateParamsBase(TypedDict, total=False):
3030
messages: Required[Iterable[ChatCompletionMessageParam]]
3131
"""A list of messages comprising the conversation so far.
3232
33-
[Example Python code](https://cookbook.openai.com/examples/how_to_format_inputs_to_chatgpt_models).
33+
Depending on the [model](https://platform.openai.com/docs/models) you use,
34+
different message types (modalities) are supported, like
35+
[text](https://platform.openai.com/docs/guides/text-generation),
36+
[images](https://platform.openai.com/docs/guides/vision), and
37+
[audio](https://platform.openai.com/docs/guides/audio).
3438
"""
3539

3640
model: Required[Union[str, ChatModel]]
@@ -105,6 +109,12 @@ class CompletionCreateParamsBase(TypedDict, total=False):
105109
[o1 series models](https://platform.openai.com/docs/guides/reasoning).
106110
"""
107111

112+
metadata: Optional[Dict[str, str]]
113+
"""
114+
Developer-defined tags and values used for filtering completions in the
115+
[dashboard](https://platform.openai.com/completions).
116+
"""
117+
108118
n: Optional[int]
109119
"""How many chat completion choices to generate for each input message.
110120
@@ -183,6 +193,12 @@ class CompletionCreateParamsBase(TypedDict, total=False):
183193
stop: Union[Optional[str], List[str]]
184194
"""Up to 4 sequences where the API will stop generating further tokens."""
185195

196+
store: Optional[bool]
197+
"""
198+
Whether or not to store the output of this completion request for traffic
199+
logging in the [dashboard](https://platform.openai.com/completions).
200+
"""
201+
186202
stream_options: Optional[ChatCompletionStreamOptionsParam]
187203
"""Options for streaming response. Only set this when you set `stream: true`."""
188204

‎src/openai/types/chat_model.py

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"gpt-4o",
1313
"gpt-4o-2024-08-06",
1414
"gpt-4o-2024-05-13",
15+
"gpt-4o-realtime-preview-2024-10-01",
1516
"chatgpt-4o-latest",
1617
"gpt-4o-mini",
1718
"gpt-4o-mini-2024-07-18",

‎src/openai/types/completion_usage.py

+15-1
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,25 @@
44

55
from .._models import BaseModel
66

7-
__all__ = ["CompletionUsage", "CompletionTokensDetails"]
7+
__all__ = ["CompletionUsage", "CompletionTokensDetails", "PromptTokensDetails"]
88

99

1010
class CompletionTokensDetails(BaseModel):
11+
audio_tokens: Optional[int] = None
12+
"""Audio input tokens generated by the model."""
13+
1114
reasoning_tokens: Optional[int] = None
1215
"""Tokens generated by the model for reasoning."""
1316

1417

18+
class PromptTokensDetails(BaseModel):
19+
audio_tokens: Optional[int] = None
20+
"""Audio input tokens present in the prompt."""
21+
22+
cached_tokens: Optional[int] = None
23+
"""Cached tokens present in the prompt."""
24+
25+
1526
class CompletionUsage(BaseModel):
1627
completion_tokens: int
1728
"""Number of tokens in the generated completion."""
@@ -24,3 +35,6 @@ class CompletionUsage(BaseModel):
2435

2536
completion_tokens_details: Optional[CompletionTokensDetails] = None
2637
"""Breakdown of tokens used in a completion."""
38+
39+
prompt_tokens_details: Optional[PromptTokensDetails] = None
40+
"""Breakdown of tokens used in the prompt."""

‎tests/api_resources/chat/test_completions.py

+8
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,15 @@ def test_method_create_with_all_params_overload_1(self, client: OpenAI) -> None:
5757
logprobs=True,
5858
max_completion_tokens=0,
5959
max_tokens=0,
60+
metadata={"foo": "string"},
6061
n=1,
6162
parallel_tool_calls=True,
6263
presence_penalty=-2,
6364
response_format={"type": "text"},
6465
seed=-9007199254740991,
6566
service_tier="auto",
6667
stop="string",
68+
store=True,
6769
stream=False,
6870
stream_options={"include_usage": True},
6971
temperature=1,
@@ -178,13 +180,15 @@ def test_method_create_with_all_params_overload_2(self, client: OpenAI) -> None:
178180
logprobs=True,
179181
max_completion_tokens=0,
180182
max_tokens=0,
183+
metadata={"foo": "string"},
181184
n=1,
182185
parallel_tool_calls=True,
183186
presence_penalty=-2,
184187
response_format={"type": "text"},
185188
seed=-9007199254740991,
186189
service_tier="auto",
187190
stop="string",
191+
store=True,
188192
stream_options={"include_usage": True},
189193
temperature=1,
190194
tool_choice="none",
@@ -318,13 +322,15 @@ async def test_method_create_with_all_params_overload_1(self, async_client: Asyn
318322
logprobs=True,
319323
max_completion_tokens=0,
320324
max_tokens=0,
325+
metadata={"foo": "string"},
321326
n=1,
322327
parallel_tool_calls=True,
323328
presence_penalty=-2,
324329
response_format={"type": "text"},
325330
seed=-9007199254740991,
326331
service_tier="auto",
327332
stop="string",
333+
store=True,
328334
stream=False,
329335
stream_options={"include_usage": True},
330336
temperature=1,
@@ -439,13 +445,15 @@ async def test_method_create_with_all_params_overload_2(self, async_client: Asyn
439445
logprobs=True,
440446
max_completion_tokens=0,
441447
max_tokens=0,
448+
metadata={"foo": "string"},
442449
n=1,
443450
parallel_tool_calls=True,
444451
presence_penalty=-2,
445452
response_format={"type": "text"},
446453
seed=-9007199254740991,
447454
service_tier="auto",
448455
stop="string",
456+
store=True,
449457
stream_options={"include_usage": True},
450458
temperature=1,
451459
tool_choice="none",

‎tests/lib/chat/test_completions.py

+12-6
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,9 @@ def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pyte
7676
system_fingerprint='fp_b40fb1c6fb',
7777
usage=CompletionUsage(
7878
completion_tokens=37,
79-
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
79+
completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
8080
prompt_tokens=14,
81+
prompt_tokens_details=None,
8182
total_tokens=51
8283
)
8384
)
@@ -136,8 +137,9 @@ class Location(BaseModel):
136137
system_fingerprint='fp_5050236cbd',
137138
usage=CompletionUsage(
138139
completion_tokens=14,
139-
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
140+
completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
140141
prompt_tokens=79,
142+
prompt_tokens_details=None,
141143
total_tokens=93
142144
)
143145
)
@@ -198,8 +200,9 @@ class Location(BaseModel):
198200
system_fingerprint='fp_b40fb1c6fb',
199201
usage=CompletionUsage(
200202
completion_tokens=14,
201-
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
203+
completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
202204
prompt_tokens=88,
205+
prompt_tokens_details=None,
203206
total_tokens=102
204207
)
205208
)
@@ -385,8 +388,9 @@ class CalendarEvent:
385388
system_fingerprint='fp_7568d46099',
386389
usage=CompletionUsage(
387390
completion_tokens=17,
388-
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
391+
completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
389392
prompt_tokens=92,
393+
prompt_tokens_details=None,
390394
total_tokens=109
391395
)
392396
)
@@ -829,8 +833,9 @@ class Location(BaseModel):
829833
system_fingerprint='fp_5050236cbd',
830834
usage=CompletionUsage(
831835
completion_tokens=14,
832-
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
836+
completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
833837
prompt_tokens=79,
838+
prompt_tokens_details=None,
834839
total_tokens=93
835840
)
836841
)
@@ -897,8 +902,9 @@ class Location(BaseModel):
897902
system_fingerprint='fp_5050236cbd',
898903
usage=CompletionUsage(
899904
completion_tokens=14,
900-
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
905+
completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
901906
prompt_tokens=79,
907+
prompt_tokens_details=None,
902908
total_tokens=93
903909
)
904910
)

‎tests/lib/chat/test_completions_streaming.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,9 @@ def on_event(stream: ChatCompletionStream[Location], event: ChatCompletionStream
155155
system_fingerprint='fp_5050236cbd',
156156
usage=CompletionUsage(
157157
completion_tokens=14,
158-
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
158+
completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
159159
prompt_tokens=79,
160+
prompt_tokens_details=None,
160161
total_tokens=93
161162
)
162163
)

0 commit comments

Comments
 (0)
Please sign in to comment.