-
Notifications
You must be signed in to change notification settings - Fork 13.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
anthropic[patch]: multimodal #18517
anthropic[patch]: multimodal #18517
Changes from all commits
8c87726
9777ae9
85cf3e4
eefe029
e3ac592
46135bc
755c418
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -1,5 +1,6 @@ | ||||||
import os | ||||||
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Tuple | ||||||
import re | ||||||
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union | ||||||
|
||||||
import anthropic | ||||||
from langchain_core._api.deprecation import deprecated | ||||||
|
@@ -24,6 +25,33 @@ | |||||
_message_type_lookups = {"human": "user", "ai": "assistant"} | ||||||
|
||||||
|
||||||
def _format_image(image_url: str) -> Dict: | ||||||
""" | ||||||
Formats an image of format data:image/jpeg;base64,{b64_string} | ||||||
to a dict for anthropic api | ||||||
|
||||||
{ | ||||||
"type": "base64", | ||||||
"media_type": "image/jpeg", | ||||||
"data": "/9j/4AAQSkZJRg...", | ||||||
} | ||||||
|
||||||
And throws an error if it's not a b64 image | ||||||
""" | ||||||
regex = r"^data:(?P<media_type>image/.+);base64,(?P<data>.+)$" | ||||||
match = re.match(regex, image_url) | ||||||
if match is None: | ||||||
raise ValueError( | ||||||
"Anthropic only supports base64-encoded images currently." | ||||||
" Example: data:image/png;base64,'/9j/4AAQSk'..." | ||||||
) | ||||||
return { | ||||||
"type": "base64", | ||||||
"media_type": match.group("media_type"), | ||||||
"data": match.group("data"), | ||||||
} | ||||||
|
||||||
|
||||||
def _format_messages(messages: List[BaseMessage]) -> Tuple[Optional[str], List[Dict]]: | ||||||
"""Format messages for anthropic.""" | ||||||
|
||||||
|
@@ -36,22 +64,66 @@ def _format_messages(messages: List[BaseMessage]) -> Tuple[Optional[str], List[D | |||||
for m in messages | ||||||
] | ||||||
""" | ||||||
system = None | ||||||
formatted_messages = [] | ||||||
system: Optional[str] = None | ||||||
formatted_messages: List[Dict] = [] | ||||||
for i, message in enumerate(messages): | ||||||
if not isinstance(message.content, str): | ||||||
raise ValueError("Anthropic Messages API only supports text generation.") | ||||||
if message.type == "system": | ||||||
if i != 0: | ||||||
raise ValueError("System message must be at beginning of message list.") | ||||||
if not isinstance(message.content, str): | ||||||
raise ValueError( | ||||||
"System message must be a string, " | ||||||
f"instead was: {type(message.content)}" | ||||||
) | ||||||
system = message.content | ||||||
continue | ||||||
|
||||||
role = _message_type_lookups[message.type] | ||||||
content: Union[str, List[Dict]] | ||||||
|
||||||
if not isinstance(message.content, str): | ||||||
# parse as dict | ||||||
assert isinstance( | ||||||
message.content, list | ||||||
), "Anthropic message content must be str or list of dicts" | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why not accept list of str There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it does? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
|
||||||
# populate content | ||||||
content = [] | ||||||
for item in message.content: | ||||||
if isinstance(item, str): | ||||||
content.append( | ||||||
{ | ||||||
"type": "text", | ||||||
"text": item, | ||||||
} | ||||||
) | ||||||
elif isinstance(item, dict): | ||||||
if "type" not in item: | ||||||
raise ValueError("Dict content item must have a type key") | ||||||
if item["type"] == "image_url": | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should we support "image_url" or "image"? as an Anthropic user i might expect "image" support There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Supports both. |
||||||
# convert format | ||||||
source = _format_image(item["image_url"]["url"]) | ||||||
content.append( | ||||||
{ | ||||||
"type": "image", | ||||||
"source": source, | ||||||
} | ||||||
) | ||||||
else: | ||||||
content.append(item) | ||||||
else: | ||||||
raise ValueError( | ||||||
f"Content items must be str or dict, instead was: {type(item)}" | ||||||
) | ||||||
else: | ||||||
formatted_messages.append( | ||||||
{ | ||||||
"role": _message_type_lookups[message.type], | ||||||
"content": message.content, | ||||||
} | ||||||
) | ||||||
content = message.content | ||||||
|
||||||
formatted_messages.append( | ||||||
{ | ||||||
"role": role, | ||||||
"content": content, | ||||||
} | ||||||
) | ||||||
return system, formatted_messages | ||||||
|
||||||
|
||||||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
FYI @dqbd the default anthropic format won't render properly in langsmith if implemented this way
Think fine here