Skip to content

Commit

Permalink
core[minor]: Add aload to document loader (#19936)
Browse files Browse the repository at this point in the history
Add aload to document loader
  • Loading branch information
eyurtsev authored and hinthornw committed Apr 26, 2024
1 parent deec556 commit 35102f3
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ def lazy_load(self) -> Iterator[Document]:
metadata = _build_metadata(soup, path)
yield Document(page_content=text, metadata=metadata)

def aload(self) -> List[Document]:
def aload(self) -> List[Document]: # type: ignore
"""Load text from the urls in web_path async into Documents."""

results = self.scrape_all(self.web_paths)
Expand Down
4 changes: 4 additions & 0 deletions libs/core/langchain_core/document_loaders/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ def load(self) -> List[Document]:
"""Load data into Document objects."""
return list(self.lazy_load())

async def aload(self) -> List[Document]:
"""Load data into Document objects."""
return [document async for document in self.alazy_load()]

def load_and_split(
self, text_splitter: Optional[TextSplitter] = None
) -> List[Document]:
Expand Down
1 change: 1 addition & 0 deletions libs/core/tests/unit_tests/document_loaders/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,4 @@ def lazy_load(self) -> Iterator[Document]:
docs = loader.load()
assert docs == [Document(page_content="foo"), Document(page_content="bar")]
assert docs == [doc async for doc in loader.alazy_load()]
assert docs == await loader.aload()

0 comments on commit 35102f3

Please sign in to comment.