community: Implement lazy_load() for GithubFileLoader (langchain-ai#1…

…8584)
thebhulawat · Mar 6, 2024 · 0a61954 · 0a61954
1 parent 7165b59
commit 0a61954
Showing 1 changed file with 2 additions and 6 deletions.
diff --git a/libs/community/langchain_community/document_loaders/github.py b/libs/community/langchain_community/document_loaders/github.py
@@ -217,9 +217,7 @@ def get_file_content_by_path(self, path: str) -> str:
 
         return ""
 
-    def load(self) -> List[Document]:
-        documents = []
-
+    def lazy_load(self) -> Iterator[Document]:
         files = self.get_file_paths()
         for file in files:
             content = self.get_file_content_by_path(file["path"])
@@ -232,6 +230,4 @@ def load(self) -> List[Document]:
                 "source": f"{self.github_api_url}/{self.repo}/{file['type']}/"
                 f"{self.branch}/{file['path']}",
             }
-            documents.append(Document(page_content=content, metadata=metadata))
-
-        return documents
+            yield Document(page_content=content, metadata=metadata)