Skip to content

Commit

Permalink
refactored, ut-s
Browse files Browse the repository at this point in the history
  • Loading branch information
@leo-gan
leo-gan committed Mar 11, 2024
1 parent 0d888a6 commit e2770d8
Show file tree
Hide file tree
Showing 13 changed files with 500 additions and 2,722 deletions.
72 changes: 27 additions & 45 deletions libs/community/langchain_community/document_transformers/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,48 +15,30 @@
Document
""" # noqa: E501

from langchain_community.document_transformers.beautiful_soup_transformer import (
BeautifulSoupTransformer,
)
from langchain_community.document_transformers.doctran_text_extract import (
DoctranPropertyExtractor,
)
from langchain_community.document_transformers.doctran_text_qa import (
DoctranQATransformer,
)
from langchain_community.document_transformers.doctran_text_translate import (
DoctranTextTranslator,
)
from langchain_community.document_transformers.embeddings_redundant_filter import (
EmbeddingsClusteringFilter,
EmbeddingsRedundantFilter,
get_stateful_documents,
)
from langchain_community.document_transformers.google_translate import (
GoogleTranslateTransformer,
)
from langchain_community.document_transformers.html2text import Html2TextTransformer
from langchain_community.document_transformers.long_context_reorder import (
LongContextReorder,
)
from langchain_community.document_transformers.nuclia_text_transform import (
NucliaTextTransformer,
)
from langchain_community.document_transformers.openai_functions import (
OpenAIMetadataTagger,
)

__all__ = [
"BeautifulSoupTransformer",
"DoctranQATransformer",
"DoctranTextTranslator",
"DoctranPropertyExtractor",
"EmbeddingsClusteringFilter",
"EmbeddingsRedundantFilter",
"GoogleTranslateTransformer",
"get_stateful_documents",
"LongContextReorder",
"NucliaTextTransformer",
"OpenAIMetadataTagger",
"Html2TextTransformer",
]
import importlib
from typing import Any

_module_lookup = {
"BeautifulSoupTransformer": "langchain_community.document_transformers.beautiful_soup_transformer", # noqa: E501
"DoctranPropertyExtractor": "langchain_community.document_transformers.doctran_text_extract", # noqa: E501
"DoctranQATransformer": "langchain_community.document_transformers.doctran_text_qa",
"DoctranTextTranslator": "langchain_community.document_transformers.doctran_text_translate", # noqa: E501
"EmbeddingsClusteringFilter": "langchain_community.document_transformers.embeddings_redundant_filter", # noqa: E501
"EmbeddingsRedundantFilter": "langchain_community.document_transformers.embeddings_redundant_filter", # noqa: E501
"GoogleTranslateTransformer": "langchain_community.document_transformers.google_translate", # noqa: E501
"Html2TextTransformer": "langchain_community.document_transformers.html2text",
"LongContextReorder": "langchain_community.document_transformers.long_context_reorder", # noqa: E501
"NucliaTextTransformer": "langchain_community.document_transformers.nuclia_text_transform", # noqa: E501
"OpenAIMetadataTagger": "langchain_community.document_transformers.openai_functions", # noqa: E501
"get_stateful_documents": "langchain_community.document_transformers.embeddings_redundant_filter", # noqa: E501
}


def __getattr__(name: str) -> Any:
if name in _module_lookup:
module = importlib.import_module(_module_lookup[name])
return getattr(module, name)
raise AttributeError(f"module {__name__} has no attribute {name}")


__all__ = list(_module_lookup.keys())
221 changes: 76 additions & 145 deletions libs/community/langchain_community/embeddings/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,156 +11,87 @@
"""


import importlib
import logging
from typing import Any

from langchain_community.embeddings.aleph_alpha import (
AlephAlphaAsymmetricSemanticEmbedding,
AlephAlphaSymmetricSemanticEmbedding,
)
from langchain_community.embeddings.awa import AwaEmbeddings
from langchain_community.embeddings.azure_openai import AzureOpenAIEmbeddings
from langchain_community.embeddings.baichuan import BaichuanTextEmbeddings
from langchain_community.embeddings.baidu_qianfan_endpoint import (
QianfanEmbeddingsEndpoint,
)
from langchain_community.embeddings.bedrock import BedrockEmbeddings
from langchain_community.embeddings.bookend import BookendEmbeddings
from langchain_community.embeddings.clarifai import ClarifaiEmbeddings
from langchain_community.embeddings.cohere import CohereEmbeddings
from langchain_community.embeddings.dashscope import DashScopeEmbeddings
from langchain_community.embeddings.databricks import DatabricksEmbeddings
from langchain_community.embeddings.deepinfra import DeepInfraEmbeddings
from langchain_community.embeddings.edenai import EdenAiEmbeddings
from langchain_community.embeddings.elasticsearch import ElasticsearchEmbeddings
from langchain_community.embeddings.embaas import EmbaasEmbeddings
from langchain_community.embeddings.ernie import ErnieEmbeddings
from langchain_community.embeddings.fake import (
DeterministicFakeEmbedding,
FakeEmbeddings,
)
from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
from langchain_community.embeddings.google_palm import GooglePalmEmbeddings
from langchain_community.embeddings.gpt4all import GPT4AllEmbeddings
from langchain_community.embeddings.gradient_ai import GradientEmbeddings
from langchain_community.embeddings.huggingface import (
HuggingFaceBgeEmbeddings,
HuggingFaceEmbeddings,
HuggingFaceInferenceAPIEmbeddings,
HuggingFaceInstructEmbeddings,
)
from langchain_community.embeddings.huggingface_hub import HuggingFaceHubEmbeddings
from langchain_community.embeddings.infinity import InfinityEmbeddings
from langchain_community.embeddings.infinity_local import InfinityEmbeddingsLocal
from langchain_community.embeddings.javelin_ai_gateway import JavelinAIGatewayEmbeddings
from langchain_community.embeddings.jina import JinaEmbeddings
from langchain_community.embeddings.johnsnowlabs import JohnSnowLabsEmbeddings
from langchain_community.embeddings.laser import LaserEmbeddings
from langchain_community.embeddings.llamacpp import LlamaCppEmbeddings
from langchain_community.embeddings.llamafile import LlamafileEmbeddings
from langchain_community.embeddings.llm_rails import LLMRailsEmbeddings
from langchain_community.embeddings.localai import LocalAIEmbeddings
from langchain_community.embeddings.minimax import MiniMaxEmbeddings
from langchain_community.embeddings.mlflow import (
MlflowCohereEmbeddings,
MlflowEmbeddings,
)
from langchain_community.embeddings.mlflow_gateway import MlflowAIGatewayEmbeddings
from langchain_community.embeddings.modelscope_hub import ModelScopeEmbeddings
from langchain_community.embeddings.mosaicml import MosaicMLInstructorEmbeddings
from langchain_community.embeddings.nemo import NeMoEmbeddings
from langchain_community.embeddings.nlpcloud import NLPCloudEmbeddings
from langchain_community.embeddings.oci_generative_ai import OCIGenAIEmbeddings
from langchain_community.embeddings.octoai_embeddings import OctoAIEmbeddings
from langchain_community.embeddings.ollama import OllamaEmbeddings
from langchain_community.embeddings.openai import OpenAIEmbeddings
from langchain_community.embeddings.optimum_intel import QuantizedBiEncoderEmbeddings
from langchain_community.embeddings.sagemaker_endpoint import (
SagemakerEndpointEmbeddings,
)
from langchain_community.embeddings.self_hosted import SelfHostedEmbeddings
from langchain_community.embeddings.self_hosted_hugging_face import (
SelfHostedHuggingFaceEmbeddings,
SelfHostedHuggingFaceInstructEmbeddings,
)
from langchain_community.embeddings.sentence_transformer import (
SentenceTransformerEmbeddings,
)
from langchain_community.embeddings.spacy_embeddings import SpacyEmbeddings
from langchain_community.embeddings.sparkllm import SparkLLMTextEmbeddings
from langchain_community.embeddings.tensorflow_hub import TensorflowHubEmbeddings
from langchain_community.embeddings.vertexai import VertexAIEmbeddings
from langchain_community.embeddings.volcengine import VolcanoEmbeddings
from langchain_community.embeddings.voyageai import VoyageEmbeddings
from langchain_community.embeddings.xinference import XinferenceEmbeddings
_module_lookup = {
"AlephAlphaAsymmetricSemanticEmbedding": "langchain_community.embeddings.aleph_alpha", # noqa: E501
"AlephAlphaSymmetricSemanticEmbedding": "langchain_community.embeddings.aleph_alpha", # noqa: E501
"AwaEmbeddings": "langchain_community.embeddings.awa",
"AzureOpenAIEmbeddings": "langchain_community.embeddings.azure_openai",
"BaichuanTextEmbeddings": "langchain_community.embeddings.baichuan",
"BedrockEmbeddings": "langchain_community.embeddings.bedrock",
"BookendEmbeddings": "langchain_community.embeddings.bookend",
"ClarifaiEmbeddings": "langchain_community.embeddings.clarifai",
"CohereEmbeddings": "langchain_community.embeddings.cohere",
"DashScopeEmbeddings": "langchain_community.embeddings.dashscope",
"DatabricksEmbeddings": "langchain_community.embeddings.databricks",
"DeepInfraEmbeddings": "langchain_community.embeddings.deepinfra",
"DeterministicFakeEmbedding": "langchain_community.embeddings.fake",
"EdenAiEmbeddings": "langchain_community.embeddings.edenai",
"ElasticsearchEmbeddings": "langchain_community.embeddings.elasticsearch",
"EmbaasEmbeddings": "langchain_community.embeddings.embaas",
"ErnieEmbeddings": "langchain_community.embeddings.ernie",
"FakeEmbeddings": "langchain_community.embeddings.fake",
"FastEmbedEmbeddings": "langchain_community.embeddings.fastembed",
"GPT4AllEmbeddings": "langchain_community.embeddings.gpt4all",
"GooglePalmEmbeddings": "langchain_community.embeddings.google_palm",
"GradientEmbeddings": "langchain_community.embeddings.gradient_ai",
"HuggingFaceBgeEmbeddings": "langchain_community.embeddings.huggingface",
"HuggingFaceEmbeddings": "langchain_community.embeddings.huggingface",
"HuggingFaceHubEmbeddings": "langchain_community.embeddings.huggingface_hub",
"HuggingFaceInferenceAPIEmbeddings": "langchain_community.embeddings.huggingface",
"HuggingFaceInstructEmbeddings": "langchain_community.embeddings.huggingface",
"InfinityEmbeddings": "langchain_community.embeddings.infinity",
"InfinityEmbeddingsLocal": "langchain_community.embeddings.infinity_local",
"JavelinAIGatewayEmbeddings": "langchain_community.embeddings.javelin_ai_gateway",
"JinaEmbeddings": "langchain_community.embeddings.jina",
"JohnSnowLabsEmbeddings": "langchain_community.embeddings.johnsnowlabs",
"LLMRailsEmbeddings": "langchain_community.embeddings.llm_rails",
"LaserEmbeddings": "langchain_community.embeddings.laser",
"LlamaCppEmbeddings": "langchain_community.embeddings.llamacpp",
"LlamafileEmbeddings": "langchain_community.embeddings.llamafile",
"LocalAIEmbeddings": "langchain_community.embeddings.localai",
"MiniMaxEmbeddings": "langchain_community.embeddings.minimax",
"MlflowAIGatewayEmbeddings": "langchain_community.embeddings.mlflow_gateway",
"MlflowCohereEmbeddings": "langchain_community.embeddings.mlflow",
"MlflowEmbeddings": "langchain_community.embeddings.mlflow",
"ModelScopeEmbeddings": "langchain_community.embeddings.modelscope_hub",
"MosaicMLInstructorEmbeddings": "langchain_community.embeddings.mosaicml",
"NLPCloudEmbeddings": "langchain_community.embeddings.nlpcloud",
"NeMoEmbeddings": "langchain_community.embeddings.nemo",
"OCIGenAIEmbeddings": "langchain_community.embeddings.oci_generative_ai",
"OctoAIEmbeddings": "langchain_community.embeddings.octoai_embeddings",
"OllamaEmbeddings": "langchain_community.embeddings.ollama",
"OpenAIEmbeddings": "langchain_community.embeddings.openai",
"QianfanEmbeddingsEndpoint": "langchain_community.embeddings.baidu_qianfan_endpoint", # noqa: E501
"QuantizedBiEncoderEmbeddings": "langchain_community.embeddings.optimum_intel",
"SagemakerEndpointEmbeddings": "langchain_community.embeddings.sagemaker_endpoint",
"SelfHostedEmbeddings": "langchain_community.embeddings.self_hosted",
"SelfHostedHuggingFaceEmbeddings": "langchain_community.embeddings.self_hosted_hugging_face", # noqa: E501
"SelfHostedHuggingFaceInstructEmbeddings": "langchain_community.embeddings.self_hosted_hugging_face", # noqa: E501
"SentenceTransformerEmbeddings": "langchain_community.embeddings.sentence_transformer", # noqa: E501
"SpacyEmbeddings": "langchain_community.embeddings.spacy_embeddings",
"SparkLLMTextEmbeddings": "langchain_community.embeddings.sparkllm",
"TensorflowHubEmbeddings": "langchain_community.embeddings.tensorflow_hub",
"VertexAIEmbeddings": "langchain_community.embeddings.vertexai",
"VolcanoEmbeddings": "langchain_community.embeddings.volcengine",
"VoyageEmbeddings": "langchain_community.embeddings.voyageai",
"XinferenceEmbeddings": "langchain_community.embeddings.xinference",
}


def __getattr__(name: str) -> Any:
if name in _module_lookup:
module = importlib.import_module(_module_lookup[name])
return getattr(module, name)
raise AttributeError(f"module {__name__} has no attribute {name}")

logger = logging.getLogger(__name__)

__all__ = [
"OpenAIEmbeddings",
"AzureOpenAIEmbeddings",
"BaichuanTextEmbeddings",
"ClarifaiEmbeddings",
"CohereEmbeddings",
"DatabricksEmbeddings",
"ElasticsearchEmbeddings",
"FastEmbedEmbeddings",
"HuggingFaceEmbeddings",
"HuggingFaceInferenceAPIEmbeddings",
"InfinityEmbeddings",
"InfinityEmbeddingsLocal",
"GradientEmbeddings",
"JinaEmbeddings",
"LaserEmbeddings",
"LlamaCppEmbeddings",
"LlamafileEmbeddings",
"LLMRailsEmbeddings",
"HuggingFaceHubEmbeddings",
"MlflowEmbeddings",
"MlflowCohereEmbeddings",
"MlflowAIGatewayEmbeddings",
"ModelScopeEmbeddings",
"TensorflowHubEmbeddings",
"SagemakerEndpointEmbeddings",
"HuggingFaceInstructEmbeddings",
"MosaicMLInstructorEmbeddings",
"SelfHostedEmbeddings",
"SelfHostedHuggingFaceEmbeddings",
"SelfHostedHuggingFaceInstructEmbeddings",
"FakeEmbeddings",
"DeterministicFakeEmbedding",
"AlephAlphaAsymmetricSemanticEmbedding",
"AlephAlphaSymmetricSemanticEmbedding",
"SentenceTransformerEmbeddings",
"GooglePalmEmbeddings",
"MiniMaxEmbeddings",
"VertexAIEmbeddings",
"BedrockEmbeddings",
"DeepInfraEmbeddings",
"EdenAiEmbeddings",
"DashScopeEmbeddings",
"EmbaasEmbeddings",
"OctoAIEmbeddings",
"SpacyEmbeddings",
"NLPCloudEmbeddings",
"GPT4AllEmbeddings",
"XinferenceEmbeddings",
"LocalAIEmbeddings",
"AwaEmbeddings",
"HuggingFaceBgeEmbeddings",
"ErnieEmbeddings",
"JavelinAIGatewayEmbeddings",
"OllamaEmbeddings",
"QianfanEmbeddingsEndpoint",
"JohnSnowLabsEmbeddings",
"VoyageEmbeddings",
"BookendEmbeddings",
"VolcanoEmbeddings",
"OCIGenAIEmbeddings",
"QuantizedBiEncoderEmbeddings",
"NeMoEmbeddings",
"SparkLLMTextEmbeddings",
]
__all__ = list(_module_lookup.keys())

logger = logging.getLogger(__name__)


# TODO: this is in here to maintain backwards compatibility
Expand Down
58 changes: 28 additions & 30 deletions libs/community/langchain_community/graphs/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,31 @@
"""**Graphs** provide a natural language interface to graph databases."""

from langchain_community.graphs.arangodb_graph import ArangoGraph
from langchain_community.graphs.falkordb_graph import FalkorDBGraph
from langchain_community.graphs.gremlin_graph import GremlinGraph
from langchain_community.graphs.hugegraph import HugeGraph
from langchain_community.graphs.kuzu_graph import KuzuGraph
from langchain_community.graphs.memgraph_graph import MemgraphGraph
from langchain_community.graphs.nebula_graph import NebulaGraph
from langchain_community.graphs.neo4j_graph import Neo4jGraph
from langchain_community.graphs.neptune_graph import NeptuneGraph
from langchain_community.graphs.neptune_rdf_graph import NeptuneRdfGraph
from langchain_community.graphs.networkx_graph import NetworkxEntityGraph
from langchain_community.graphs.ontotext_graphdb_graph import OntotextGraphDBGraph
from langchain_community.graphs.rdf_graph import RdfGraph
from langchain_community.graphs.tigergraph_graph import TigerGraph
import importlib
from typing import Any

__all__ = [
"MemgraphGraph",
"NetworkxEntityGraph",
"Neo4jGraph",
"NebulaGraph",
"NeptuneGraph",
"NeptuneRdfGraph",
"KuzuGraph",
"HugeGraph",
"RdfGraph",
"ArangoGraph",
"FalkorDBGraph",
"TigerGraph",
"OntotextGraphDBGraph",
"GremlinGraph",
]
_module_lookup = {
"ArangoGraph": "langchain_community.graphs.arangodb_graph",
"FalkorDBGraph": "langchain_community.graphs.falkordb_graph",
"GremlinGraph": "langchain_community.graphs.gremlin_graph",
"HugeGraph": "langchain_community.graphs.hugegraph",
"KuzuGraph": "langchain_community.graphs.kuzu_graph",
"MemgraphGraph": "langchain_community.graphs.memgraph_graph",
"NebulaGraph": "langchain_community.graphs.nebula_graph",
"Neo4jGraph": "langchain_community.graphs.neo4j_graph",
"NeptuneGraph": "langchain_community.graphs.neptune_graph",
"NeptuneRdfGraph": "langchain_community.graphs.neptune_rdf_graph",
"NetworkxEntityGraph": "langchain_community.graphs.networkx_graph",
"OntotextGraphDBGraph": "langchain_community.graphs.ontotext_graphdb_graph",
"RdfGraph": "langchain_community.graphs.rdf_graph",
"TigerGraph": "langchain_community.graphs.tigergraph_graph",
}


def __getattr__(name: str) -> Any:
if name in _module_lookup:
module = importlib.import_module(_module_lookup[name])
return getattr(module, name)
raise AttributeError(f"module {__name__} has no attribute {name}")


__all__ = list(_module_lookup.keys())

0 comments on commit e2770d8

Please sign in to comment.