From a3867a576a6cd273809d1080a31bd6e0a4480f5c Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 1 Feb 2024 09:08:46 +0530 Subject: [PATCH 01/41] feat: use new openai models in chat and query enhancer --- src/wandbot/chat/config.py | 2 +- src/wandbot/chat/query_enhancer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wandbot/chat/config.py b/src/wandbot/chat/config.py index 7be15d3..4e1b467 100644 --- a/src/wandbot/chat/config.py +++ b/src/wandbot/chat/config.py @@ -19,7 +19,7 @@ class ChatConfig(BaseSettings): - chat_model_name: str = "gpt-4-1106-preview" + chat_model_name: str = "gpt-4-0125-preview" max_retries: int = 2 fallback_model_name: str = "gpt-3.5-turbo-1106" max_fallback_retries: int = 6 diff --git a/src/wandbot/chat/query_enhancer.py b/src/wandbot/chat/query_enhancer.py index d3d2db2..63e80b7 100644 --- a/src/wandbot/chat/query_enhancer.py +++ b/src/wandbot/chat/query_enhancer.py @@ -290,7 +290,7 @@ class QueryHandlerConfig(BaseSettings): validation_alias="default_query_clf_model", ) fallback_query_clf_model: str = Field( - "gpt-4-1106-preview", + "gpt-3.5-turbo-1106", description="The name of the fallback model to use for query classification", ) tokenizer: str = Field( From 7d6a187e5ff707986f4bcae97b79c3b86766f2b1 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 1 Feb 2024 11:18:02 +0530 Subject: [PATCH 02/41] feat: use new models and embeddings model --- pyproject.toml | 3 +- src/wandbot/chat/chat.py | 15 ++++--- src/wandbot/chat/config.py | 5 +-- src/wandbot/chat/retriever.py | 55 +++++++++++++----------- src/wandbot/ingestion/config.py | 6 +-- src/wandbot/ingestion/preprocess_data.py | 25 +++++++++-- src/wandbot/ingestion/vectorstores.py | 8 ++-- src/wandbot/utils.py | 29 ++++++------- 8 files changed, 83 insertions(+), 63 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7c335d..f40dd2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ slack-sdk = "^3.21.3" discord = "^2.3.2" markdown = "^3.5.1" fastapi = "^0.104.1" -llama-index = "0.9.35" +llama-index = "^0.9.40" tree-sitter-languages = "^1.7.1" cohere = "^4.32" markdownify = "^0.11.6" @@ -46,7 +46,6 @@ instructor = "^0.4.5" langchain-community = "^0.0.11" langchain = "^0.1.0" langchain-openai = "^0.0.2" -# setuptools = "69.0.2" # needed to install on replit (added 2024-23-01) [tool.poetry.dev-dependencies] #fasttext = {git = "https://github.com/cfculhane/fastText"} # FastText doesn't come with pybind11 and we need to use this workaround. diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 3acea63..e183abc 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -28,6 +28,7 @@ import json from typing import Any, Dict, List, Optional, Tuple +import wandb from llama_index import ServiceContext from llama_index.callbacks import ( CallbackManager, @@ -44,9 +45,6 @@ from llama_index.postprocessor.types import BaseNodePostprocessor from llama_index.schema import MetadataMode, NodeWithScore, QueryBundle from llama_index.tools import ToolOutput -from weave.monitoring import StreamTable - -import wandb from wandbot.chat.config import ChatConfig from wandbot.chat.prompts import load_chat_prompt, partial_format from wandbot.chat.query_enhancer import CompleteQuery, QueryHandler @@ -58,6 +56,7 @@ ) from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.utils import Timer, get_logger, load_service_context +from weave.monitoring import StreamTable logger = get_logger(__name__) @@ -255,14 +254,16 @@ def __init__(self, config: ChatConfig): llm=self.config.chat_model_name, temperature=self.config.chat_temperature, max_retries=self.config.max_retries, - embeddings_cache=str(self.config.embeddings_cache), + embeddings_model=self.config.embeddings_model, + embeddings_size=self.config.embeddings_dim, callback_manager=self.callback_manager, ) self.fallback_service_context = load_service_context( llm=self.config.fallback_model_name, temperature=self.config.chat_temperature, max_retries=self.config.max_fallback_retries, - embeddings_cache=str(self.config.embeddings_cache), + embeddings_model=self.config.embeddings_model, + embeddings_size=self.config.embeddings_dim, callback_manager=self.callback_manager, ) @@ -279,8 +280,8 @@ def _load_chat_engine( service_context: ServiceContext, query_intent: str = "\n", language: str = "en", - initial_k: int = 15, - top_k: int = 5, + initial_k: int = 10, + top_k: int = 10, ) -> WandbContextChatEngine: """Loads the chat engine with the given model name and maximum retries. diff --git a/src/wandbot/chat/config.py b/src/wandbot/chat/config.py index 4e1b467..d8875ec 100644 --- a/src/wandbot/chat/config.py +++ b/src/wandbot/chat/config.py @@ -30,9 +30,8 @@ class ChatConfig(BaseSettings): env="WANDB_INDEX_ARTIFACT", validation_alias="wandb_index_artifact", ) - embeddings_cache: pathlib.Path = Field( - pathlib.Path("data/cache/embeddings"), env="EMBEDDINGS_CACHE_PATH" - ) + embeddings_model: str = "text-embedding-3-small" + embeddings_dim: int = 512 verbose: bool = False wandb_project: str | None = Field("wandbot_public", env="WANDB_PROJECT") wandb_entity: str | None = Field("wandbot", env="WANDB_ENTITY") diff --git a/src/wandbot/chat/retriever.py b/src/wandbot/chat/retriever.py index 8feb088..4c4137f 100644 --- a/src/wandbot/chat/retriever.py +++ b/src/wandbot/chat/retriever.py @@ -1,8 +1,8 @@ import os -import pathlib from typing import Any, Dict, List, Optional import requests +import wandb from llama_index import ( QueryBundle, ServiceContext, @@ -22,12 +22,10 @@ from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict - -import wandb from wandbot.utils import ( - get_logger, - load_service_context, - create_no_result_dummy_node + create_no_result_dummy_node, + get_logger, + load_service_context, ) logger = get_logger(__name__) @@ -102,7 +100,9 @@ def _postprocess_nodes( new_nodes.append(node) if len(new_nodes) < self.min_result_size: dummy_node = create_no_result_dummy_node() - new_nodes.extend([dummy_node] * (self.min_result_size - len(new_nodes))) + new_nodes.extend( + [dummy_node] * (self.min_result_size - len(new_nodes)) + ) return new_nodes @@ -139,19 +139,22 @@ def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]: else: results = response.json() - search_hits = [ - ( - "\n".join(hit["snippets"]), - { - "source": hit["url"], - "language": "en", - "description": hit["description"], - "title": hit["title"], - "tags": ["you.com"], - }, - ) + snippets = [hit["snippets"] for hit in results["hits"]] + snippet_metadata = [ + { + "source": hit["url"], + "language": "en", + "description": hit["description"], + "title": hit["title"], + "tags": ["you.com"], + } for hit in results["hits"] ] + search_hits = [] + for snippet_list, metadata in zip(snippets, snippet_metadata): + for snippet in snippet_list: + search_hits.append((snippet, metadata)) + return [ NodeWithScore( node=TextNode(text=s[0], metadata=s[1]), @@ -232,15 +235,14 @@ class RetrieverConfig(BaseSettings): env="WANDB_INDEX_ARTIFACT", validation_alias="wandb_index_artifact", ) - embeddings_cache: pathlib.Path = Field( - pathlib.Path("data/cache/embeddings"), env="EMBEDDINGS_CACHE_PATH" - ) + embeddings_model: str = "text-embedding-3-small" + embeddings_size: int = 512 top_k: int = Field( - default=5, + default=10, env="RETRIEVER_TOP_K", ) similarity_top_k: int = Field( - default=20, + default=10, env="RETRIEVER_SIMILARITY_TOP_K", ) language: str = Field( @@ -290,7 +292,8 @@ def __init__( service_context if service_context else load_service_context( - embeddings_cache=str(self.config.embeddings_cache), + embeddings_model=self.config.embeddings_model, + embeddings_size=self.config.embeddings_dim, callback_manager=callback_manager, ) ) @@ -299,7 +302,9 @@ def __init__( artifact_url=self.config.index_artifact ) - self.index = load_index_from_storage(self.storage_context) + self.index = load_index_from_storage( + self.storage_context, service_context=self.service_context + ) self._retriever = HybridRetriever( index=self.index, similarity_top_k=self.config.similarity_top_k, diff --git a/src/wandbot/ingestion/config.py b/src/wandbot/ingestion/config.py index 322f4f1..3362fba 100644 --- a/src/wandbot/ingestion/config.py +++ b/src/wandbot/ingestion/config.py @@ -223,9 +223,9 @@ def _set_cache_paths(cls, values: "DataStoreConfig") -> "DataStoreConfig": class VectorStoreConfig(BaseSettings): name: str = "vectorstore" - embedding_dim: int = 1536 + embeddings_model: str = "text-embedding-3-small" + embedding_dim: int = 512 persist_dir: pathlib.Path = pathlib.Path("data/cache/vectorstore") - chat_model_name: str = "gpt-3.5-turbo-0613" + chat_model_name: str = "gpt-3.5-turbo-1106" temperature: float = 0.1 max_retries: int = 3 - embeddings_cache: pathlib.Path = pathlib.Path("data/cache/embeddings") diff --git a/src/wandbot/ingestion/preprocess_data.py b/src/wandbot/ingestion/preprocess_data.py index e858661..80509c3 100644 --- a/src/wandbot/ingestion/preprocess_data.py +++ b/src/wandbot/ingestion/preprocess_data.py @@ -30,7 +30,6 @@ TokenTextSplitter, ) from llama_index.schema import BaseNode, TextNode - from wandbot.utils import get_logger logger = get_logger(__name__) @@ -120,7 +119,7 @@ def convert_lc_to_llama(document: LcDocument) -> LlamaDocument: return llama_document -def load(documents: List[LcDocument], chunk_size: int = 512) -> List[Any]: +def load(documents: List[LcDocument], chunk_size: int = 384) -> List[Any]: """Loads documents and returns a list of nodes. Args: @@ -159,6 +158,26 @@ def load(documents: List[LcDocument], chunk_size: int = 512) -> List[Any]: nodes = node_parser.get_nodes_from_documents(nodes) - nodes = list(filter(lambda x: len(x.get_content()) > 10, nodes)) + def filter_smaller_nodes( + text_nodes: List[TextNode], min_size: int = 5 + ) -> List[TextNode]: + """Filters out nodes that are smaller than the chunk size. + + Args: + text_nodes: A list of nodes. + min_size: The minimum size of a node. + + Returns: + A list of nodes. + """ + + for node in text_nodes: + content = node.get_content() + word_len = len( + [c for c in content.strip().split() if c and len(c) > 2] + ) + if word_len >= min_size: + yield node + nodes = list(filter_smaller_nodes(nodes)) return nodes diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index 0d79403..6463277 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -16,10 +16,9 @@ import pathlib from typing import Any, Dict, List +import wandb from langchain.schema import Document as LcDocument from llama_index.callbacks import WandbCallbackHandler - -import wandb from wandbot.ingestion import preprocess_data from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import ( @@ -55,7 +54,7 @@ def load( wandb.Error: An error occurred during the loading process. """ config: VectorStoreConfig = VectorStoreConfig() - run: wandb.Run = wandb.init( + run: wandb.wandb_sdk.wandb_run.Run = wandb.init( project=project, entity=entity, job_type="create_vectorstore" ) artifact: wandb.Artifact = run.use_artifact( @@ -64,7 +63,8 @@ def load( artifact_dir: str = artifact.download() storage_context = load_storage_context(config.embedding_dim) service_context = load_service_context( - embeddings_cache=str(config.embeddings_cache), + embeddings_model=config.embeddings_model, + embeddings_size=config.embedding_dim, llm="gpt-3.5-turbo-16k-0613", temperature=config.temperature, max_retries=config.max_retries, diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index f88a96c..5514bdf 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -85,26 +85,19 @@ def elapsed(self) -> float: return (self.stop - self.start).total_seconds() -def load_embeddings(cache_dir: str) -> OpenAIEmbedding: +def load_embeddings( + model_name: str = "text-embedding-3-small", dimensions: int = 512 +) -> OpenAIEmbedding: """Loads embeddings from cache or creates new ones if not found. Args: - cache_dir: The directory where the embeddings cache is stored. + model_name: The name of the model to load. + dimensions: The dimensions of the embeddings. Returns: A cached embedder instance. """ - # underlying_embeddings = OpenAIEmbeddings() - # - # embeddings_cache_fs = LocalFileStore(cache_dir) - # cached_embedder = CacheBackedEmbeddings.from_bytes_store( - # underlying_embeddings, - # embeddings_cache_fs, - # namespace=underlying_embeddings.model + "/", - # ) - # - # return cast(LCEmbeddings, cached_embedder) - embeddings = OpenAIEmbedding() + embeddings = OpenAIEmbedding(model=model_name, dimensions=dimensions) return embeddings @@ -139,7 +132,8 @@ def load_llm( def load_service_context( - embeddings_cache: str, + embeddings_model: str = "text-embedding-3-small", + embeddings_size: int = 512, llm: str = "gpt-3.5-turbo-16k-0613", temperature: float = 0.1, max_retries: int = 2, @@ -148,9 +142,10 @@ def load_service_context( """Loads a service context with the specified parameters. Args: + embeddings_model: The name of the embeddings model to load. + embeddings_size: The size of the embeddings. llm: The language model to load. temperature: The temperature parameter for the model. - embeddings_cache: The directory where the embeddings cache is stored. max_retries: The maximum number of retries for loading the model. callback_manager: The callback manager for the service context (optional). @@ -158,7 +153,9 @@ def load_service_context( A service context instance with the specified parameters. """ - embed_model = load_embeddings(embeddings_cache) + embed_model = load_embeddings( + model_name=embeddings_model, dimensions=embeddings_size + ) llm = load_llm( model_name=llm, temperature=temperature, From 50aadb5aa48aa2c122fce306c70239179126ec90 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 1 Feb 2024 11:20:08 +0530 Subject: [PATCH 03/41] chore: run linting and formatting changes --- src/wandbot/chat/chat.py | 5 +++-- src/wandbot/chat/query_enhancer.py | 1 + src/wandbot/chat/retriever.py | 3 ++- src/wandbot/ingestion/config.py | 1 + src/wandbot/ingestion/prepare_data.py | 3 ++- src/wandbot/ingestion/preprocess_data.py | 1 + src/wandbot/ingestion/utils.py | 1 + src/wandbot/ingestion/vectorstores.py | 3 ++- src/wandbot/utils.py | 3 ++- 9 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index e183abc..5b700a9 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -28,7 +28,6 @@ import json from typing import Any, Dict, List, Optional, Tuple -import wandb from llama_index import ServiceContext from llama_index.callbacks import ( CallbackManager, @@ -45,6 +44,9 @@ from llama_index.postprocessor.types import BaseNodePostprocessor from llama_index.schema import MetadataMode, NodeWithScore, QueryBundle from llama_index.tools import ToolOutput +from weave.monitoring import StreamTable + +import wandb from wandbot.chat.config import ChatConfig from wandbot.chat.prompts import load_chat_prompt, partial_format from wandbot.chat.query_enhancer import CompleteQuery, QueryHandler @@ -56,7 +58,6 @@ ) from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.utils import Timer, get_logger, load_service_context -from weave.monitoring import StreamTable logger = get_logger(__name__) diff --git a/src/wandbot/chat/query_enhancer.py b/src/wandbot/chat/query_enhancer.py index 63e80b7..63f54a5 100644 --- a/src/wandbot/chat/query_enhancer.py +++ b/src/wandbot/chat/query_enhancer.py @@ -15,6 +15,7 @@ from pydantic.v1 import BaseModel as BaseModelV1 from pydantic_settings import BaseSettings, SettingsConfigDict from tenacity import retry, stop_after_attempt, wait_random_exponential + from wandbot.chat.schemas import ChatRequest from wandbot.database.schemas import QuestionAnswer from wandbot.utils import FastTextLangDetect, get_logger diff --git a/src/wandbot/chat/retriever.py b/src/wandbot/chat/retriever.py index 4c4137f..0d0b6d6 100644 --- a/src/wandbot/chat/retriever.py +++ b/src/wandbot/chat/retriever.py @@ -2,7 +2,6 @@ from typing import Any, Dict, List, Optional import requests -import wandb from llama_index import ( QueryBundle, ServiceContext, @@ -22,6 +21,8 @@ from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict + +import wandb from wandbot.utils import ( create_no_result_dummy_node, get_logger, diff --git a/src/wandbot/ingestion/config.py b/src/wandbot/ingestion/config.py index 3362fba..5148016 100644 --- a/src/wandbot/ingestion/config.py +++ b/src/wandbot/ingestion/config.py @@ -17,6 +17,7 @@ from pydantic import BaseModel, Field, model_validator from pydantic_settings import BaseSettings + from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/ingestion/prepare_data.py b/src/wandbot/ingestion/prepare_data.py index 18ee9c9..776d7ee 100644 --- a/src/wandbot/ingestion/prepare_data.py +++ b/src/wandbot/ingestion/prepare_data.py @@ -22,13 +22,14 @@ import nbformat import pandas as pd -import wandb from google.cloud import bigquery from langchain.schema import Document from langchain_community.document_loaders import TextLoader from langchain_community.document_loaders.base import BaseLoader from nbconvert import MarkdownExporter from tqdm import tqdm + +import wandb from wandbot.ingestion.config import ( DataStoreConfig, DocodileEnglishStoreConfig, diff --git a/src/wandbot/ingestion/preprocess_data.py b/src/wandbot/ingestion/preprocess_data.py index 80509c3..3a69c1e 100644 --- a/src/wandbot/ingestion/preprocess_data.py +++ b/src/wandbot/ingestion/preprocess_data.py @@ -30,6 +30,7 @@ TokenTextSplitter, ) from llama_index.schema import BaseNode, TextNode + from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/ingestion/utils.py b/src/wandbot/ingestion/utils.py index dfe9ffa..064fcf7 100644 --- a/src/wandbot/ingestion/utils.py +++ b/src/wandbot/ingestion/utils.py @@ -37,6 +37,7 @@ import markdownify from bs4 import BeautifulSoup, Comment from git import Repo + from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index 6463277..2ec0481 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -16,9 +16,10 @@ import pathlib from typing import Any, Dict, List -import wandb from langchain.schema import Document as LcDocument from llama_index.callbacks import WandbCallbackHandler + +import wandb from wandbot.ingestion import preprocess_data from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import ( diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 5514bdf..7a13124 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -31,7 +31,6 @@ import faiss import fasttext -import wandb from llama_index import ( ServiceContext, StorageContext, @@ -45,6 +44,8 @@ from llama_index.vector_stores import FaissVectorStore from pydantic_settings import BaseSettings +import wandb + def get_logger(name: str) -> logging.Logger: """Creates and returns a logger with the specified name. From c027dde96b2ebddde5fd236314a69eca1e45542d Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 1 Feb 2024 18:49:35 +0530 Subject: [PATCH 04/41] chore: add simple multi-index query engine with router --- src/wandbot/chat/retriever.py | 136 ++++++++++++++++++++--- src/wandbot/ingestion/__main__.py | 8 +- src/wandbot/ingestion/prepare_data.py | 19 ++-- src/wandbot/ingestion/preprocess_data.py | 3 +- src/wandbot/ingestion/vectorstores.py | 42 ++++--- src/wandbot/utils.py | 36 +++--- 6 files changed, 180 insertions(+), 64 deletions(-) diff --git a/src/wandbot/chat/retriever.py b/src/wandbot/chat/retriever.py index 0d0b6d6..b4ed06f 100644 --- a/src/wandbot/chat/retriever.py +++ b/src/wandbot/chat/retriever.py @@ -1,32 +1,34 @@ import os -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import requests +import wandb from llama_index import ( QueryBundle, ServiceContext, StorageContext, - load_index_from_storage, + load_indices_from_storage, ) from llama_index.callbacks import CallbackManager, CBEventType, EventPayload +from llama_index.constants import DEFAULT_SIMILARITY_TOP_K from llama_index.core.base_retriever import BaseRetriever +from llama_index.llms.utils import LLMType from llama_index.postprocessor import CohereRerank from llama_index.postprocessor.types import BaseNodePostprocessor from llama_index.query_engine import RetrieverQueryEngine from llama_index.response_synthesizers import BaseSynthesizer, ResponseMode -from llama_index.retrievers import BM25Retriever -from llama_index.schema import NodeWithScore, QueryType, TextNode -from llama_index.vector_stores import FaissVectorStore +from llama_index.retrievers import BM25Retriever, QueryFusionRetriever +from llama_index.retrievers.fusion_retriever import FUSION_MODES +from llama_index.schema import IndexNode, NodeWithScore, QueryType, TextNode from llama_index.vector_stores.simple import DEFAULT_VECTOR_STORE, NAMESPACE_SEP from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict - -import wandb from wandbot.utils import ( create_no_result_dummy_node, get_logger, load_service_context, + load_storage_context, ) logger = get_logger(__name__) @@ -230,6 +232,89 @@ def retrieve( return nodes +class FusionRetriever(QueryFusionRetriever): + def __init__( + self, + retrievers: List[HybridRetriever], + llm: Optional[LLMType] = "default", + query_gen_prompt: Optional[str] = None, + mode: FUSION_MODES = FUSION_MODES.SIMPLE, + similarity_top_k: int = DEFAULT_SIMILARITY_TOP_K, + num_queries: int = 4, + use_async: bool = True, + verbose: bool = False, + callback_manager: Optional[CallbackManager] = None, + objects: Optional[List[IndexNode]] = None, + object_map: Optional[dict] = None, + ) -> None: + super().__init__( + retrievers=retrievers, + llm=llm, + query_gen_prompt=query_gen_prompt, + mode=mode, + similarity_top_k=similarity_top_k, + num_queries=num_queries, + use_async=use_async, + verbose=verbose, + callback_manager=callback_manager, + objects=objects, + object_map=object_map, + ) + self._retrievers = retrievers + + def _run_sync_queries( + self, queries: List[str], **kwargs + ) -> Dict[Tuple[str, int], List[NodeWithScore]]: + results = {} + for query in queries: + for i, retriever in enumerate(self._retrievers): + results[(query, i)] = retriever.retrieve(query, **kwargs) + + return results + + def _retrieve( + self, query_bundle: QueryBundle, **kwargs + ) -> List[NodeWithScore]: + if self.num_queries > 1: + queries = self._get_queries(query_bundle.query_str) + else: + queries = [query_bundle.query_str] + + if self.use_async: + results = self._run_nested_async_queries(queries) + else: + results = self._run_sync_queries(queries, **kwargs) + + if self.mode == FUSION_MODES.RECIPROCAL_RANK: + return self._reciprocal_rerank_fusion(results)[ + : self.similarity_top_k + ] + elif self.mode == FUSION_MODES.SIMPLE: + return self._simple_fusion(results)[: self.similarity_top_k] + else: + raise ValueError(f"Invalid fusion mode: {self.mode}") + + def retrieve( + self, str_or_query_bundle: QueryType, **kwargs + ) -> List[NodeWithScore]: + self._check_callback_manager() + + if isinstance(str_or_query_bundle, str): + query_bundle = QueryBundle(str_or_query_bundle) + else: + query_bundle = str_or_query_bundle + with self.callback_manager.as_trace("query"): + with self.callback_manager.event( + CBEventType.RETRIEVE, + payload={EventPayload.QUERY_STR: query_bundle.query_str}, + ) as retrieve_event: + nodes = self._retrieve(query_bundle, **kwargs) + retrieve_event.on_end( + payload={EventPayload.NODES: nodes}, + ) + return nodes + + class RetrieverConfig(BaseSettings): index_artifact: str = Field( "wandbot/wandbot-dev/wandbot_index:latest", @@ -258,7 +343,7 @@ class RetrieverConfig(BaseSettings): class WandbRetrieverQueryEngine(RetrieverQueryEngine): def __init__( self, - retriever: HybridRetriever, + retriever: FusionRetriever, response_synthesizer: Optional[BaseSynthesizer] = None, node_postprocessors: Optional[List[BaseNodePostprocessor]] = None, callback_manager: Optional[CallbackManager] = None, @@ -269,6 +354,7 @@ def __init__( node_postprocessors=node_postprocessors, callback_manager=callback_manager, ) + self._retriever = retriever def retrieve( self, query_bundle: QueryBundle, **kwargs @@ -299,23 +385,37 @@ def __init__( ) ) - self.storage_context = self.load_storage_context_from_artifact( + ( + self.storage_context, + index_ids, + ) = self.load_storage_context_from_artifact( artifact_url=self.config.index_artifact ) - self.index = load_index_from_storage( - self.storage_context, service_context=self.service_context + self.indices = load_indices_from_storage( + self.storage_context, + service_context=self.service_context, + index_ids=index_ids, ) - self._retriever = HybridRetriever( - index=self.index, + retriever_list = [] + for index in self.indices: + retriever = HybridRetriever( + index=index, + similarity_top_k=self.config.similarity_top_k, + storage_context=self.storage_context, + ) + retriever_list.append(retriever) + self._retriever = FusionRetriever( + retriever_list, similarity_top_k=self.config.similarity_top_k, - storage_context=self.storage_context, + num_queries=1, + use_async=False, ) self.is_avoid_query: bool | None = None def load_storage_context_from_artifact( self, artifact_url: str - ) -> StorageContext: + ) -> Tuple[StorageContext, Dict[str, str]]: """Loads the storage context from the given artifact URL. Args: @@ -328,11 +428,11 @@ def load_storage_context_from_artifact( artifact_dir = artifact.download() index_path = f"{artifact_dir}/{DEFAULT_VECTOR_STORE}{NAMESPACE_SEP}{DEFAULT_PERSIST_FNAME}" logger.debug(f"Loading index from {index_path}") - storage_context = StorageContext.from_defaults( - vector_store=FaissVectorStore.from_persist_path(index_path), + storage_context = load_storage_context( + embed_dimensions=self.config.embeddings_size, persist_dir=artifact_dir, ) - return storage_context + return storage_context, artifact.metadata["index_ids"] def load_query_engine( self, diff --git a/src/wandbot/ingestion/__main__.py b/src/wandbot/ingestion/__main__.py index f2b4e50..b911ab0 100644 --- a/src/wandbot/ingestion/__main__.py +++ b/src/wandbot/ingestion/__main__.py @@ -1,7 +1,6 @@ import os -from wandbot.ingestion import prepare_data, vectorstores -from wandbot.ingestion.report import create_ingestion_report +from wandbot.ingestion import vectorstores from wandbot.utils import get_logger logger = get_logger(__name__) @@ -11,10 +10,11 @@ def main(): project = os.environ.get("WANDB_PROJECT", "wandbot-dev") entity = os.environ.get("WANDB_ENTITY", "wandbot") - raw_artifact = prepare_data.load(project, entity) + # raw_artifact = prepare_data.load(project, entity) + raw_artifact = "wandbot/wandbot-dev/raw_dataset:v28" vectorstore_artifact = vectorstores.load(project, entity, raw_artifact) # TODO: include ingestion report - create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) + # create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) print(vectorstore_artifact) diff --git a/src/wandbot/ingestion/prepare_data.py b/src/wandbot/ingestion/prepare_data.py index 776d7ee..13b11b7 100644 --- a/src/wandbot/ingestion/prepare_data.py +++ b/src/wandbot/ingestion/prepare_data.py @@ -22,14 +22,13 @@ import nbformat import pandas as pd +import wandb from google.cloud import bigquery from langchain.schema import Document from langchain_community.document_loaders import TextLoader from langchain_community.document_loaders.base import BaseLoader from nbconvert import MarkdownExporter from tqdm import tqdm - -import wandb from wandbot.ingestion.config import ( DataStoreConfig, DocodileEnglishStoreConfig, @@ -852,14 +851,14 @@ def load( for loader in [ en_docodile_loader, ja_docodile_loader, - examples_code_loader, - examples_notebook_loader, - sdk_code_loader, - sdk_tests_loader, - weave_code_loader, - weave_examples_loader, - wandb_edu_code_loader, - fc_reports_loader, + # examples_code_loader, + # examples_notebook_loader, + # sdk_code_loader, + # sdk_tests_loader, + # weave_code_loader, + # weave_examples_loader, + # wandb_edu_code_loader, + # fc_reports_loader, ]: loader.config.docstore_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/wandbot/ingestion/preprocess_data.py b/src/wandbot/ingestion/preprocess_data.py index 3a69c1e..4726d7c 100644 --- a/src/wandbot/ingestion/preprocess_data.py +++ b/src/wandbot/ingestion/preprocess_data.py @@ -30,7 +30,6 @@ TokenTextSplitter, ) from llama_index.schema import BaseNode, TextNode - from wandbot.utils import get_logger logger = get_logger(__name__) @@ -120,7 +119,7 @@ def convert_lc_to_llama(document: LcDocument) -> LlamaDocument: return llama_document -def load(documents: List[LcDocument], chunk_size: int = 384) -> List[Any]: +def load(documents: List[LcDocument], chunk_size: int = 384) -> List[TextNode]: """Loads documents and returns a list of nodes. Args: diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index 2ec0481..cb1fd2b 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -16,10 +16,9 @@ import pathlib from typing import Any, Dict, List -from langchain.schema import Document as LcDocument -from llama_index.callbacks import WandbCallbackHandler - import wandb +from langchain.schema import Document as LcDocument +from llama_index.schema import TextNode from wandbot.ingestion import preprocess_data from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import ( @@ -75,7 +74,7 @@ def load( pathlib.Path(artifact_dir).rglob("documents.jsonl") ) - transformed_documents: List[LcDocument] = [] + transformed_documents: Dict[str, List[TextNode]] = {} for document_file in document_files: documents: List[LcDocument] = [] with document_file.open() as f: @@ -83,18 +82,31 @@ def load( doc_dict: Dict[str, Any] = json.loads(line) doc: LcDocument = LcDocument(**doc_dict) documents.append(doc) - transformed_documents.extend(preprocess_data.load(documents)) - unique_objects = {obj.hash: obj for obj in transformed_documents} - transformed_documents = list(unique_objects.values()) - index = load_index( - transformed_documents, - service_context, - storage_context, - persist_dir=str(config.persist_dir), + preprocessed_documents = preprocess_data.load(documents) + unique_objects = {obj.hash: obj for obj in preprocessed_documents} + preprocessed_documents = list(unique_objects.values()) + transformed_documents[ + document_file.parent.name + ] = preprocessed_documents + + for store_name, doc_list in transformed_documents.items(): + logger.info(f"Number of documents: {len(doc_list)}") + _ = load_index( + doc_list, + service_context, + storage_context, + index_id=store_name, + persist_dir=str(config.persist_dir), + ) + artifact = wandb.Artifact( + name="wandbot_index", + type="storage_context", + metadata={"index_ids": list(transformed_documents.keys())}, + ) + artifact.add_dir( + local_path=str(config.persist_dir), ) - wandb_callback: WandbCallbackHandler = WandbCallbackHandler() + run.log_artifact(artifact) - wandb_callback.persist_index(index, index_name=result_artifact_name) - wandb_callback.finish() run.finish() return f"{entity}/{project}/{result_artifact_name}:latest" diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 7a13124..1d7ad1e 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -31,11 +31,12 @@ import faiss import fasttext +import wandb from llama_index import ( ServiceContext, StorageContext, VectorStoreIndex, - load_index_from_storage, + load_indices_from_storage, ) from llama_index.embeddings import OpenAIEmbedding from llama_index.llms import LiteLLM @@ -44,8 +45,6 @@ from llama_index.vector_stores import FaissVectorStore from pydantic_settings import BaseSettings -import wandb - def get_logger(name: str) -> logging.Logger: """Creates and returns a logger with the specified name. @@ -168,11 +167,14 @@ def load_service_context( ) -def load_storage_context(embed_dimensions: int) -> StorageContext: +def load_storage_context( + embed_dimensions: int, persist_dir: str | None = None +) -> StorageContext: """Loads a storage context with the specified parameters. Args: embed_dimensions: The dimensions of the embeddings. + persist_dir: The directory where the storage context is persisted. Returns: A storage context instance with the specified parameters. @@ -181,14 +183,16 @@ def load_storage_context(embed_dimensions: int) -> StorageContext: faiss_index = faiss.IndexFlatL2(embed_dimensions) storage_context = StorageContext.from_defaults( vector_store=FaissVectorStore(faiss_index), + persist_dir=persist_dir, ) return storage_context def load_index( - nodes: Any, + nodes: List[TextNode], service_context: ServiceContext, storage_context: StorageContext, + index_id: str, persist_dir: str, ) -> VectorStoreIndex: """Loads an index from storage or creates a new one if not found. @@ -197,21 +201,20 @@ def load_index( nodes: The nodes to include in the index. service_context: The service context for the index. storage_context: The storage context for the index. + index_id: The ID of the index. persist_dir: The directory where the index is persisted. Returns: An index instance with the specified parameters. """ - try: - index = load_index_from_storage(storage_context) - except Exception: - index = VectorStoreIndex( - nodes=nodes, - service_context=service_context, - storage_context=storage_context, - show_progress=True, - ) - index.storage_context.persist(persist_dir=persist_dir) + index = VectorStoreIndex( + nodes=nodes, + service_context=service_context, + storage_context=storage_context, + show_progress=True, + ) + index.set_index_id(index_id) + index.storage_context.persist(persist_dir=persist_dir) return index @@ -321,3 +324,6 @@ def _load_model(self): ).download(root=str(self.config.fasttext_file_path.parent)) self._model = fasttext.load_model(str(self.config.fasttext_file_path)) return self._model + + +load_indices_from_storage From ac58107b0a96811dc78475e39dab4b607510d584 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 2 Feb 2024 10:08:20 +0530 Subject: [PATCH 05/41] refactor: move retriever to separate module. --- src/wandbot/chat/chat.py | 13 +- src/wandbot/chat/retriever.py | 525 ------------------------ src/wandbot/ingestion/prepare_data.py | 16 +- src/wandbot/retriever/__init__.py | 0 src/wandbot/retriever/base.py | 241 +++++++++++ src/wandbot/retriever/external.py | 71 ++++ src/wandbot/retriever/fusion.py | 163 ++++++++ src/wandbot/retriever/postprocessors.py | 83 ++++ 8 files changed, 572 insertions(+), 540 deletions(-) delete mode 100644 src/wandbot/chat/retriever.py create mode 100644 src/wandbot/retriever/__init__.py create mode 100644 src/wandbot/retriever/base.py create mode 100644 src/wandbot/retriever/external.py create mode 100644 src/wandbot/retriever/fusion.py create mode 100644 src/wandbot/retriever/postprocessors.py diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 5b700a9..99fe55d 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -28,6 +28,7 @@ import json from typing import Any, Dict, List, Optional, Tuple +import wandb from llama_index import ServiceContext from llama_index.callbacks import ( CallbackManager, @@ -44,20 +45,18 @@ from llama_index.postprocessor.types import BaseNodePostprocessor from llama_index.schema import MetadataMode, NodeWithScore, QueryBundle from llama_index.tools import ToolOutput -from weave.monitoring import StreamTable - -import wandb from wandbot.chat.config import ChatConfig from wandbot.chat.prompts import load_chat_prompt, partial_format from wandbot.chat.query_enhancer import CompleteQuery, QueryHandler -from wandbot.chat.retriever import ( - HybridRetriever, +from wandbot.chat.schemas import ChatRequest, ChatResponse +from wandbot.retriever.base import Retriever +from wandbot.retriever.fusion import HybridRetriever +from wandbot.retriever.postprocessors import ( LanguageFilterPostprocessor, MetadataPostprocessor, - Retriever, ) -from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.utils import Timer, get_logger, load_service_context +from weave.monitoring import StreamTable logger = get_logger(__name__) diff --git a/src/wandbot/chat/retriever.py b/src/wandbot/chat/retriever.py deleted file mode 100644 index b4ed06f..0000000 --- a/src/wandbot/chat/retriever.py +++ /dev/null @@ -1,525 +0,0 @@ -import os -from typing import Any, Dict, List, Optional, Tuple - -import requests -import wandb -from llama_index import ( - QueryBundle, - ServiceContext, - StorageContext, - load_indices_from_storage, -) -from llama_index.callbacks import CallbackManager, CBEventType, EventPayload -from llama_index.constants import DEFAULT_SIMILARITY_TOP_K -from llama_index.core.base_retriever import BaseRetriever -from llama_index.llms.utils import LLMType -from llama_index.postprocessor import CohereRerank -from llama_index.postprocessor.types import BaseNodePostprocessor -from llama_index.query_engine import RetrieverQueryEngine -from llama_index.response_synthesizers import BaseSynthesizer, ResponseMode -from llama_index.retrievers import BM25Retriever, QueryFusionRetriever -from llama_index.retrievers.fusion_retriever import FUSION_MODES -from llama_index.schema import IndexNode, NodeWithScore, QueryType, TextNode -from llama_index.vector_stores.simple import DEFAULT_VECTOR_STORE, NAMESPACE_SEP -from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict -from wandbot.utils import ( - create_no_result_dummy_node, - get_logger, - load_service_context, - load_storage_context, -) - -logger = get_logger(__name__) - - -class LanguageFilterPostprocessor(BaseNodePostprocessor): - """Language-based Node processor.""" - - languages: List[str] = ["en", "python"] - min_result_size: int = 10 - - @classmethod - def class_name(cls) -> str: - return "LanguageFilterPostprocessor" - - def _postprocess_nodes( - self, - nodes: List[NodeWithScore], - query_bundle: Optional[QueryBundle] = None, - ) -> List[NodeWithScore]: - """Postprocess nodes.""" - - new_nodes = [] - for node in nodes: - if node.metadata["language"] in self.languages: - new_nodes.append(node) - - if len(new_nodes) < self.min_result_size: - return new_nodes + nodes[: self.min_result_size - len(new_nodes)] - - return new_nodes - - -class MetadataPostprocessor(BaseNodePostprocessor): - """Metadata-based Node processor.""" - - min_result_size: int = 10 - include_tags: List[str] | None = None - exclude_tags: List[str] | None = None - - @classmethod - def class_name(cls) -> str: - return "MetadataPostprocessor" - - def _postprocess_nodes( - self, - nodes: List[NodeWithScore], - query_bundle: Optional[QueryBundle] = None, - ) -> List[NodeWithScore]: - """Postprocess nodes.""" - if not self.include_tags and not self.exclude_tags: - return nodes - new_nodes = [] - for node in nodes: - normalized_tags = [ - tag.lower().strip() for tag in node.metadata["tags"] - ] - if self.include_tags: - normalized_include_tags = [ - tag.lower().strip() for tag in self.include_tags - ] - if not set(normalized_include_tags).issubset( - set(normalized_tags) - ): - continue - if self.exclude_tags: - normalized_exclude_tags = [ - tag.lower().strip() for tag in self.exclude_tags - ] - if set(normalized_exclude_tags).issubset(set(normalized_tags)): - continue - new_nodes.append(node) - if len(new_nodes) < self.min_result_size: - dummy_node = create_no_result_dummy_node() - new_nodes.extend( - [dummy_node] * (self.min_result_size - len(new_nodes)) - ) - return new_nodes - - -class YouRetriever(BaseRetriever): - """You retriever.""" - - def __init__( - self, - api_key: Optional[str] = None, - similarity_top_k: int = 10, - callback_manager: Optional[CallbackManager] = None, - ) -> None: - """Init params.""" - self._api_key = api_key or os.environ["YOU_API_KEY"] - self.similarity_top_k = ( - similarity_top_k if similarity_top_k <= 20 else 20 - ) - super().__init__(callback_manager) - - def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]: - """Retrieve.""" - try: - headers = {"X-API-Key": self._api_key} - url = "https://api.ydc-index.io/search" - - querystring = { - "query": "Weights & Biases, W&B, wandb or Weave " - + query_bundle.query_str, - "num_web_results": self.similarity_top_k, - } - response = requests.get(url, headers=headers, params=querystring) - if response.status_code != 200: - return [] - else: - results = response.json() - - snippets = [hit["snippets"] for hit in results["hits"]] - snippet_metadata = [ - { - "source": hit["url"], - "language": "en", - "description": hit["description"], - "title": hit["title"], - "tags": ["you.com"], - } - for hit in results["hits"] - ] - search_hits = [] - for snippet_list, metadata in zip(snippets, snippet_metadata): - for snippet in snippet_list: - search_hits.append((snippet, metadata)) - - return [ - NodeWithScore( - node=TextNode(text=s[0], metadata=s[1]), - score=1.0, - ) - for s in search_hits - ] - except Exception as e: - return [] - - -class HybridRetriever(BaseRetriever): - def __init__( - self, - index, - storage_context, - similarity_top_k: int = 20, - ): - self.index = index - self.storage_context = storage_context - - self.vector_retriever = self.index.as_retriever( - similarity_top_k=similarity_top_k, - storage_context=self.storage_context, - ) - self.bm25_retriever = BM25Retriever.from_defaults( - docstore=self.index.docstore, - similarity_top_k=similarity_top_k, - ) - self.you_retriever = YouRetriever( - api_key=os.environ.get("YOU_API_KEY"), - similarity_top_k=similarity_top_k, - ) - super().__init__() - - def _retrieve(self, query: QueryBundle, **kwargs): - bm25_nodes = self.bm25_retriever.retrieve(query) - vector_nodes = self.vector_retriever.retrieve(query) - you_nodes = ( - self.you_retriever.retrieve(query) - if not kwargs.get("is_avoid_query", False) - else [] - ) - - # combine the two lists of nodes - all_nodes = [] - node_ids = set() - for n in bm25_nodes + vector_nodes + you_nodes: - if n.node.node_id not in node_ids: - all_nodes.append(n) - node_ids.add(n.node.node_id) - return all_nodes - - def retrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - self._check_callback_manager() - - if isinstance(str_or_query_bundle, str): - query_bundle = QueryBundle(str_or_query_bundle) - else: - query_bundle = str_or_query_bundle - with self.callback_manager.as_trace("query"): - with self.callback_manager.event( - CBEventType.RETRIEVE, - payload={EventPayload.QUERY_STR: query_bundle.query_str}, - ) as retrieve_event: - nodes = self._retrieve(query_bundle, **kwargs) - retrieve_event.on_end( - payload={EventPayload.NODES: nodes}, - ) - return nodes - - -class FusionRetriever(QueryFusionRetriever): - def __init__( - self, - retrievers: List[HybridRetriever], - llm: Optional[LLMType] = "default", - query_gen_prompt: Optional[str] = None, - mode: FUSION_MODES = FUSION_MODES.SIMPLE, - similarity_top_k: int = DEFAULT_SIMILARITY_TOP_K, - num_queries: int = 4, - use_async: bool = True, - verbose: bool = False, - callback_manager: Optional[CallbackManager] = None, - objects: Optional[List[IndexNode]] = None, - object_map: Optional[dict] = None, - ) -> None: - super().__init__( - retrievers=retrievers, - llm=llm, - query_gen_prompt=query_gen_prompt, - mode=mode, - similarity_top_k=similarity_top_k, - num_queries=num_queries, - use_async=use_async, - verbose=verbose, - callback_manager=callback_manager, - objects=objects, - object_map=object_map, - ) - self._retrievers = retrievers - - def _run_sync_queries( - self, queries: List[str], **kwargs - ) -> Dict[Tuple[str, int], List[NodeWithScore]]: - results = {} - for query in queries: - for i, retriever in enumerate(self._retrievers): - results[(query, i)] = retriever.retrieve(query, **kwargs) - - return results - - def _retrieve( - self, query_bundle: QueryBundle, **kwargs - ) -> List[NodeWithScore]: - if self.num_queries > 1: - queries = self._get_queries(query_bundle.query_str) - else: - queries = [query_bundle.query_str] - - if self.use_async: - results = self._run_nested_async_queries(queries) - else: - results = self._run_sync_queries(queries, **kwargs) - - if self.mode == FUSION_MODES.RECIPROCAL_RANK: - return self._reciprocal_rerank_fusion(results)[ - : self.similarity_top_k - ] - elif self.mode == FUSION_MODES.SIMPLE: - return self._simple_fusion(results)[: self.similarity_top_k] - else: - raise ValueError(f"Invalid fusion mode: {self.mode}") - - def retrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - self._check_callback_manager() - - if isinstance(str_or_query_bundle, str): - query_bundle = QueryBundle(str_or_query_bundle) - else: - query_bundle = str_or_query_bundle - with self.callback_manager.as_trace("query"): - with self.callback_manager.event( - CBEventType.RETRIEVE, - payload={EventPayload.QUERY_STR: query_bundle.query_str}, - ) as retrieve_event: - nodes = self._retrieve(query_bundle, **kwargs) - retrieve_event.on_end( - payload={EventPayload.NODES: nodes}, - ) - return nodes - - -class RetrieverConfig(BaseSettings): - index_artifact: str = Field( - "wandbot/wandbot-dev/wandbot_index:latest", - env="WANDB_INDEX_ARTIFACT", - validation_alias="wandb_index_artifact", - ) - embeddings_model: str = "text-embedding-3-small" - embeddings_size: int = 512 - top_k: int = Field( - default=10, - env="RETRIEVER_TOP_K", - ) - similarity_top_k: int = Field( - default=10, - env="RETRIEVER_SIMILARITY_TOP_K", - ) - language: str = Field( - default="en", - env="RETRIEVER_LANGUAGE", - ) - model_config = SettingsConfigDict( - env_file=".env", env_file_encoding="utf-8", extra="allow" - ) - - -class WandbRetrieverQueryEngine(RetrieverQueryEngine): - def __init__( - self, - retriever: FusionRetriever, - response_synthesizer: Optional[BaseSynthesizer] = None, - node_postprocessors: Optional[List[BaseNodePostprocessor]] = None, - callback_manager: Optional[CallbackManager] = None, - ) -> None: - super().__init__( - retriever=retriever, - response_synthesizer=response_synthesizer, - node_postprocessors=node_postprocessors, - callback_manager=callback_manager, - ) - self._retriever = retriever - - def retrieve( - self, query_bundle: QueryBundle, **kwargs - ) -> List[NodeWithScore]: - nodes = self._retriever.retrieve(query_bundle, **kwargs) - return self._apply_node_postprocessors(nodes, query_bundle=query_bundle) - - -class Retriever: - def __init__( - self, - config: RetrieverConfig | None = None, - run: wandb.wandb_sdk.wandb_run.Run | None = None, - service_context: ServiceContext | None = None, - callback_manager: CallbackManager | None = None, - ): - self.config = ( - config if isinstance(config, RetrieverConfig) else RetrieverConfig() - ) - self.run = run - self.service_context = ( - service_context - if service_context - else load_service_context( - embeddings_model=self.config.embeddings_model, - embeddings_size=self.config.embeddings_dim, - callback_manager=callback_manager, - ) - ) - - ( - self.storage_context, - index_ids, - ) = self.load_storage_context_from_artifact( - artifact_url=self.config.index_artifact - ) - - self.indices = load_indices_from_storage( - self.storage_context, - service_context=self.service_context, - index_ids=index_ids, - ) - retriever_list = [] - for index in self.indices: - retriever = HybridRetriever( - index=index, - similarity_top_k=self.config.similarity_top_k, - storage_context=self.storage_context, - ) - retriever_list.append(retriever) - self._retriever = FusionRetriever( - retriever_list, - similarity_top_k=self.config.similarity_top_k, - num_queries=1, - use_async=False, - ) - self.is_avoid_query: bool | None = None - - def load_storage_context_from_artifact( - self, artifact_url: str - ) -> Tuple[StorageContext, Dict[str, str]]: - """Loads the storage context from the given artifact URL. - - Args: - artifact_url: A string representing the URL of the artifact. - - Returns: - An instance of StorageContext. - """ - artifact = self.run.use_artifact(artifact_url) - artifact_dir = artifact.download() - index_path = f"{artifact_dir}/{DEFAULT_VECTOR_STORE}{NAMESPACE_SEP}{DEFAULT_PERSIST_FNAME}" - logger.debug(f"Loading index from {index_path}") - storage_context = load_storage_context( - embed_dimensions=self.config.embeddings_size, - persist_dir=artifact_dir, - ) - return storage_context, artifact.metadata["index_ids"] - - def load_query_engine( - self, - top_k: int | None = None, - language: str | None = None, - include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = None, - ) -> WandbRetrieverQueryEngine: - top_k = top_k or self.config.top_k - language = language or self.config.language - - if is_avoid_query is not None: - self.is_avoid_query = is_avoid_query - - node_postprocessors = [ - MetadataPostprocessor( - include_tags=include_tags, - exclude_tags=exclude_tags, - min_result_size=top_k, - ), - LanguageFilterPostprocessor( - languages=[language, "python"], min_result_size=top_k - ), - CohereRerank(top_n=top_k, model="rerank-english-v2.0") - if language == "en" - else CohereRerank(top_n=top_k, model="rerank-multilingual-v2.0"), - ] - query_engine = WandbRetrieverQueryEngine.from_args( - retriever=self._retriever, - node_postprocessors=node_postprocessors, - response_mode=ResponseMode.NO_TEXT, - service_context=self.service_context, - ) - return query_engine - - def retrieve( - self, - query: str, - language: str | None = None, - top_k: int | None = None, - include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = False, - ): - """Retrieves the top k results from the index for the given query. - - Args: - query: A string representing the query. - language: A string representing the language of the query. - top_k: An integer representing the number of top results to retrieve. - include_tags: A list of strings representing the tags to include in the results. - exclude_tags: A list of strings representing the tags to exclude from the results. - - Returns: - A list of dictionaries representing the retrieved results. - """ - top_k = top_k or self.config.top_k - language = language or self.config.language - - retrieval_engine = self.load_query_engine( - top_k=top_k, - language=language, - include_tags=include_tags, - exclude_tags=exclude_tags, - ) - - avoid_query = self.is_avoid_query or is_avoid_query - - query_bundle = QueryBundle(query_str=query) - results = retrieval_engine.retrieve( - query_bundle, is_avoid_query=bool(avoid_query) - ) - - outputs = [ - { - "text": node.get_text(), - "metadata": node.metadata, - "score": node.get_score(), - } - for node in results - ] - self.is_avoid_query = None - return outputs - - def __call__(self, query: str, **kwargs) -> List[Dict[str, Any]]: - retrievals = self.retrieve(query, **kwargs) - logger.debug(f"Retrieved {len(retrievals)} results.") - logger.debug(f"Retrieval: {retrievals[0]}") - return retrievals diff --git a/src/wandbot/ingestion/prepare_data.py b/src/wandbot/ingestion/prepare_data.py index 13b11b7..18ee9c9 100644 --- a/src/wandbot/ingestion/prepare_data.py +++ b/src/wandbot/ingestion/prepare_data.py @@ -851,14 +851,14 @@ def load( for loader in [ en_docodile_loader, ja_docodile_loader, - # examples_code_loader, - # examples_notebook_loader, - # sdk_code_loader, - # sdk_tests_loader, - # weave_code_loader, - # weave_examples_loader, - # wandb_edu_code_loader, - # fc_reports_loader, + examples_code_loader, + examples_notebook_loader, + sdk_code_loader, + sdk_tests_loader, + weave_code_loader, + weave_examples_loader, + wandb_edu_code_loader, + fc_reports_loader, ]: loader.config.docstore_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/wandbot/retriever/__init__.py b/src/wandbot/retriever/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py new file mode 100644 index 0000000..fc689fb --- /dev/null +++ b/src/wandbot/retriever/base.py @@ -0,0 +1,241 @@ +from typing import Any, Dict, List, Optional, Tuple + +import wandb +from llama_index import ( + QueryBundle, + ServiceContext, + StorageContext, + load_indices_from_storage, +) +from llama_index.callbacks import CallbackManager +from llama_index.postprocessor import BaseNodePostprocessor, CohereRerank +from llama_index.query_engine import RetrieverQueryEngine +from llama_index.response_synthesizers import BaseSynthesizer, ResponseMode +from llama_index.schema import NodeWithScore +from llama_index.vector_stores.simple import DEFAULT_VECTOR_STORE, NAMESPACE_SEP +from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict +from wandbot.retriever.fusion import FusionRetriever, HybridRetriever +from wandbot.retriever.postprocessors import ( + LanguageFilterPostprocessor, + MetadataPostprocessor, +) +from wandbot.utils import get_logger, load_service_context, load_storage_context + +logger = get_logger(__name__) + + +class WandbRetrieverQueryEngine(RetrieverQueryEngine): + def __init__( + self, + retriever: FusionRetriever, + response_synthesizer: Optional[BaseSynthesizer] = None, + node_postprocessors: Optional[List[BaseNodePostprocessor]] = None, + callback_manager: Optional[CallbackManager] = None, + ) -> None: + super().__init__( + retriever=retriever, + response_synthesizer=response_synthesizer, + node_postprocessors=node_postprocessors, + callback_manager=callback_manager, + ) + self._retriever = retriever + + def retrieve( + self, query_bundle: QueryBundle, **kwargs + ) -> List[NodeWithScore]: + nodes = self._retriever.retrieve(query_bundle, **kwargs) + return self._apply_node_postprocessors(nodes, query_bundle=query_bundle) + + +class RetrieverConfig(BaseSettings): + index_artifact: str = Field( + "wandbot/wandbot-dev/wandbot_index:latest", + env="WANDB_INDEX_ARTIFACT", + validation_alias="wandb_index_artifact", + ) + embeddings_model: str = "text-embedding-3-small" + embeddings_size: int = 512 + top_k: int = Field( + default=10, + env="RETRIEVER_TOP_K", + ) + similarity_top_k: int = Field( + default=10, + env="RETRIEVER_SIMILARITY_TOP_K", + ) + language: str = Field( + default="en", + env="RETRIEVER_LANGUAGE", + ) + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="allow" + ) + + +class Retriever: + def __init__( + self, + config: RetrieverConfig | None = None, + run: wandb.wandb_sdk.wandb_run.Run | None = None, + service_context: ServiceContext | None = None, + callback_manager: CallbackManager | None = None, + ): + self.config = ( + config if isinstance(config, RetrieverConfig) else RetrieverConfig() + ) + self.run = run + self.service_context = ( + service_context + if service_context + else load_service_context( + embeddings_model=self.config.embeddings_model, + embeddings_size=self.config.embeddings_dim, + callback_manager=callback_manager, + ) + ) + + ( + self.storage_context, + index_ids, + ) = self.load_storage_context_from_artifact( + artifact_url=self.config.index_artifact + ) + + self.indices = load_indices_from_storage( + self.storage_context, + service_context=self.service_context, + index_ids=index_ids, + ) + retriever_list = [] + for index in self.indices: + retriever = HybridRetriever( + index=index, + similarity_top_k=self.config.similarity_top_k, + storage_context=self.storage_context, + ) + retriever_list.append(retriever) + self._retriever = FusionRetriever( + retriever_list, + similarity_top_k=self.config.similarity_top_k, + num_queries=1, + use_async=False, + ) + self.is_avoid_query: bool | None = None + + def load_storage_context_from_artifact( + self, artifact_url: str + ) -> Tuple[StorageContext, Dict[str, str]]: + """Loads the storage context from the given artifact URL. + + Args: + artifact_url: A string representing the URL of the artifact. + + Returns: + An instance of StorageContext. + """ + artifact = self.run.use_artifact(artifact_url) + artifact_dir = artifact.download() + index_path = f"{artifact_dir}/{DEFAULT_VECTOR_STORE}{NAMESPACE_SEP}{DEFAULT_PERSIST_FNAME}" + logger.debug(f"Loading index from {index_path}") + storage_context = load_storage_context( + embed_dimensions=self.config.embeddings_size, + persist_dir=artifact_dir, + ) + return storage_context, artifact.metadata["index_ids"] + + def load_query_engine( + self, + top_k: int | None = None, + language: str | None = None, + include_tags: List[str] | None = None, + exclude_tags: List[str] | None = None, + is_avoid_query: bool | None = None, + ) -> WandbRetrieverQueryEngine: + top_k = top_k or self.config.top_k + language = language or self.config.language + + if is_avoid_query is not None: + self.is_avoid_query = is_avoid_query + + node_postprocessors = [ + MetadataPostprocessor( + include_tags=include_tags, + exclude_tags=exclude_tags, + min_result_size=top_k, + ), + LanguageFilterPostprocessor( + languages=[language, "python"], min_result_size=top_k + ), + CohereRerank(top_n=top_k, model="rerank-english-v2.0") + if language == "en" + else CohereRerank(top_n=top_k, model="rerank-multilingual-v2.0"), + ] + query_engine = WandbRetrieverQueryEngine.from_args( + retriever=self._retriever, + node_postprocessors=node_postprocessors, + response_mode=ResponseMode.NO_TEXT, + service_context=self.service_context, + ) + return query_engine + + def retrieve( + self, + query: str, + language: str | None = None, + top_k: int | None = None, + include_tags: List[str] | None = None, + exclude_tags: List[str] | None = None, + is_avoid_query: bool | None = False, + ): + """Retrieves the top k results from the index for the given query. + + Args: + query: A string representing the query. + language: A string representing the language of the query. + top_k: An integer representing the number of top results to retrieve. + include_tags: A list of strings representing the tags to include in the results. + exclude_tags: A list of strings representing the tags to exclude from the results. + + Returns: + A list of dictionaries representing the retrieved results. + """ + top_k = top_k or self.config.top_k + language = language or self.config.language + + retrieval_engine = self.load_query_engine( + top_k=top_k, + language=language, + include_tags=include_tags, + exclude_tags=exclude_tags, + ) + + avoid_query = self.is_avoid_query or is_avoid_query + + query_bundle = QueryBundle( + query_str=query, + embedding=self.service_context.embed_model.get_query_embedding( + query=query + ), + ) + results = retrieval_engine.retrieve( + query_bundle, is_avoid_query=bool(avoid_query) + ) + + outputs = [ + { + "text": node.get_text(), + "metadata": node.metadata, + "score": node.get_score(), + } + for node in results + ] + self.is_avoid_query = None + return outputs + + def __call__(self, query: str, **kwargs) -> List[Dict[str, Any]]: + retrievals = self.retrieve(query, **kwargs) + logger.debug(f"Retrieved {len(retrievals)} results.") + logger.debug(f"Retrieval: {retrievals[0]}") + return retrievals diff --git a/src/wandbot/retriever/external.py b/src/wandbot/retriever/external.py new file mode 100644 index 0000000..7a42917 --- /dev/null +++ b/src/wandbot/retriever/external.py @@ -0,0 +1,71 @@ +import os +from typing import List, Optional + +import requests +from llama_index import QueryBundle +from llama_index.callbacks import CallbackManager +from llama_index.core.base_retriever import BaseRetriever +from llama_index.schema import NodeWithScore, TextNode +from wandbot.utils import get_logger + +logger = get_logger(__name__) + + +class YouRetriever(BaseRetriever): + """You retriever.""" + + def __init__( + self, + api_key: Optional[str] = None, + similarity_top_k: int = 10, + callback_manager: Optional[CallbackManager] = None, + ) -> None: + """Init params.""" + self._api_key = api_key or os.environ["YOU_API_KEY"] + self.similarity_top_k = ( + similarity_top_k if similarity_top_k <= 20 else 20 + ) + super().__init__(callback_manager) + + def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]: + """Retrieve.""" + try: + headers = {"X-API-Key": self._api_key} + url = "https://api.ydc-index.io/search" + + querystring = { + "query": "Weights & Biases, W&B, wandb or Weave " + + query_bundle.query_str, + "num_web_results": self.similarity_top_k, + } + response = requests.get(url, headers=headers, params=querystring) + if response.status_code != 200: + return [] + else: + results = response.json() + + snippets = [hit["snippets"] for hit in results["hits"]] + snippet_metadata = [ + { + "source": hit["url"], + "language": "en", + "description": hit["description"], + "title": hit["title"], + "tags": ["you.com"], + } + for hit in results["hits"] + ] + search_hits = [] + for snippet_list, metadata in zip(snippets, snippet_metadata): + for snippet in snippet_list: + search_hits.append((snippet, metadata)) + + return [ + NodeWithScore( + node=TextNode(text=s[0], metadata=s[1]), + score=1.0, + ) + for s in search_hits + ] + except Exception as e: + return [] diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py new file mode 100644 index 0000000..5fc78a5 --- /dev/null +++ b/src/wandbot/retriever/fusion.py @@ -0,0 +1,163 @@ +import os +from typing import Dict, List, Optional, Tuple + +from llama_index import QueryBundle +from llama_index.callbacks import CallbackManager, CBEventType, EventPayload +from llama_index.constants import DEFAULT_SIMILARITY_TOP_K +from llama_index.core.base_retriever import BaseRetriever +from llama_index.llms.utils import LLMType +from llama_index.retrievers import BM25Retriever, QueryFusionRetriever +from llama_index.retrievers.fusion_retriever import FUSION_MODES +from llama_index.schema import IndexNode, NodeWithScore, QueryType +from wandbot.retriever.external import YouRetriever +from wandbot.utils import get_logger + +logger = get_logger(__name__) + + +class HybridRetriever(BaseRetriever): + def __init__( + self, + index, + storage_context, + similarity_top_k: int = 20, + ): + self.index = index + self.storage_context = storage_context + + self.vector_retriever = self.index.as_retriever( + similarity_top_k=similarity_top_k, + storage_context=self.storage_context, + ) + self.bm25_retriever = BM25Retriever.from_defaults( + docstore=self.index.docstore, + similarity_top_k=similarity_top_k, + ) + self.you_retriever = YouRetriever( + api_key=os.environ.get("YOU_API_KEY"), + similarity_top_k=similarity_top_k, + ) + super().__init__() + + def _retrieve(self, query: QueryBundle, **kwargs): + bm25_nodes = self.bm25_retriever.retrieve(query) + vector_nodes = self.vector_retriever.retrieve(query) + you_nodes = ( + self.you_retriever.retrieve(query) + if not kwargs.get("is_avoid_query", False) + else [] + ) + + # combine the two lists of nodes + all_nodes = [] + node_ids = set() + for n in bm25_nodes + vector_nodes + you_nodes: + if n.node.node_id not in node_ids: + all_nodes.append(n) + node_ids.add(n.node.node_id) + return all_nodes + + def retrieve( + self, str_or_query_bundle: QueryType, **kwargs + ) -> List[NodeWithScore]: + self._check_callback_manager() + + if isinstance(str_or_query_bundle, str): + query_bundle = QueryBundle(str_or_query_bundle) + else: + query_bundle = str_or_query_bundle + with self.callback_manager.as_trace("query"): + with self.callback_manager.event( + CBEventType.RETRIEVE, + payload={EventPayload.QUERY_STR: query_bundle.query_str}, + ) as retrieve_event: + nodes = self._retrieve(query_bundle, **kwargs) + retrieve_event.on_end( + payload={EventPayload.NODES: nodes}, + ) + return nodes + + +class FusionRetriever(QueryFusionRetriever): + def __init__( + self, + retrievers: List[HybridRetriever], + llm: Optional[LLMType] = "default", + query_gen_prompt: Optional[str] = None, + mode: FUSION_MODES = FUSION_MODES.SIMPLE, + similarity_top_k: int = DEFAULT_SIMILARITY_TOP_K, + num_queries: int = 4, + use_async: bool = True, + verbose: bool = False, + callback_manager: Optional[CallbackManager] = None, + objects: Optional[List[IndexNode]] = None, + object_map: Optional[dict] = None, + ) -> None: + super().__init__( + retrievers=retrievers, + llm=llm, + query_gen_prompt=query_gen_prompt, + mode=mode, + similarity_top_k=similarity_top_k, + num_queries=num_queries, + use_async=use_async, + verbose=verbose, + callback_manager=callback_manager, + objects=objects, + object_map=object_map, + ) + self._retrievers = retrievers + + def _run_sync_queries( + self, queries: List[QueryBundle], **kwargs + ) -> Dict[Tuple[str, int], List[NodeWithScore]]: + results = {} + for query in queries: + for i, retriever in enumerate(self._retrievers): + results[(query.query_str, i)] = retriever.retrieve( + query, **kwargs + ) + + return results + + def _retrieve( + self, query_bundle: QueryBundle, **kwargs + ) -> List[NodeWithScore]: + if self.num_queries > 1: + queries = self._get_queries(query_bundle.query_str) + else: + queries = [query_bundle] + + if self.use_async: + results = self._run_nested_async_queries(queries) + else: + results = self._run_sync_queries(queries, **kwargs) + + if self.mode == FUSION_MODES.RECIPROCAL_RANK: + return self._reciprocal_rerank_fusion(results)[ + : self.similarity_top_k + ] + elif self.mode == FUSION_MODES.SIMPLE: + return self._simple_fusion(results)[: self.similarity_top_k] + else: + raise ValueError(f"Invalid fusion mode: {self.mode}") + + def retrieve( + self, str_or_query_bundle: QueryType, **kwargs + ) -> List[NodeWithScore]: + self._check_callback_manager() + + if isinstance(str_or_query_bundle, str): + query_bundle = QueryBundle(str_or_query_bundle) + else: + query_bundle = str_or_query_bundle + with self.callback_manager.as_trace("query"): + with self.callback_manager.event( + CBEventType.RETRIEVE, + payload={EventPayload.QUERY_STR: query_bundle.query_str}, + ) as retrieve_event: + nodes = self._retrieve(query_bundle, **kwargs) + retrieve_event.on_end( + payload={EventPayload.NODES: nodes}, + ) + return nodes diff --git a/src/wandbot/retriever/postprocessors.py b/src/wandbot/retriever/postprocessors.py new file mode 100644 index 0000000..544f22b --- /dev/null +++ b/src/wandbot/retriever/postprocessors.py @@ -0,0 +1,83 @@ +from typing import List, Optional + +from llama_index import QueryBundle +from llama_index.postprocessor import BaseNodePostprocessor +from llama_index.schema import NodeWithScore +from wandbot.utils import create_no_result_dummy_node, get_logger + +logger = get_logger(__name__) + + +class LanguageFilterPostprocessor(BaseNodePostprocessor): + """Language-based Node processor.""" + + languages: List[str] = ["en", "python"] + min_result_size: int = 10 + + @classmethod + def class_name(cls) -> str: + return "LanguageFilterPostprocessor" + + def _postprocess_nodes( + self, + nodes: List[NodeWithScore], + query_bundle: Optional[QueryBundle] = None, + ) -> List[NodeWithScore]: + """Postprocess nodes.""" + + new_nodes = [] + for node in nodes: + if node.metadata["language"] in self.languages: + new_nodes.append(node) + + if len(new_nodes) < self.min_result_size: + return new_nodes + nodes[: self.min_result_size - len(new_nodes)] + + return new_nodes + + +class MetadataPostprocessor(BaseNodePostprocessor): + """Metadata-based Node processor.""" + + min_result_size: int = 10 + include_tags: List[str] | None = None + exclude_tags: List[str] | None = None + + @classmethod + def class_name(cls) -> str: + return "MetadataPostprocessor" + + def _postprocess_nodes( + self, + nodes: List[NodeWithScore], + query_bundle: Optional[QueryBundle] = None, + ) -> List[NodeWithScore]: + """Postprocess nodes.""" + if not self.include_tags and not self.exclude_tags: + return nodes + new_nodes = [] + for node in nodes: + normalized_tags = [ + tag.lower().strip() for tag in node.metadata["tags"] + ] + if self.include_tags: + normalized_include_tags = [ + tag.lower().strip() for tag in self.include_tags + ] + if not set(normalized_include_tags).issubset( + set(normalized_tags) + ): + continue + if self.exclude_tags: + normalized_exclude_tags = [ + tag.lower().strip() for tag in self.exclude_tags + ] + if set(normalized_exclude_tags).issubset(set(normalized_tags)): + continue + new_nodes.append(node) + if len(new_nodes) < self.min_result_size: + dummy_node = create_no_result_dummy_node() + new_nodes.extend( + [dummy_node] * (self.min_result_size - len(new_nodes)) + ) + return new_nodes From aa0427af526e98795b65a725de1d7a79001a2161 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 2 Feb 2024 11:46:25 +0530 Subject: [PATCH 06/41] refactor: split out routers in the api --- src/wandbot/api/app.py | 266 +++++----------------------- src/wandbot/api/routers/__init__.py | 0 src/wandbot/api/routers/chat.py | 43 +++++ src/wandbot/api/routers/database.py | 119 +++++++++++++ src/wandbot/api/routers/retrieve.py | 50 ++++++ src/wandbot/database/client.py | 5 +- src/wandbot/retriever/base.py | 104 ++++++++++- src/wandbot/retriever/fusion.py | 103 ++++++++++- src/wandbot/utils.py | 48 ++++- 9 files changed, 497 insertions(+), 241 deletions(-) create mode 100644 src/wandbot/api/routers/__init__.py create mode 100644 src/wandbot/api/routers/chat.py create mode 100644 src/wandbot/api/routers/database.py create mode 100644 src/wandbot/api/routers/retrieve.py diff --git a/src/wandbot/api/app.py b/src/wandbot/api/app.py index a0faf70..cea5004 100644 --- a/src/wandbot/api/app.py +++ b/src/wandbot/api/app.py @@ -29,247 +29,75 @@ """ import asyncio +from contextlib import asynccontextmanager from datetime import datetime, timezone import pandas as pd -from fastapi import FastAPI, Response, status - import wandb -from wandbot.api.schemas import ( - APICreateChatThreadRequest, - APIFeedbackRequest, - APIFeedbackResponse, - APIGetChatThreadResponse, - APIQueryRequest, - APIQueryResponse, - APIQuestionAnswerRequest, - APIQuestionAnswerResponse, - APIRetrievalRequest, - APIRetrievalResponse, - APIRetrievalResult, -) -from wandbot.chat.chat import Chat -from wandbot.chat.config import ChatConfig -from wandbot.chat.schemas import ChatRequest -from wandbot.database.client import DatabaseClient -from wandbot.database.database import engine -from wandbot.database.models import Base +from fastapi import FastAPI +from wandbot.api.routers import chat as chat_router +from wandbot.api.routers import database as database_router +from wandbot.api.routers import retrieve as retrieve_router from wandbot.utils import get_logger logger = get_logger(__name__) - -Base.metadata.create_all(bind=engine) -chat: Chat | None = None -app = FastAPI(name="wandbot", version="1.0.0") -db_client: DatabaseClient | None = None last_backup = datetime.now().astimezone(timezone.utc) -async def backup_db(): - """Periodically backs up the database to a table. +@asynccontextmanager +async def lifespan(app: FastAPI): + """Handles the lifespan of the application. - This function runs periodically and retrieves all question-answer threads from the database since the last backup. - It then creates a pandas DataFrame from the retrieved threads and logs it to a table using Weights & Biases. - The last backup timestamp is updated after each backup. + This function is called by the Uvicorn server to handle the lifespan of the application. + It is used to perform any necessary startup and shutdown operations. Returns: None """ - global last_backup - while True: - chat_threads = db_client.get_all_question_answers(last_backup) - if chat_threads is not None: - chat_table = pd.DataFrame( - [chat_thread for chat_thread in chat_threads] - ) - last_backup = datetime.now().astimezone(timezone.utc) - logger.info( - f"Backing up database to Table at {last_backup}: Number of chat threads: {len(chat_table)}" - ) - wandb.log( - {"question_answers_db": wandb.Table(dataframe=chat_table)} + chat_router.chat = chat_router.Chat(chat_router.chat_config) + database_router.db_client = database_router.DatabaseClient() + retrieve_router.retriever = chat_router.chat.retriever + + async def backup_db(): + """Periodically backs up the database to a table. + + This function runs periodically and retrieves all question-answer threads from the database since the last backup. + It then creates a pandas DataFrame from the retrieved threads and logs it to a table using Weights & Biases. + The last backup timestamp is updated after each backup. + + Returns: + None + """ + global last_backup + while True: + chat_threads = database_router.db_client.get_all_question_answers( + last_backup ) - await asyncio.sleep(600) - - -@app.on_event("startup") -def startup_event(): - """Handles the startup event. - - This function initializes the chat and database client objects and creates a task to backup the database. - - Returns: - None - """ - global chat, db_client - chat = Chat(ChatConfig()) - db_client = DatabaseClient() - asyncio.create_task(backup_db()) - - -@app.post( - "/question_answer", - response_model=APIQuestionAnswerResponse, - status_code=status.HTTP_201_CREATED, -) -async def create_question_answer( - request: APIQuestionAnswerRequest, response: Response -) -> APIQuestionAnswerResponse | None: - """Creates a question answer. - - Args: - request: The request object containing the question answer data. - response: The response object to update with the result. - - Returns: - The created question answer or None if creation failed. - """ - question_answer = db_client.create_question_answer(request) - if question_answer is None: - response.status_code = status.HTTP_400_BAD_REQUEST - return question_answer - - -@app.get( - "/chat_thread/{application}/{thread_id}", - response_model=APIGetChatThreadResponse | None, - status_code=status.HTTP_200_OK, -) -async def get_chat_thread( - application: str, thread_id: str, response: Response -) -> APIGetChatThreadResponse: - """Retrieves a chat thread from the database. - - If the chat thread does not exist, it creates a new chat thread. - - Args: - application: The application name. - thread_id: The ID of the chat thread. - response: The HTTP response object. - - Returns: - The retrieved or created chat thread. - """ - chat_thread = db_client.get_chat_thread( - application=application, - thread_id=thread_id, - ) - if chat_thread is None: - chat_thread = db_client.create_chat_thread( - APICreateChatThreadRequest( - application=application, - thread_id=thread_id, - ) - ) - response.status_code = status.HTTP_201_CREATED - if chat_thread is None: - response.status_code = status.HTTP_400_BAD_REQUEST - return chat_thread - - -@app.post( - "/query", response_model=APIQueryResponse, status_code=status.HTTP_200_OK -) -async def query( - request: APIQueryRequest, -) -> APIQueryResponse: - """Executes a query using the chat function and returns the result as an APIQueryResponse. - - Args: - request: The APIQueryRequest object containing the question and chat history. - - Returns: - The APIQueryResponse object containing the result of the query. - """ - result = chat( - ChatRequest( - question=request.question, - chat_history=request.chat_history, - language=request.language, - application=request.application, - ), - ) - result = APIQueryResponse(**result.model_dump()) - - return result - - -@app.post( - "/feedback", - response_model=APIFeedbackResponse | None, - status_code=status.HTTP_201_CREATED, -) -async def feedback( - request: APIFeedbackRequest, response: Response -) -> APIFeedbackResponse: - """Handles the feedback request and logs the feedback data. - - Args: - request: The feedback request object. - response: The response object. - - Returns: - The feedback response object. - """ - feedback_response = db_client.create_feedback(request) - if feedback_response is not None: - wandb.log( - { - "feedback": wandb.Table( - columns=list(request.model_dump().keys()), - data=[list(request.model_dump().values())], + if chat_threads is not None: + chat_table = pd.DataFrame( + [chat_thread for chat_thread in chat_threads] ) - } - ) - else: - response.status_code = status.HTTP_400_BAD_REQUEST - return feedback_response - - -@app.post( - "/retrieve", - response_model=APIRetrievalResponse, - status_code=status.HTTP_200_OK, -) -async def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: - """Retrieves the top k results for a given query. + last_backup = datetime.now().astimezone(timezone.utc) + logger.info( + f"Backing up database to Table at {last_backup}: Number of chat threads: {len(chat_table)}" + ) + wandb.log( + {"question_answers_db": wandb.Table(dataframe=chat_table)} + ) + await asyncio.sleep(600) - Args: - request: The APIRetrievalRequest object containing the query and other parameters. + _ = asyncio.create_task(backup_db()) + yield + if wandb.run is not None: + wandb.run.finish() - Returns: - The APIRetrievalResponse object containing the query and top k results. - """ - results = chat.retriever( - query=request.query, - language=request.language, - top_k=request.top_k, - include_tags=request.include_tags, - exclude_tags=request.exclude_tags, - ) - return APIRetrievalResponse( - query=request.query, - top_k=[ - APIRetrievalResult( - text=result["text"], - score=result["score"], - metadata=result["metadata"], - ) - for result in results - ], - ) +app = FastAPI(name="wandbot", version="1.0.0", lifespan=lifespan) -@app.on_event("shutdown") -def shutdown_event(): - """Finish the current run if wandb.run is not None. - - Returns: - None - """ - if wandb.run is not None: - wandb.run.finish() +app.include_router(chat_router.router) +app.include_router(database_router.router) +app.include_router(retrieve_router.router) if __name__ == "__main__": diff --git a/src/wandbot/api/routers/__init__.py b/src/wandbot/api/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/wandbot/api/routers/chat.py b/src/wandbot/api/routers/chat.py new file mode 100644 index 0000000..4ca3133 --- /dev/null +++ b/src/wandbot/api/routers/chat.py @@ -0,0 +1,43 @@ +from fastapi import APIRouter +from starlette import status +from wandbot.api.schemas import APIQueryRequest, APIQueryResponse +from wandbot.chat.chat import Chat, ChatConfig +from wandbot.chat.schemas import ChatRequest +from wandbot.utils import get_logger + +logger = get_logger(__name__) + +chat_config = ChatConfig() +chat: Chat | None = None + +router = APIRouter( + prefix="/chat", + tags=["chat"], +) + + +@router.post( + "/query", response_model=APIQueryResponse, status_code=status.HTTP_200_OK +) +async def query( + request: APIQueryRequest, +) -> APIQueryResponse: + """Executes a query using the chat function and returns the result as an APIQueryResponse. + + Args: + request: The APIQueryRequest object containing the question and chat history. + + Returns: + The APIQueryResponse object containing the result of the query. + """ + result = chat( + ChatRequest( + question=request.question, + chat_history=request.chat_history, + language=request.language, + application=request.application, + ), + ) + result = APIQueryResponse(**result.model_dump()) + + return result diff --git a/src/wandbot/api/routers/database.py b/src/wandbot/api/routers/database.py new file mode 100644 index 0000000..0997b35 --- /dev/null +++ b/src/wandbot/api/routers/database.py @@ -0,0 +1,119 @@ +import wandb +from fastapi import APIRouter +from starlette import status +from starlette.responses import Response +from wandbot.api.schemas import ( + APICreateChatThreadRequest, + APIFeedbackRequest, + APIFeedbackResponse, + APIGetChatThreadResponse, + APIQuestionAnswerRequest, + APIQuestionAnswerResponse, +) +from wandbot.database.client import DatabaseClient +from wandbot.database.database import engine +from wandbot.database.models import Base +from wandbot.utils import get_logger + +logger = get_logger(__name__) + +Base.metadata.create_all(bind=engine) + +db_client: DatabaseClient | None = None + +router = APIRouter( + prefix="/data", + tags=["database"], +) + + +@router.post( + "/question_answer", + response_model=APIQuestionAnswerResponse, + status_code=status.HTTP_201_CREATED, +) +async def create_question_answer( + request: APIQuestionAnswerRequest, response: Response +) -> APIQuestionAnswerResponse | None: + """Creates a question answer. + + Args: + request: The request object containing the question answer data. + response: The response object to update with the result. + + Returns: + The created question answer or None if creation failed. + """ + question_answer = db_client.create_question_answer(request) + if question_answer is None: + response.status_code = status.HTTP_400_BAD_REQUEST + return question_answer + + +@router.get( + "/chat_thread/{application}/{thread_id}", + response_model=APIGetChatThreadResponse | None, + status_code=status.HTTP_200_OK, +) +async def get_chat_thread( + application: str, thread_id: str, response: Response +) -> APIGetChatThreadResponse: + """Retrieves a chat thread from the database. + + If the chat thread does not exist, it creates a new chat thread. + + Args: + application: The application name. + thread_id: The ID of the chat thread. + response: The HTTP response object. + + Returns: + The retrieved or created chat thread. + """ + chat_thread = db_client.get_chat_thread( + application=application, + thread_id=thread_id, + ) + if chat_thread is None: + chat_thread = db_client.create_chat_thread( + APICreateChatThreadRequest( + application=application, + thread_id=thread_id, + ) + ) + response.status_code = status.HTTP_201_CREATED + if chat_thread is None: + response.status_code = status.HTTP_400_BAD_REQUEST + return chat_thread + + +@router.post( + "/feedback", + response_model=APIFeedbackResponse | None, + status_code=status.HTTP_201_CREATED, +) +async def feedback( + request: APIFeedbackRequest, response: Response +) -> APIFeedbackResponse: + """Handles the feedback request and logs the feedback data. + + Args: + request: The feedback request object. + response: The response object. + + Returns: + The feedback response object. + """ + feedback_response = db_client.create_feedback(request) + if feedback_response is not None: + wandb.log( + { + "feedback": wandb.Table( + columns=list(request.model_dump().keys()), + data=[list(request.model_dump().values())], + ) + } + ) + else: + response.status_code = status.HTTP_400_BAD_REQUEST + return feedback_response diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py new file mode 100644 index 0000000..ec13541 --- /dev/null +++ b/src/wandbot/api/routers/retrieve.py @@ -0,0 +1,50 @@ +from fastapi import APIRouter +from starlette import status +from wandbot.api.schemas import ( + APIRetrievalRequest, + APIRetrievalResponse, + APIRetrievalResult, +) +from wandbot.retriever.base import Retriever + +router = APIRouter( + prefix="/retrieve", + tags=["retrievers"], +) + +retriever: Retriever | None = None + + +@router.post( + "/", + response_model=APIRetrievalResponse, + status_code=status.HTTP_200_OK, +) +async def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: + """Retrieves the top k results for a given query. + + Args: + request: The APIRetrievalRequest object containing the query and other parameters. + + Returns: + The APIRetrievalResponse object containing the query and top k results. + """ + results = retriever( + query=request.query, + language=request.language, + top_k=request.top_k, + include_tags=request.include_tags, + exclude_tags=request.exclude_tags, + ) + + return APIRetrievalResponse( + query=request.query, + top_k=[ + APIRetrievalResult( + text=result["text"], + score=result["score"], + metadata=result["metadata"], + ) + for result in results + ], + ) diff --git a/src/wandbot/database/client.py b/src/wandbot/database/client.py index e3dcfd7..d5097d5 100644 --- a/src/wandbot/database/client.py +++ b/src/wandbot/database/client.py @@ -17,16 +17,13 @@ from sqlalchemy.future import create_engine from sqlalchemy.orm import sessionmaker - from wandbot.database.config import DataBaseConfig from wandbot.database.models import ChatThread as ChatThreadModel from wandbot.database.models import FeedBack as FeedBackModel from wandbot.database.models import QuestionAnswer as QuestionAnswerModel from wandbot.database.schemas import ChatThreadCreate as ChatThreadCreateSchema from wandbot.database.schemas import Feedback as FeedbackSchema -from wandbot.database.schemas import ( - QuestionAnswerCreate as QuestionAnswerCreateSchema, -) +from wandbot.database.schemas import QuestionAnswerCreate as QuestionAnswerCreateSchema from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index fc689fb..57dc9e8 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -8,9 +8,11 @@ load_indices_from_storage, ) from llama_index.callbacks import CallbackManager +from llama_index.core.base_retriever import BaseRetriever from llama_index.postprocessor import BaseNodePostprocessor, CohereRerank from llama_index.query_engine import RetrieverQueryEngine from llama_index.response_synthesizers import BaseSynthesizer, ResponseMode +from llama_index.retrievers.fusion_retriever import FUSION_MODES from llama_index.schema import NodeWithScore from llama_index.vector_stores.simple import DEFAULT_VECTOR_STORE, NAMESPACE_SEP from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME @@ -85,7 +87,15 @@ def __init__( self.config = ( config if isinstance(config, RetrieverConfig) else RetrieverConfig() ) - self.run = run + self.run = ( + run + if run + else wandb.init( + project=self.config.wandb_project, + entity=self.config.wandb_entity, + job_type="retrieve", + ) + ) self.service_context = ( service_context if service_context @@ -120,8 +130,12 @@ def __init__( retriever_list, similarity_top_k=self.config.similarity_top_k, num_queries=1, - use_async=False, + use_async=True, + mode=FUSION_MODES.RECIPROCAL_RANK, ) + + self._retriever_map = dict(zip(index_ids, retriever_list)) + self.is_avoid_query: bool | None = None def load_storage_context_from_artifact( @@ -147,6 +161,7 @@ def load_storage_context_from_artifact( def load_query_engine( self, + retriever: BaseRetriever | None = None, top_k: int | None = None, language: str | None = None, include_tags: List[str] | None = None, @@ -173,16 +188,17 @@ def load_query_engine( else CohereRerank(top_n=top_k, model="rerank-multilingual-v2.0"), ] query_engine = WandbRetrieverQueryEngine.from_args( - retriever=self._retriever, + retriever=retriever, node_postprocessors=node_postprocessors, response_mode=ResponseMode.NO_TEXT, service_context=self.service_context, ) return query_engine - def retrieve( + def _retrieve( self, query: str, + index: str | None = None, language: str | None = None, top_k: int | None = None, include_tags: List[str] | None = None, @@ -193,6 +209,7 @@ def retrieve( Args: query: A string representing the query. + index: A string representing the index to retrieve the results from. language: A string representing the language of the query. top_k: An integer representing the number of top results to retrieve. include_tags: A list of strings representing the tags to include in the results. @@ -203,8 +220,16 @@ def retrieve( """ top_k = top_k or self.config.top_k language = language or self.config.language + retriever = self._retriever_map.get(index) + if not retriever: + if index: + logger.warning( + f"Index {index} not found in retriever map. Defaulting to main retriever." + ) + retriever = self._retriever retrieval_engine = self.load_query_engine( + retriever=retriever, top_k=top_k, language=language, include_tags=include_tags, @@ -234,8 +259,77 @@ def retrieve( self.is_avoid_query = None return outputs + def retrieve( + self, + query: str, + language: str | None = None, + top_k: int | None = None, + include_tags: List[str] | None = None, + exclude_tags: List[str] | None = None, + is_avoid_query: bool | None = False, + ): + """Retrieves the top k results from the index for the given query. + + Args: + query: A string representing the query. + language: A string representing the language of the query. + top_k: An integer representing the number of top results to retrieve. + include_tags: A list of strings representing the tags to include in the results. + exclude_tags: A list of strings representing the tags to exclude from the results. + + Returns: + A list of dictionaries representing the retrieved results. + """ + + return self._retrieve( + query, + index=None, + language=language, + top_k=top_k, + include_tags=include_tags, + exclude_tags=exclude_tags, + is_avoid_query=is_avoid_query, + ) + + def retrieve_from_index( + self, + query: str, + index: str, + language: str | None = None, + top_k: int | None = None, + include_tags: List[str] | None = None, + exclude_tags: List[str] | None = None, + is_avoid_query: bool | None = False, + ): + """Retrieves the top k results from the index for the given query. + + Args: + query: A string representing the query. + index: A string representing the index to retrieve the results from. + language: A string representing the language of the query. + top_k: An integer representing the number of top results to retrieve. + include_tags: A list of strings representing the tags to include in the results. + exclude_tags: A list of strings representing the tags to exclude from the results. + + Returns: + A list of dictionaries representing the retrieved results. + """ + return self._retrieve( + query, + index=index, + language=language, + top_k=top_k, + include_tags=include_tags, + exclude_tags=exclude_tags, + is_avoid_query=is_avoid_query, + ) + def __call__(self, query: str, **kwargs) -> List[Dict[str, Any]]: - retrievals = self.retrieve(query, **kwargs) + if "index" in kwargs: + retrievals = self.retrieve_from_index(query, **kwargs) + else: + retrievals = self.retrieve(query, **kwargs) + logger.debug(f"Retrieved {len(retrievals)} results.") logger.debug(f"Retrieval: {retrievals[0]}") return retrievals diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index 5fc78a5..c284987 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -1,6 +1,8 @@ +import asyncio import os from typing import Dict, List, Optional, Tuple +import nest_asyncio from llama_index import QueryBundle from llama_index.callbacks import CallbackManager, CBEventType, EventPayload from llama_index.constants import DEFAULT_SIMILARITY_TOP_K @@ -10,7 +12,7 @@ from llama_index.retrievers.fusion_retriever import FUSION_MODES from llama_index.schema import IndexNode, NodeWithScore, QueryType from wandbot.retriever.external import YouRetriever -from wandbot.utils import get_logger +from wandbot.utils import get_logger, run_async_tasks logger = get_logger(__name__) @@ -40,10 +42,14 @@ def __init__( super().__init__() def _retrieve(self, query: QueryBundle, **kwargs): - bm25_nodes = self.bm25_retriever.retrieve(query) - vector_nodes = self.vector_retriever.retrieve(query) + nest_asyncio.apply() + return asyncio.run(self._aretrieve(query, **kwargs)) + + async def _aretrieve(self, query: QueryBundle, **kwargs): + bm25_nodes = await self.bm25_retriever.aretrieve(query) + vector_nodes = await self.vector_retriever.aretrieve(query) you_nodes = ( - self.you_retriever.retrieve(query) + await self.you_retriever.aretrieve(query) if not kwargs.get("is_avoid_query", False) else [] ) @@ -59,6 +65,12 @@ def _retrieve(self, query: QueryBundle, **kwargs): def retrieve( self, str_or_query_bundle: QueryType, **kwargs + ) -> List[NodeWithScore]: + nest_asyncio.apply() + return asyncio.run(self.aretrieve(str_or_query_bundle, **kwargs)) + + async def aretrieve( + self, str_or_query_bundle: QueryType, **kwargs ) -> List[NodeWithScore]: self._check_callback_manager() @@ -71,7 +83,7 @@ def retrieve( CBEventType.RETRIEVE, payload={EventPayload.QUERY_STR: query_bundle.query_str}, ) as retrieve_event: - nodes = self._retrieve(query_bundle, **kwargs) + nodes = await self._aretrieve(query_bundle, **kwargs) retrieve_event.on_end( payload={EventPayload.NODES: nodes}, ) @@ -108,6 +120,25 @@ def __init__( ) self._retrievers = retrievers + def _run_nested_async_queries( + self, queries: List[QueryBundle], **kwargs + ) -> Dict[Tuple[str, int], List[NodeWithScore]]: + tasks, task_queries = [], [] + for query in queries: + for i, retriever in enumerate(self._retrievers): + tasks.append(retriever.aretrieve(query, **kwargs)) + task_queries.append(query) + + task_results = run_async_tasks(tasks) + + results = {} + for i, (query, query_result) in enumerate( + zip(task_queries, task_results) + ): + results[(query.query_str, i)] = query_result + + return results + def _run_sync_queries( self, queries: List[QueryBundle], **kwargs ) -> Dict[Tuple[str, int], List[NodeWithScore]]: @@ -120,6 +151,25 @@ def _run_sync_queries( return results + async def _run_async_queries( + self, queries: List[QueryBundle], **kwargs + ) -> Dict[Tuple[str, int], List[NodeWithScore]]: + tasks, task_queries = [], [] + for query in queries: + for i, retriever in enumerate(self._retrievers): + tasks.append(retriever.aretrieve(query, **kwargs)) + task_queries.append(query) + + task_results = await asyncio.gather(*tasks) + + results = {} + for i, (query, query_result) in enumerate( + zip(task_queries, task_results) + ): + results[(query.query_str, i)] = query_result + + return results + def _retrieve( self, query_bundle: QueryBundle, **kwargs ) -> List[NodeWithScore]: @@ -142,6 +192,25 @@ def _retrieve( else: raise ValueError(f"Invalid fusion mode: {self.mode}") + async def _aretrieve( + self, query_bundle: QueryBundle, **kwargs + ) -> List[NodeWithScore]: + if self.num_queries > 1: + queries = self._get_queries(query_bundle.query_str) + else: + queries = [query_bundle] + + results = await self._run_async_queries(queries, **kwargs) + + if self.mode == FUSION_MODES.RECIPROCAL_RANK: + return self._reciprocal_rerank_fusion(results)[ + : self.similarity_top_k + ] + elif self.mode == FUSION_MODES.SIMPLE: + return self._simple_fusion(results)[: self.similarity_top_k] + else: + raise ValueError(f"Invalid fusion mode: {self.mode}") + def retrieve( self, str_or_query_bundle: QueryType, **kwargs ) -> List[NodeWithScore]: @@ -161,3 +230,27 @@ def retrieve( payload={EventPayload.NODES: nodes}, ) return nodes + + async def aretrieve( + self, str_or_query_bundle: QueryType, **kwargs + ) -> List[NodeWithScore]: + self._check_callback_manager() + + if isinstance(str_or_query_bundle, str): + query_bundle = QueryBundle(str_or_query_bundle) + else: + query_bundle = str_or_query_bundle + with self.callback_manager.as_trace("query"): + with self.callback_manager.event( + CBEventType.RETRIEVE, + payload={EventPayload.QUERY_STR: query_bundle.query_str}, + ) as retrieve_event: + nodes = await self._aretrieve(query_bundle, **kwargs) + nodes = await self._ahandle_recursive_retrieval( + query_bundle, nodes + ) + retrieve_event.on_end( + payload={EventPayload.NODES: nodes}, + ) + + return nodes diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 1d7ad1e..3238906 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -20,6 +20,7 @@ storage_context = load_storage_context(768, "/path/to/persist") index = load_index(nodes, service_context, storage_context, "/path/to/persist") """ +import asyncio import datetime import hashlib import json @@ -27,17 +28,13 @@ import os import pathlib import sqlite3 -from typing import Any, List, Optional +from typing import Any, Coroutine, List, Optional, Tuple import faiss import fasttext +import nest_asyncio import wandb -from llama_index import ( - ServiceContext, - StorageContext, - VectorStoreIndex, - load_indices_from_storage, -) +from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.embeddings import OpenAIEmbedding from llama_index.llms import LiteLLM from llama_index.llms.llm import LLM @@ -326,4 +323,39 @@ def _load_model(self): return self._model -load_indices_from_storage +def run_async_tasks( + tasks: List[Coroutine], + show_progress: bool = False, + progress_bar_desc: str = "Running async tasks", +) -> Tuple[Any]: + """Run a list of async tasks.""" + tasks_to_execute: List[Any] = tasks + + nest_asyncio.apply() + if show_progress: + try: + from tqdm.asyncio import tqdm + + # jupyter notebooks already have an event loop running + # we need to reuse it instead of creating a new one + + loop = asyncio.get_event_loop() + + async def _tqdm_gather() -> List[Any]: + return await tqdm.gather( + *tasks_to_execute, desc=progress_bar_desc + ) + + tqdm_outputs: Tuple[Any] = loop.run_until_complete(_tqdm_gather()) + return tqdm_outputs + # run the operation w/o tqdm on hitting a fatal + # may occur in some environments where tqdm.asyncio + # is not supported + except Exception: + pass + + async def _gather() -> Tuple[Any]: + return await asyncio.gather(*tasks_to_execute) + + outputs: Tuple[Any] = asyncio.run(_gather()) + return outputs From be5b8c2e16f760b1a48ce7f0320fa0a66a03f081 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 2 Feb 2024 12:21:53 +0530 Subject: [PATCH 07/41] feat: add retriever routing and params --- src/wandbot/api/client.py | 8 +-- src/wandbot/api/routers/chat.py | 11 +++- src/wandbot/api/routers/database.py | 44 ++++++++++++--- src/wandbot/api/routers/retrieve.py | 49 ++++++++++++++-- src/wandbot/api/schemas.py | 86 ----------------------------- src/wandbot/apps/utils.py | 3 +- src/wandbot/ingestion/config.py | 23 ++++---- src/wandbot/retriever/base.py | 44 ++++++++++----- 8 files changed, 136 insertions(+), 132 deletions(-) delete mode 100644 src/wandbot/api/schemas.py diff --git a/src/wandbot/api/client.py b/src/wandbot/api/client.py index cb32f67..6354249 100644 --- a/src/wandbot/api/client.py +++ b/src/wandbot/api/client.py @@ -15,16 +15,16 @@ import aiohttp import requests - -from wandbot.api.schemas import ( +from wandbot.api.routers.chat import APIQueryRequest, APIQueryResponse +from wandbot.api.routers.database import ( APIFeedbackRequest, APIFeedbackResponse, APIGetChatThreadRequest, APIGetChatThreadResponse, - APIQueryRequest, - APIQueryResponse, APIQuestionAnswerRequest, APIQuestionAnswerResponse, +) +from wandbot.api.routers.retrieve import ( APIRetrievalRequest, APIRetrievalResponse, ) diff --git a/src/wandbot/api/routers/chat.py b/src/wandbot/api/routers/chat.py index 4ca3133..72aa9d4 100644 --- a/src/wandbot/api/routers/chat.py +++ b/src/wandbot/api/routers/chat.py @@ -1,8 +1,7 @@ from fastapi import APIRouter from starlette import status -from wandbot.api.schemas import APIQueryRequest, APIQueryResponse from wandbot.chat.chat import Chat, ChatConfig -from wandbot.chat.schemas import ChatRequest +from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.utils import get_logger logger = get_logger(__name__) @@ -16,6 +15,14 @@ ) +class APIQueryRequest(ChatRequest): + pass + + +class APIQueryResponse(ChatResponse): + pass + + @router.post( "/query", response_model=APIQueryResponse, status_code=status.HTTP_200_OK ) diff --git a/src/wandbot/api/routers/database.py b/src/wandbot/api/routers/database.py index 0997b35..a625953 100644 --- a/src/wandbot/api/routers/database.py +++ b/src/wandbot/api/routers/database.py @@ -2,17 +2,17 @@ from fastapi import APIRouter from starlette import status from starlette.responses import Response -from wandbot.api.schemas import ( - APICreateChatThreadRequest, - APIFeedbackRequest, - APIFeedbackResponse, - APIGetChatThreadResponse, - APIQuestionAnswerRequest, - APIQuestionAnswerResponse, -) from wandbot.database.client import DatabaseClient from wandbot.database.database import engine from wandbot.database.models import Base +from wandbot.database.schemas import ( + ChatThread, + ChatThreadCreate, + Feedback, + FeedbackCreate, + QuestionAnswer, + QuestionAnswerCreate, +) from wandbot.utils import get_logger logger = get_logger(__name__) @@ -27,6 +27,14 @@ ) +class APIQuestionAnswerRequest(QuestionAnswerCreate): + pass + + +class APIQuestionAnswerResponse(QuestionAnswer): + pass + + @router.post( "/question_answer", response_model=APIQuestionAnswerResponse, @@ -50,6 +58,10 @@ async def create_question_answer( return question_answer +class APIGetChatThreadResponse(ChatThread): + pass + + @router.get( "/chat_thread/{application}/{thread_id}", response_model=APIGetChatThreadResponse | None, @@ -87,6 +99,14 @@ async def get_chat_thread( return chat_thread +class APIFeedbackRequest(FeedbackCreate): + pass + + +class APIFeedbackResponse(Feedback): + pass + + @router.post( "/feedback", response_model=APIFeedbackResponse | None, @@ -117,3 +137,11 @@ async def feedback( else: response.status_code = status.HTTP_400_BAD_REQUEST return feedback_response + + +class APIGetChatThreadRequest(ChatThreadCreate): + pass + + +class APICreateChatThreadRequest(ChatThreadCreate): + pass diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py index ec13541..0652778 100644 --- a/src/wandbot/api/routers/retrieve.py +++ b/src/wandbot/api/routers/retrieve.py @@ -1,10 +1,9 @@ +from enum import Enum +from typing import Any, List + from fastapi import APIRouter +from pydantic import BaseModel from starlette import status -from wandbot.api.schemas import ( - APIRetrievalRequest, - APIRetrievalResponse, - APIRetrievalResult, -) from wandbot.retriever.base import Retriever router = APIRouter( @@ -15,6 +14,43 @@ retriever: Retriever | None = None +class APIRetrievalResult(BaseModel): + text: str + score: float + metadata: dict[str, Any] + + +class APIRetrievalResponse(BaseModel): + query: str + top_k: List[APIRetrievalResult] + + +class Indices(str, Enum): + """The indices available for retrieval.""" + + DOCODILE_EN = "docodile_en" + DOCODILE_JA = "docodile_ja" + WANDB_EXAMPLES_CODE = "wandb_examples_code" + WANDB_EXAMPLES_COLAB = "wandb_examples_colab" + WANDB_SDK_CODE = "wandb_sdk_code" + WANDB_SDK_TESTS = "wandb_sdk_tests" + WEAVE_SDK_CODE = "weave_sdk_code" + WEAVE_EXAMPLES = "weave_examples" + WANDB_EDU_CODE = "wandb_edu_code" + WEAVE_JS = "weave_js" + FC_REPORTS = "fc_reports" + + +class APIRetrievalRequest(BaseModel): + query: str + indices: List[Indices] | None = None + language: str = "en" + initial_k: int = 10 + top_k: int = 5 + include_tags: List[str] = [] + exclude_tags: List[str] = [] + + @router.post( "/", response_model=APIRetrievalResponse, @@ -31,6 +67,9 @@ async def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: """ results = retriever( query=request.query, + indices=[idx.value for idx in request.indices] + if request.indices + else None, language=request.language, top_k=request.top_k, include_tags=request.include_tags, diff --git a/src/wandbot/api/schemas.py b/src/wandbot/api/schemas.py deleted file mode 100644 index c280636..0000000 --- a/src/wandbot/api/schemas.py +++ /dev/null @@ -1,86 +0,0 @@ -"""A module for API schemas. - -This module provides the schemas for API requests and responses. -It includes classes for creating question answers, getting chat threads, -creating chat threads, querying, creating feedback, and more. - -Classes: - APIQuestionAnswerRequest: Request schema for creating a question answer. - APIQuestionAnswerResponse: Response schema for a question answer. - APIGetChatThreadRequest: Request schema for getting a chat thread. - APIGetChatThreadResponse: Response schema for a chat thread. - APICreateChatThreadRequest: Request schema for creating a chat thread. - APIQueryRequest: Request schema for querying. - APIQueryResponse: Response schema for a query. - APIFeedbackRequest: Request schema for creating feedback. - APIFeedbackResponse: Response schema for feedback. -""" -from typing import Any, List - -from pydantic import BaseModel - -from wandbot.chat.schemas import ChatRequest, ChatResponse -from wandbot.database.schemas import ( - ChatThread, - ChatThreadCreate, - Feedback, - FeedbackCreate, - QuestionAnswer, - QuestionAnswerCreate, -) - - -class APIQuestionAnswerRequest(QuestionAnswerCreate): - pass - - -class APIQuestionAnswerResponse(QuestionAnswer): - pass - - -class APIGetChatThreadRequest(ChatThreadCreate): - pass - - -class APIGetChatThreadResponse(ChatThread): - pass - - -class APICreateChatThreadRequest(ChatThreadCreate): - pass - - -class APIQueryRequest(ChatRequest): - pass - - -class APIQueryResponse(ChatResponse): - pass - - -class APIFeedbackRequest(FeedbackCreate): - pass - - -class APIFeedbackResponse(Feedback): - pass - - -class APIRetrievalResult(BaseModel): - text: str - score: float - metadata: dict[str, Any] - - -class APIRetrievalResponse(BaseModel): - query: str - top_k: List[APIRetrievalResult] - - -class APIRetrievalRequest(BaseModel): - query: str - language: str = "en" - initial_k: int = 10 - top_k: int = 5 - include_tags: List[str] = [] - exclude_tags: List[str] = [] diff --git a/src/wandbot/apps/utils.py b/src/wandbot/apps/utils.py index e46ba85..bdf5762 100644 --- a/src/wandbot/apps/utils.py +++ b/src/wandbot/apps/utils.py @@ -16,8 +16,7 @@ from typing import Any, List from pydantic_settings import BaseSettings - -from wandbot.api.schemas import APIQueryResponse +from wandbot.api.routers.chat import APIQueryResponse def deduplicate(input_list: List[Any]) -> List[Any]: diff --git a/src/wandbot/ingestion/config.py b/src/wandbot/ingestion/config.py index 5148016..1e9b90c 100644 --- a/src/wandbot/ingestion/config.py +++ b/src/wandbot/ingestion/config.py @@ -17,7 +17,6 @@ from pydantic import BaseModel, Field, model_validator from pydantic_settings import BaseSettings - from wandbot.utils import get_logger logger = get_logger(__name__) @@ -82,7 +81,7 @@ class DocodileEnglishStoreConfig(DataStoreConfig): is_git_repo=True, ) language: str = "en" - docstore_dir: pathlib.Path = pathlib.Path("docstore_en") + docstore_dir: pathlib.Path = pathlib.Path("docodile_en") class DocodileJapaneseStoreConfig(DataStoreConfig): @@ -95,7 +94,7 @@ class DocodileJapaneseStoreConfig(DataStoreConfig): is_git_repo=True, ) language: str = "ja" - docstore_dir: pathlib.Path = pathlib.Path("docstore_ja") + docstore_dir: pathlib.Path = pathlib.Path("docodile_ja") class ExampleCodeStoreConfig(DataStoreConfig): @@ -107,7 +106,7 @@ class ExampleCodeStoreConfig(DataStoreConfig): file_pattern="*.py", is_git_repo=True, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_example_code") + docstore_dir: pathlib.Path = pathlib.Path("wandb_examples_code") class ExampleNotebookStoreConfig(DataStoreConfig): @@ -119,7 +118,7 @@ class ExampleNotebookStoreConfig(DataStoreConfig): file_pattern="*.ipynb", is_git_repo=True, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_example_colab") + docstore_dir: pathlib.Path = pathlib.Path("wandb_examples_colab") class SDKCodeStoreConfig(DataStoreConfig): @@ -131,7 +130,7 @@ class SDKCodeStoreConfig(DataStoreConfig): file_pattern="*.py", is_git_repo=True, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_sdk_code") + docstore_dir: pathlib.Path = pathlib.Path("wandb_sdk_code") class SDKTestsStoreConfig(DataStoreConfig): @@ -143,7 +142,7 @@ class SDKTestsStoreConfig(DataStoreConfig): file_pattern="*.py", is_git_repo=True, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_sdk_tests") + docstore_dir: pathlib.Path = pathlib.Path("wandb_sdk_tests") class WeaveCodeStoreConfig(DataStoreConfig): @@ -155,7 +154,7 @@ class WeaveCodeStoreConfig(DataStoreConfig): file_pattern=["*.py", "*.ipynb"], is_git_repo=True, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_weave_code") + docstore_dir: pathlib.Path = pathlib.Path("weave_sdk_code") class WeaveExamplesStoreConfig(DataStoreConfig): @@ -167,7 +166,7 @@ class WeaveExamplesStoreConfig(DataStoreConfig): file_pattern=["*.py", "*.ipynb"], is_git_repo=True, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_weave_examples") + docstore_dir: pathlib.Path = pathlib.Path("weave_examples") class WandbEduCodeStoreConfig(DataStoreConfig): @@ -179,7 +178,7 @@ class WandbEduCodeStoreConfig(DataStoreConfig): file_pattern=["*.py", "*.ipynb", ".*md"], is_git_repo=True, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_wandb_edu") + docstore_dir: pathlib.Path = pathlib.Path("wandb_edu_code") class WeaveJsStoreConfig(DataStoreConfig): @@ -191,7 +190,7 @@ class WeaveJsStoreConfig(DataStoreConfig): file_pattern=["*.js", "*.ts"], is_git_repo=True, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_weave_js") + docstore_dir: pathlib.Path = pathlib.Path("weave_js") class FCReportsStoreConfig(DataStoreConfig): @@ -203,7 +202,7 @@ class FCReportsStoreConfig(DataStoreConfig): file_pattern=["*.json"], is_git_repo=False, ) - docstore_dir: pathlib.Path = pathlib.Path("docstore_fc_reports") + docstore_dir: pathlib.Path = pathlib.Path("fc_reports") @model_validator(mode="after") def _set_cache_paths(cls, values: "DataStoreConfig") -> "DataStoreConfig": diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index 57dc9e8..c99fd74 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -198,7 +198,7 @@ def load_query_engine( def _retrieve( self, query: str, - index: str | None = None, + indices: List[str] | None = None, language: str | None = None, top_k: int | None = None, include_tags: List[str] | None = None, @@ -209,7 +209,7 @@ def _retrieve( Args: query: A string representing the query. - index: A string representing the index to retrieve the results from. + indices: A list of strings representing the indices to retrieve the results from. language: A string representing the language of the query. top_k: An integer representing the number of top results to retrieve. include_tags: A list of strings representing the tags to include in the results. @@ -220,13 +220,28 @@ def _retrieve( """ top_k = top_k or self.config.top_k language = language or self.config.language - retriever = self._retriever_map.get(index) - if not retriever: - if index: + retrievers = [] + for index in indices or []: + retriever = self._retriever_map.get(index) + if not retriever and index: logger.warning( - f"Index {index} not found in retriever map. Defaulting to main retriever." + f"Index {index} not found in retriever map. Skipping the index" ) + retrievers.append(retriever) + + retrievers = [retriever for retriever in retrievers if retriever] + + if not retrievers: + logger.warning("No retrievers found. Defaulting to all retrievers") retriever = self._retriever + else: + retriever = FusionRetriever( + retrievers, + similarity_top_k=self.config.similarity_top_k, + num_queries=1, + use_async=True, + mode=FUSION_MODES.RECIPROCAL_RANK, + ) retrieval_engine = self.load_query_engine( retriever=retriever, @@ -283,7 +298,7 @@ def retrieve( return self._retrieve( query, - index=None, + indices=None, language=language, top_k=top_k, include_tags=include_tags, @@ -291,10 +306,10 @@ def retrieve( is_avoid_query=is_avoid_query, ) - def retrieve_from_index( + def retrieve_from_indices( self, query: str, - index: str, + indices: List[str], language: str | None = None, top_k: int | None = None, include_tags: List[str] | None = None, @@ -305,7 +320,7 @@ def retrieve_from_index( Args: query: A string representing the query. - index: A string representing the index to retrieve the results from. + indices: A string representing the index to retrieve the results from. language: A string representing the language of the query. top_k: An integer representing the number of top results to retrieve. include_tags: A list of strings representing the tags to include in the results. @@ -316,7 +331,7 @@ def retrieve_from_index( """ return self._retrieve( query, - index=index, + indices=indices, language=language, top_k=top_k, include_tags=include_tags, @@ -325,8 +340,11 @@ def retrieve_from_index( ) def __call__(self, query: str, **kwargs) -> List[Dict[str, Any]]: - if "index" in kwargs: - retrievals = self.retrieve_from_index(query, **kwargs) + indices = kwargs.pop("indices") + if indices and isinstance(indices, list): + retrievals = self.retrieve_from_indices( + query, indices=indices, **kwargs + ) else: retrievals = self.retrieve(query, **kwargs) From aced990dbfef3b004b4938810544393b74c79d2e Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 2 Feb 2024 12:41:08 +0530 Subject: [PATCH 08/41] feat: add routes to api client and update operation ids in openapi spec --- src/wandbot/api/app.py | 37 ++++++++++++++++++++++++++++- src/wandbot/api/client.py | 8 +++---- src/wandbot/api/routers/database.py | 2 +- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/wandbot/api/app.py b/src/wandbot/api/app.py index cea5004..b01e244 100644 --- a/src/wandbot/api/app.py +++ b/src/wandbot/api/app.py @@ -35,6 +35,7 @@ import pandas as pd import wandb from fastapi import FastAPI +from fastapi.routing import APIRoute from wandbot.api.routers import chat as chat_router from wandbot.api.routers import database as database_router from wandbot.api.routers import retrieve as retrieve_router @@ -92,7 +93,12 @@ async def backup_db(): wandb.run.finish() -app = FastAPI(name="wandbot", version="1.0.0", lifespan=lifespan) +app = FastAPI( + title="Wandbot", + description="An API to access Wandbot - The Weights & Biases AI Assistant.", + version="1.3.0", + lifespan=lifespan, +) app.include_router(chat_router.router) @@ -100,6 +106,35 @@ async def backup_db(): app.include_router(retrieve_router.router) +def route_to_camel_case(route_name: str) -> str: + """Converts a route name to camel case. + + Args: + route_name: The name of the route. + + Returns: + The route name in camel case. + """ + words = route_name.split("_") + if len(words) == 1: + return words[0].title() + return words[0] + "".join(word.title() for word in words[1:]) + + +def use_route_names_as_operation_ids(app: FastAPI) -> None: + """ + Simplify operation IDs so that generated API clients have simpler function + names. + + Should be called only after all routes have been added. + """ + for route in app.routes: + if isinstance(route, APIRoute): + route.operation_id = route_to_camel_case(route.name) + + +use_route_names_as_operation_ids(app) + if __name__ == "__main__": import uvicorn diff --git a/src/wandbot/api/client.py b/src/wandbot/api/client.py index 6354249..ca4b681 100644 --- a/src/wandbot/api/client.py +++ b/src/wandbot/api/client.py @@ -53,11 +53,11 @@ def __init__(self, url: str): url: The base URL for the API. """ self.url = url - self.query_endpoint = urljoin(str(self.url), "query") - self.feedback_endpoint = urljoin(str(self.url), "feedback") - self.chat_thread_endpoint = urljoin(str(self.url), "chat_thread") + self.query_endpoint = urljoin(str(self.url), "chat/query") + self.feedback_endpoint = urljoin(str(self.url), "data/feedback") + self.chat_thread_endpoint = urljoin(str(self.url), "data/chat_thread") self.chat_question_answer_endpoint = urljoin( - str(self.url), "question_answer" + str(self.url), "data/question_answer" ) self.retrieve_endpoint = urljoin(str(self.url), "retrieve") diff --git a/src/wandbot/api/routers/database.py b/src/wandbot/api/routers/database.py index a625953..5464df6 100644 --- a/src/wandbot/api/routers/database.py +++ b/src/wandbot/api/routers/database.py @@ -23,7 +23,7 @@ router = APIRouter( prefix="/data", - tags=["database"], + tags=["database", "crud"], ) From 477d83b93730d8b01b57022bee1f92f07b33e70c Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 2 Feb 2024 16:47:52 +0530 Subject: [PATCH 09/41] feat: add you retriever separately in the end --- src/wandbot/api/routers/chat.py | 2 +- src/wandbot/api/routers/database.py | 22 ++-- src/wandbot/api/routers/retrieve.py | 8 +- src/wandbot/ingestion/__main__.py | 8 +- src/wandbot/retriever/base.py | 27 +++-- src/wandbot/retriever/external.py | 150 ++++++++++++++++++++-------- src/wandbot/retriever/fusion.py | 33 +++--- 7 files changed, 168 insertions(+), 82 deletions(-) diff --git a/src/wandbot/api/routers/chat.py b/src/wandbot/api/routers/chat.py index 72aa9d4..c51fc03 100644 --- a/src/wandbot/api/routers/chat.py +++ b/src/wandbot/api/routers/chat.py @@ -26,7 +26,7 @@ class APIQueryResponse(ChatResponse): @router.post( "/query", response_model=APIQueryResponse, status_code=status.HTTP_200_OK ) -async def query( +def query( request: APIQueryRequest, ) -> APIQueryResponse: """Executes a query using the chat function and returns the result as an APIQueryResponse. diff --git a/src/wandbot/api/routers/database.py b/src/wandbot/api/routers/database.py index 5464df6..3a5fe32 100644 --- a/src/wandbot/api/routers/database.py +++ b/src/wandbot/api/routers/database.py @@ -40,7 +40,7 @@ class APIQuestionAnswerResponse(QuestionAnswer): response_model=APIQuestionAnswerResponse, status_code=status.HTTP_201_CREATED, ) -async def create_question_answer( +def create_question_answer( request: APIQuestionAnswerRequest, response: Response ) -> APIQuestionAnswerResponse | None: """Creates a question answer. @@ -62,12 +62,20 @@ class APIGetChatThreadResponse(ChatThread): pass +class APIGetChatThreadRequest(ChatThreadCreate): + pass + + +class APICreateChatThreadRequest(ChatThreadCreate): + pass + + @router.get( "/chat_thread/{application}/{thread_id}", response_model=APIGetChatThreadResponse | None, status_code=status.HTTP_200_OK, ) -async def get_chat_thread( +def get_chat_thread( application: str, thread_id: str, response: Response ) -> APIGetChatThreadResponse: """Retrieves a chat thread from the database. @@ -112,7 +120,7 @@ class APIFeedbackResponse(Feedback): response_model=APIFeedbackResponse | None, status_code=status.HTTP_201_CREATED, ) -async def feedback( +def feedback( request: APIFeedbackRequest, response: Response ) -> APIFeedbackResponse: """Handles the feedback request and logs the feedback data. @@ -137,11 +145,3 @@ async def feedback( else: response.status_code = status.HTTP_400_BAD_REQUEST return feedback_response - - -class APIGetChatThreadRequest(ChatThreadCreate): - pass - - -class APICreateChatThreadRequest(ChatThreadCreate): - pass diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py index 0652778..b2e9750 100644 --- a/src/wandbot/api/routers/retrieve.py +++ b/src/wandbot/api/routers/retrieve.py @@ -43,12 +43,13 @@ class Indices(str, Enum): class APIRetrievalRequest(BaseModel): query: str - indices: List[Indices] | None = None + indices: List[Indices] = [] language: str = "en" initial_k: int = 10 top_k: int = 5 include_tags: List[str] = [] exclude_tags: List[str] = [] + include_web_results: bool = True @router.post( @@ -56,7 +57,7 @@ class APIRetrievalRequest(BaseModel): response_model=APIRetrievalResponse, status_code=status.HTTP_200_OK, ) -async def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: +def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: """Retrieves the top k results for a given query. Args: @@ -69,11 +70,12 @@ async def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: query=request.query, indices=[idx.value for idx in request.indices] if request.indices - else None, + else [], language=request.language, top_k=request.top_k, include_tags=request.include_tags, exclude_tags=request.exclude_tags, + include_web_results=request.include_web_results, ) return APIRetrievalResponse( diff --git a/src/wandbot/ingestion/__main__.py b/src/wandbot/ingestion/__main__.py index b911ab0..f2b4e50 100644 --- a/src/wandbot/ingestion/__main__.py +++ b/src/wandbot/ingestion/__main__.py @@ -1,6 +1,7 @@ import os -from wandbot.ingestion import vectorstores +from wandbot.ingestion import prepare_data, vectorstores +from wandbot.ingestion.report import create_ingestion_report from wandbot.utils import get_logger logger = get_logger(__name__) @@ -10,11 +11,10 @@ def main(): project = os.environ.get("WANDB_PROJECT", "wandbot-dev") entity = os.environ.get("WANDB_ENTITY", "wandbot") - # raw_artifact = prepare_data.load(project, entity) - raw_artifact = "wandbot/wandbot-dev/raw_dataset:v28" + raw_artifact = prepare_data.load(project, entity) vectorstore_artifact = vectorstores.load(project, entity, raw_artifact) # TODO: include ingestion report - # create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) + create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) print(vectorstore_artifact) diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index c99fd74..dc2a981 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,3 +1,4 @@ +import os from typing import Any, Dict, List, Optional, Tuple import wandb @@ -18,6 +19,7 @@ from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict +from wandbot.retriever.external import YouRetriever from wandbot.retriever.fusion import FusionRetriever, HybridRetriever from wandbot.retriever.postprocessors import ( LanguageFilterPostprocessor, @@ -126,21 +128,28 @@ def __init__( storage_context=self.storage_context, ) retriever_list.append(retriever) + self.you_retriever = YouRetriever( + api_key=os.environ.get("YOU_API_KEY"), + similarity_top_k=self.config.similarity_top_k, + ) self._retriever = FusionRetriever( retriever_list, - similarity_top_k=self.config.similarity_top_k, + similarity_top_k=self.config.similarity_top_k + * len(retriever_list) + // 2, num_queries=1, use_async=True, mode=FUSION_MODES.RECIPROCAL_RANK, ) - + retriever_list.append(self.you_retriever) + index_ids = index_ids + ["you.com"] self._retriever_map = dict(zip(index_ids, retriever_list)) self.is_avoid_query: bool | None = None def load_storage_context_from_artifact( self, artifact_url: str - ) -> Tuple[StorageContext, Dict[str, str]]: + ) -> Tuple[StorageContext, List[str]]: """Loads the storage context from the given artifact URL. Args: @@ -204,6 +213,7 @@ def _retrieve( include_tags: List[str] | None = None, exclude_tags: List[str] | None = None, is_avoid_query: bool | None = False, + **kwargs, ): """Retrieves the top k results from the index for the given query. @@ -237,7 +247,9 @@ def _retrieve( else: retriever = FusionRetriever( retrievers, - similarity_top_k=self.config.similarity_top_k, + similarity_top_k=self.config.similarity_top_k + * len(retrievers) + // 2, num_queries=1, use_async=True, mode=FUSION_MODES.RECIPROCAL_RANK, @@ -282,6 +294,7 @@ def retrieve( include_tags: List[str] | None = None, exclude_tags: List[str] | None = None, is_avoid_query: bool | None = False, + **kwargs, ): """Retrieves the top k results from the index for the given query. @@ -340,8 +353,10 @@ def retrieve_from_indices( ) def __call__(self, query: str, **kwargs) -> List[Dict[str, Any]]: - indices = kwargs.pop("indices") - if indices and isinstance(indices, list): + indices = kwargs.pop("indices", []) + if kwargs.get("include_web_results"): + indices.append("you.com") + if len(indices) > 1: retrievals = self.retrieve_from_indices( query, indices=indices, **kwargs ) diff --git a/src/wandbot/retriever/external.py b/src/wandbot/retriever/external.py index 7a42917..b455c40 100644 --- a/src/wandbot/retriever/external.py +++ b/src/wandbot/retriever/external.py @@ -3,9 +3,9 @@ import requests from llama_index import QueryBundle -from llama_index.callbacks import CallbackManager +from llama_index.callbacks import CallbackManager, CBEventType, EventPayload from llama_index.core.base_retriever import BaseRetriever -from llama_index.schema import NodeWithScore, TextNode +from llama_index.schema import NodeWithScore, QueryType, TextNode from wandbot.utils import get_logger logger = get_logger(__name__) @@ -27,45 +27,115 @@ def __init__( ) super().__init__(callback_manager) - def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]: + def _retrieve( + self, query_bundle: QueryBundle, **kwargs + ) -> List[NodeWithScore]: """Retrieve.""" - try: - headers = {"X-API-Key": self._api_key} - url = "https://api.ydc-index.io/search" - - querystring = { - "query": "Weights & Biases, W&B, wandb or Weave " - + query_bundle.query_str, - "num_web_results": self.similarity_top_k, - } - response = requests.get(url, headers=headers, params=querystring) - if response.status_code != 200: - return [] - else: - results = response.json() - - snippets = [hit["snippets"] for hit in results["hits"]] - snippet_metadata = [ - { - "source": hit["url"], - "language": "en", - "description": hit["description"], - "title": hit["title"], - "tags": ["you.com"], + if kwargs.get("is_avoid_query", False): + try: + headers = {"X-API-Key": self._api_key} + url = "https://api.ydc-index.io/search" + + querystring = { + "query": "Weights & Biases, W&B, wandb or Weave " + + query_bundle.query_str, + "num_web_results": self.similarity_top_k, } - for hit in results["hits"] - ] - search_hits = [] - for snippet_list, metadata in zip(snippets, snippet_metadata): - for snippet in snippet_list: - search_hits.append((snippet, metadata)) - - return [ - NodeWithScore( - node=TextNode(text=s[0], metadata=s[1]), - score=1.0, + response = requests.get( + url, headers=headers, params=querystring ) - for s in search_hits - ] - except Exception as e: + if response.status_code != 200: + return [] + else: + results = response.json() + + snippets = [hit["snippets"] for hit in results["hits"]] + snippet_metadata = [ + { + "source": hit["url"], + "language": "en", + "description": hit["description"], + "title": hit["title"], + "tags": ["you.com"], + } + for hit in results["hits"] + ] + search_hits = [] + for snippet_list, metadata in zip(snippets, snippet_metadata): + for snippet in snippet_list: + search_hits.append((snippet, metadata)) + + return [ + NodeWithScore( + node=TextNode(text=s[0], metadata=s[1]), + score=1.0, + ) + for s in search_hits + ] + except Exception as e: + return [] + else: return [] + + async def _aretrieve( + self, query_bundle: QueryBundle, **kwargs + ) -> List[NodeWithScore]: + """Asynchronously retrieve nodes given query. + + Implemented by the user. + + """ + return self._retrieve(query_bundle, **kwargs) + + def retrieve( + self, str_or_query_bundle: QueryType, **kwargs + ) -> List[NodeWithScore]: + """Retrieve nodes given query. + + Args: + str_or_query_bundle (QueryType): Either a query string or + a QueryBundle object. + + """ + self._check_callback_manager() + + if isinstance(str_or_query_bundle, str): + query_bundle = QueryBundle(str_or_query_bundle) + else: + query_bundle = str_or_query_bundle + with self.callback_manager.as_trace("query"): + with self.callback_manager.event( + CBEventType.RETRIEVE, + payload={EventPayload.QUERY_STR: query_bundle.query_str}, + ) as retrieve_event: + nodes = self._retrieve(query_bundle, **kwargs) + nodes = self._handle_recursive_retrieval(query_bundle, nodes) + retrieve_event.on_end( + payload={EventPayload.NODES: nodes}, + ) + + return nodes + + async def aretrieve( + self, str_or_query_bundle: QueryType, **kwargs + ) -> List[NodeWithScore]: + self._check_callback_manager() + + if isinstance(str_or_query_bundle, str): + query_bundle = QueryBundle(str_or_query_bundle) + else: + query_bundle = str_or_query_bundle + with self.callback_manager.as_trace("query"): + with self.callback_manager.event( + CBEventType.RETRIEVE, + payload={EventPayload.QUERY_STR: query_bundle.query_str}, + ) as retrieve_event: + nodes = await self._aretrieve(query_bundle, **kwargs) + nodes = await self._ahandle_recursive_retrieval( + query_bundle, nodes + ) + retrieve_event.on_end( + payload={EventPayload.NODES: nodes}, + ) + + return nodes diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index c284987..174d81e 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -1,12 +1,12 @@ import asyncio -import os -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union import nest_asyncio -from llama_index import QueryBundle +from llama_index import QueryBundle, VectorStoreIndex from llama_index.callbacks import CallbackManager, CBEventType, EventPayload from llama_index.constants import DEFAULT_SIMILARITY_TOP_K from llama_index.core.base_retriever import BaseRetriever +from llama_index.indices.base import BaseIndex from llama_index.llms.utils import LLMType from llama_index.retrievers import BM25Retriever, QueryFusionRetriever from llama_index.retrievers.fusion_retriever import FUSION_MODES @@ -20,7 +20,7 @@ class HybridRetriever(BaseRetriever): def __init__( self, - index, + index: Union[VectorStoreIndex, BaseIndex], storage_context, similarity_top_k: int = 20, ): @@ -31,14 +31,14 @@ def __init__( similarity_top_k=similarity_top_k, storage_context=self.storage_context, ) + self.bm25_retriever = BM25Retriever.from_defaults( - docstore=self.index.docstore, - similarity_top_k=similarity_top_k, - ) - self.you_retriever = YouRetriever( - api_key=os.environ.get("YOU_API_KEY"), + nodes=self.index.docstore.get_nodes( + list(self.index.index_struct.nodes_dict.values()) + ), similarity_top_k=similarity_top_k, ) + super().__init__() def _retrieve(self, query: QueryBundle, **kwargs): @@ -48,16 +48,11 @@ def _retrieve(self, query: QueryBundle, **kwargs): async def _aretrieve(self, query: QueryBundle, **kwargs): bm25_nodes = await self.bm25_retriever.aretrieve(query) vector_nodes = await self.vector_retriever.aretrieve(query) - you_nodes = ( - await self.you_retriever.aretrieve(query) - if not kwargs.get("is_avoid_query", False) - else [] - ) # combine the two lists of nodes all_nodes = [] node_ids = set() - for n in bm25_nodes + vector_nodes + you_nodes: + for n in bm25_nodes + vector_nodes: if n.node.node_id not in node_ids: all_nodes.append(n) node_ids.add(n.node.node_id) @@ -93,7 +88,7 @@ async def aretrieve( class FusionRetriever(QueryFusionRetriever): def __init__( self, - retrievers: List[HybridRetriever], + retrievers: List[Union[HybridRetriever, YouRetriever]], llm: Optional[LLMType] = "default", query_gen_prompt: Optional[str] = None, mode: FUSION_MODES = FUSION_MODES.SIMPLE, @@ -125,10 +120,14 @@ def _run_nested_async_queries( ) -> Dict[Tuple[str, int], List[NodeWithScore]]: tasks, task_queries = [], [] for query in queries: - for i, retriever in enumerate(self._retrievers): + for i, retriever in enumerate(self._retrievers[:-1]): tasks.append(retriever.aretrieve(query, **kwargs)) task_queries.append(query) + # get you retriever results + tasks.append(self._retrievers[-1].aretrieve(query, **kwargs)) + task_queries.append(query) + task_results = run_async_tasks(tasks) results = {} From 3e246eb1a9f4f9c42d4d422bf639ee73b422e77d Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 2 Feb 2024 19:25:16 +0530 Subject: [PATCH 10/41] feat: switch to chromadb from faiss for speed --- pyproject.toml | 1 + src/wandbot/chat/chat.py | 6 +- src/wandbot/ingestion/__main__.py | 8 +- src/wandbot/ingestion/prepare_data.py | 16 +-- src/wandbot/ingestion/vectorstores.py | 43 +++++--- src/wandbot/retriever/base.py | 138 +++++++++++++++++--------- src/wandbot/retriever/external.py | 2 +- src/wandbot/retriever/fusion.py | 14 ++- src/wandbot/utils.py | 36 ++++--- 9 files changed, 167 insertions(+), 97 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f40dd2e..8abfea7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ instructor = "^0.4.5" langchain-community = "^0.0.11" langchain = "^0.1.0" langchain-openai = "^0.0.2" +chromadb = "^0.4.22" [tool.poetry.dev-dependencies] #fasttext = {git = "https://github.com/cfculhane/fastText"} # FastText doesn't come with pybind11 and we need to use this workaround. diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 99fe55d..1b070d6 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -50,7 +50,7 @@ from wandbot.chat.query_enhancer import CompleteQuery, QueryHandler from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.retriever.base import Retriever -from wandbot.retriever.fusion import HybridRetriever +from wandbot.retriever.fusion import FusionRetriever from wandbot.retriever.postprocessors import ( LanguageFilterPostprocessor, MetadataPostprocessor, @@ -95,7 +95,7 @@ def rebuild_full_prompt( class WandbContextChatEngine(ContextChatEngine): def __init__( self, - retriever: HybridRetriever, + retriever: FusionRetriever, llm: LLM, memory: BaseMemory, prefix_messages: List[ChatMessage], @@ -112,7 +112,7 @@ def __init__( context_template=context_template, callback_manager=callback_manager, ) - self._retriever: HybridRetriever = retriever + self._retriever: FusionRetriever = retriever def _generate_context( self, message: str, **kwargs diff --git a/src/wandbot/ingestion/__main__.py b/src/wandbot/ingestion/__main__.py index f2b4e50..65f3e33 100644 --- a/src/wandbot/ingestion/__main__.py +++ b/src/wandbot/ingestion/__main__.py @@ -1,7 +1,6 @@ import os -from wandbot.ingestion import prepare_data, vectorstores -from wandbot.ingestion.report import create_ingestion_report +from wandbot.ingestion import vectorstores from wandbot.utils import get_logger logger = get_logger(__name__) @@ -11,10 +10,11 @@ def main(): project = os.environ.get("WANDB_PROJECT", "wandbot-dev") entity = os.environ.get("WANDB_ENTITY", "wandbot") - raw_artifact = prepare_data.load(project, entity) + # raw_artifact = prepare_data.load(project, entity) + raw_artifact = "wandbot/wandbot-dev/raw_dataset:v30" vectorstore_artifact = vectorstores.load(project, entity, raw_artifact) # TODO: include ingestion report - create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) + # create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) print(vectorstore_artifact) diff --git a/src/wandbot/ingestion/prepare_data.py b/src/wandbot/ingestion/prepare_data.py index 18ee9c9..13b11b7 100644 --- a/src/wandbot/ingestion/prepare_data.py +++ b/src/wandbot/ingestion/prepare_data.py @@ -851,14 +851,14 @@ def load( for loader in [ en_docodile_loader, ja_docodile_loader, - examples_code_loader, - examples_notebook_loader, - sdk_code_loader, - sdk_tests_loader, - weave_code_loader, - weave_examples_loader, - wandb_edu_code_loader, - fc_reports_loader, + # examples_code_loader, + # examples_notebook_loader, + # sdk_code_loader, + # sdk_tests_loader, + # weave_code_loader, + # weave_examples_loader, + # wandb_edu_code_loader, + # fc_reports_loader, ]: loader.config.docstore_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index cb1fd2b..b4c548a 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -61,7 +61,6 @@ def load( source_artifact_path, type="dataset" ) artifact_dir: str = artifact.download() - storage_context = load_storage_context(config.embedding_dim) service_context = load_service_context( embeddings_model=config.embeddings_model, embeddings_size=config.embedding_dim, @@ -70,11 +69,14 @@ def load( max_retries=config.max_retries, ) + storage_context = load_storage_context(persist_dir=str(config.persist_dir)) + document_files: List[pathlib.Path] = list( pathlib.Path(artifact_dir).rglob("documents.jsonl") ) - transformed_documents: Dict[str, List[TextNode]] = {} + transformed_documents: List[TextNode] = [] + indices = [] for document_file in document_files: documents: List[LcDocument] = [] with document_file.open() as f: @@ -85,23 +87,32 @@ def load( preprocessed_documents = preprocess_data.load(documents) unique_objects = {obj.hash: obj for obj in preprocessed_documents} preprocessed_documents = list(unique_objects.values()) - transformed_documents[ - document_file.parent.name - ] = preprocessed_documents - - for store_name, doc_list in transformed_documents.items(): - logger.info(f"Number of documents: {len(doc_list)}") - _ = load_index( - doc_list, - service_context, - storage_context, - index_id=store_name, - persist_dir=str(config.persist_dir), - ) + + for document in preprocessed_documents: + document.metadata["index"] = document_file.parent.name + tags_list = ( + document.metadata["tags"] if document.metadata["tags"] else [] + ) + + if tags_list: + document.metadata["tags"] = ",".join(tags_list) + else: + document.metadata["tags"] = "" + + transformed_documents.extend(preprocessed_documents) + indices.append(document_file.parent.name) + + logger.info(f"Number of documents: {len(transformed_documents)}") + _ = load_index( + transformed_documents, + service_context, + storage_context, + persist_dir=str(config.persist_dir), + ) artifact = wandb.Artifact( name="wandbot_index", type="storage_context", - metadata={"index_ids": list(transformed_documents.keys())}, + metadata={"indices": indices}, ) artifact.add_dir( local_path=str(config.persist_dir), diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index dc2a981..d3b4af8 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -6,21 +6,27 @@ QueryBundle, ServiceContext, StorageContext, - load_indices_from_storage, + load_index_from_storage, ) from llama_index.callbacks import CallbackManager from llama_index.core.base_retriever import BaseRetriever from llama_index.postprocessor import BaseNodePostprocessor, CohereRerank from llama_index.query_engine import RetrieverQueryEngine from llama_index.response_synthesizers import BaseSynthesizer, ResponseMode +from llama_index.retrievers import BM25Retriever from llama_index.retrievers.fusion_retriever import FUSION_MODES -from llama_index.schema import NodeWithScore +from llama_index.schema import NodeWithScore, TextNode from llama_index.vector_stores.simple import DEFAULT_VECTOR_STORE, NAMESPACE_SEP -from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME +from llama_index.vector_stores.types import ( + DEFAULT_PERSIST_FNAME, + ExactMatchFilter, + FilterCondition, + MetadataFilters, +) from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict from wandbot.retriever.external import YouRetriever -from wandbot.retriever.fusion import FusionRetriever, HybridRetriever +from wandbot.retriever.fusion import FusionRetriever from wandbot.retriever.postprocessors import ( LanguageFilterPostprocessor, MetadataPostprocessor, @@ -78,6 +84,29 @@ class RetrieverConfig(BaseSettings): ) +def load_bm25retriever(index, similarity_top_k: int, index_name: str = None): + if index_name is None: + all_docs = index.storage_context.vector_store.client.get() + else: + all_docs = index.storage_context.vector_store.client.get( + where={"index": index_name} + ) + + nodes = [] + for node_id, document, metadata in zip( + all_docs["ids"], all_docs["documents"], all_docs["metadatas"] + ): + nodes.append( + TextNode(node_id=node_id, text=document, metadata=metadata) + ) + + bm25_retriever = BM25Retriever.from_defaults( + nodes=nodes, + similarity_top_k=similarity_top_k, + ) + return bm25_retriever + + class Retriever: def __init__( self, @@ -108,43 +137,40 @@ def __init__( ) ) - ( - self.storage_context, - index_ids, - ) = self.load_storage_context_from_artifact( + self.storage_context, indices = self.load_storage_context_from_artifact( artifact_url=self.config.index_artifact ) - self.indices = load_indices_from_storage( + self.index = load_index_from_storage( self.storage_context, service_context=self.service_context, - index_ids=index_ids, ) - retriever_list = [] - for index in self.indices: - retriever = HybridRetriever( - index=index, - similarity_top_k=self.config.similarity_top_k, - storage_context=self.storage_context, + self.vector_retriever = self.index.as_retriever( + similarity_top_k=self.config.similarity_top_k, + storage_context=self.storage_context, + ) + self.bm25_retriever = load_bm25retriever( + self.index, self.config.similarity_top_k + ) + + self.bm25_retrievers_by_index = { + index: load_bm25retriever( + self.index, self.config.similarity_top_k, index ) - retriever_list.append(retriever) + for index in indices + } + self.you_retriever = YouRetriever( api_key=os.environ.get("YOU_API_KEY"), similarity_top_k=self.config.similarity_top_k, ) self._retriever = FusionRetriever( - retriever_list, - similarity_top_k=self.config.similarity_top_k - * len(retriever_list) - // 2, + [self.vector_retriever, self.bm25_retriever, self.you_retriever], + similarity_top_k=self.config.similarity_top_k, num_queries=1, - use_async=True, + use_async=False, mode=FUSION_MODES.RECIPROCAL_RANK, ) - retriever_list.append(self.you_retriever) - index_ids = index_ids + ["you.com"] - self._retriever_map = dict(zip(index_ids, retriever_list)) - self.is_avoid_query: bool | None = None def load_storage_context_from_artifact( @@ -162,11 +188,8 @@ def load_storage_context_from_artifact( artifact_dir = artifact.download() index_path = f"{artifact_dir}/{DEFAULT_VECTOR_STORE}{NAMESPACE_SEP}{DEFAULT_PERSIST_FNAME}" logger.debug(f"Loading index from {index_path}") - storage_context = load_storage_context( - embed_dimensions=self.config.embeddings_size, - persist_dir=artifact_dir, - ) - return storage_context, artifact.metadata["index_ids"] + storage_context = load_storage_context(persist_dir=artifact_dir) + return storage_context, artifact.metadata["indices"] def load_query_engine( self, @@ -230,28 +253,45 @@ def _retrieve( """ top_k = top_k or self.config.top_k language = language or self.config.language - retrievers = [] - for index in indices or []: - retriever = self._retriever_map.get(index) - if not retriever and index: - logger.warning( - f"Index {index} not found in retriever map. Skipping the index" - ) - retrievers.append(retriever) - - retrievers = [retriever for retriever in retrievers if retriever] - - if not retrievers: - logger.warning("No retrievers found. Defaulting to all retrievers") + + if not indices: + logger.warning( + "No indices were provided. Using the fusion retriever." + ) retriever = self._retriever else: + exact_match_filters = [ + ExactMatchFilter(key="index", value=idx) for idx in indices + ] + metadata_filters = MetadataFilters( + filters=exact_match_filters, + condition=FilterCondition.OR, + ) + + retrievers = [ + self.index.as_retriever( + similarity_top_k=self.config.similarity_top_k, + storage_context=self.storage_context, + filters=metadata_filters, + ), + ] + bm25_retrievers = [ + self.bm25_retrievers_by_index.get(index) for index in indices + ] + bm25_retrievers = [ + retriever + for retriever in bm25_retrievers + if retriever is not None + ] + retrievers.extend(bm25_retrievers) + if kwargs.pop("include_web_results", None): + retrievers.append(self.you_retriever) + retriever = FusionRetriever( retrievers, - similarity_top_k=self.config.similarity_top_k - * len(retrievers) - // 2, + similarity_top_k=self.config.similarity_top_k, num_queries=1, - use_async=True, + use_async=False, mode=FUSION_MODES.RECIPROCAL_RANK, ) @@ -328,6 +368,7 @@ def retrieve_from_indices( include_tags: List[str] | None = None, exclude_tags: List[str] | None = None, is_avoid_query: bool | None = False, + **kwargs, ): """Retrieves the top k results from the index for the given query. @@ -350,6 +391,7 @@ def retrieve_from_indices( include_tags=include_tags, exclude_tags=exclude_tags, is_avoid_query=is_avoid_query, + **kwargs, ) def __call__(self, query: str, **kwargs) -> List[Dict[str, Any]]: diff --git a/src/wandbot/retriever/external.py b/src/wandbot/retriever/external.py index b455c40..b14c468 100644 --- a/src/wandbot/retriever/external.py +++ b/src/wandbot/retriever/external.py @@ -31,7 +31,7 @@ def _retrieve( self, query_bundle: QueryBundle, **kwargs ) -> List[NodeWithScore]: """Retrieve.""" - if kwargs.get("is_avoid_query", False): + if not kwargs.get("is_avoid_query", False): try: headers = {"X-API-Key": self._api_key} url = "https://api.ydc-index.io/search" diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index 174d81e..a5703ab 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -7,6 +7,7 @@ from llama_index.constants import DEFAULT_SIMILARITY_TOP_K from llama_index.core.base_retriever import BaseRetriever from llama_index.indices.base import BaseIndex +from llama_index.indices.vector_store import VectorIndexRetriever from llama_index.llms.utils import LLMType from llama_index.retrievers import BM25Retriever, QueryFusionRetriever from llama_index.retrievers.fusion_retriever import FUSION_MODES @@ -88,7 +89,9 @@ async def aretrieve( class FusionRetriever(QueryFusionRetriever): def __init__( self, - retrievers: List[Union[HybridRetriever, YouRetriever]], + retrievers: List[ + Union[VectorIndexRetriever, BM25Retriever, YouRetriever] + ], llm: Optional[LLMType] = "default", query_gen_prompt: Optional[str] = None, mode: FUSION_MODES = FUSION_MODES.SIMPLE, @@ -144,9 +147,12 @@ def _run_sync_queries( results = {} for query in queries: for i, retriever in enumerate(self._retrievers): - results[(query.query_str, i)] = retriever.retrieve( - query, **kwargs - ) + if isinstance(retriever, YouRetriever): + results[(query.query_str, i)] = retriever.retrieve( + query, **kwargs + ) + else: + results[(query.query_str, i)] = retriever.retrieve(query) return results diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 3238906..0c5e172 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -30,7 +30,7 @@ import sqlite3 from typing import Any, Coroutine, List, Optional, Tuple -import faiss +import chromadb import fasttext import nest_asyncio import wandb @@ -39,7 +39,7 @@ from llama_index.llms import LiteLLM from llama_index.llms.llm import LLM from llama_index.schema import NodeWithScore, TextNode -from llama_index.vector_stores import FaissVectorStore +from llama_index.vector_stores import ChromaVectorStore from pydantic_settings import BaseSettings @@ -60,6 +60,9 @@ def get_logger(name: str) -> logging.Logger: return logger +logger = get_logger(__name__) + + class Timer: """A simple timer class for measuring elapsed time.""" @@ -164,24 +167,33 @@ def load_service_context( ) -def load_storage_context( - embed_dimensions: int, persist_dir: str | None = None -) -> StorageContext: +def load_storage_context(persist_dir: str | None = None) -> StorageContext: """Loads a storage context with the specified parameters. Args: - embed_dimensions: The dimensions of the embeddings. + embedding_function: The embedding function to use in the vectorstore. persist_dir: The directory where the storage context is persisted. Returns: A storage context instance with the specified parameters. """ - faiss_index = faiss.IndexFlatL2(embed_dimensions) - storage_context = StorageContext.from_defaults( - vector_store=FaissVectorStore(faiss_index), - persist_dir=persist_dir, - ) + chroma_client = chromadb.PersistentClient(path=persist_dir) + chroma_collection = chroma_client.get_or_create_collection("docstore") + try: + storage_context = StorageContext.from_defaults( + vector_store=ChromaVectorStore( + chroma_collection=chroma_collection, persist_dir=persist_dir + ), + persist_dir=persist_dir, + ) + except FileNotFoundError as e: + logger.debug(f"Error loading storage context: {e}") + storage_context = StorageContext.from_defaults( + vector_store=ChromaVectorStore( + chroma_collection=chroma_collection, persist_dir=persist_dir + ), + ) return storage_context @@ -189,7 +201,6 @@ def load_index( nodes: List[TextNode], service_context: ServiceContext, storage_context: StorageContext, - index_id: str, persist_dir: str, ) -> VectorStoreIndex: """Loads an index from storage or creates a new one if not found. @@ -210,7 +221,6 @@ def load_index( storage_context=storage_context, show_progress=True, ) - index.set_index_id(index_id) index.storage_context.persist(persist_dir=persist_dir) return index From c868d67f11e4e0cc4790fcf332c527a7ea777421 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Wed, 7 Feb 2024 16:31:55 +0530 Subject: [PATCH 11/41] refactor: complete project overhaul to improve efficiency --- poetry.lock | 7051 +++++++++++++++++ pyproject.toml | 14 +- src/wandbot/chat/rag.py | 60 + src/wandbot/ingestion/__main__.py | 10 +- src/wandbot/ingestion/config.py | 78 +- src/wandbot/ingestion/prepare_data.py | 166 +- src/wandbot/ingestion/preprocess_data.py | 449 +- src/wandbot/ingestion/vectorstores.py | 68 +- src/wandbot/query_handler/__init__.py | 0 src/wandbot/query_handler/history_handler.py | 75 + src/wandbot/query_handler/intents_enhancer.py | 231 + .../query_handler/keyword_search_enhancer.py | 85 + .../query_handler/language_detection.py | 25 + src/wandbot/query_handler/query_enhancer.py | 70 + .../query_handler/vector_search_enhancer.py | 73 + src/wandbot/query_handler/web_search.py | 207 + src/wandbot/retriever/base.py | 368 +- src/wandbot/retriever/fusion.py | 440 +- src/wandbot/utils.py | 64 + 19 files changed, 8640 insertions(+), 894 deletions(-) create mode 100644 poetry.lock create mode 100644 src/wandbot/chat/rag.py create mode 100644 src/wandbot/query_handler/__init__.py create mode 100644 src/wandbot/query_handler/history_handler.py create mode 100644 src/wandbot/query_handler/intents_enhancer.py create mode 100644 src/wandbot/query_handler/keyword_search_enhancer.py create mode 100644 src/wandbot/query_handler/language_detection.py create mode 100644 src/wandbot/query_handler/query_enhancer.py create mode 100644 src/wandbot/query_handler/vector_search_enhancer.py create mode 100644 src/wandbot/query_handler/web_search.py diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..769e293 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,7051 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "aiofiles" +version = "23.2.1" +description = "File support for asyncio." +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiofiles-23.2.1-py3-none-any.whl", hash = "sha256:19297512c647d4b27a2cf7c34caa7e405c0d60b5560618a29a9fe027b18b0107"}, + {file = "aiofiles-23.2.1.tar.gz", hash = "sha256:84ec2218d8419404abcb9f0c02df3f34c6e0a68ed41072acfb1cef5cbc29051a"}, +] + +[[package]] +name = "aiohttp" +version = "3.9.3" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, +] + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "brotlicffi"] + +[[package]] +name = "aioprocessing" +version = "2.0.1" +description = "A Python 3.5+ library that integrates the multiprocessing module with asyncio." +optional = false +python-versions = ">=3.5" +files = [ + {file = "aioprocessing-2.0.1-py3-none-any.whl", hash = "sha256:8fcac4b0108b72eb9df76e06a9d7e05720ee1e8330829d3fd53fa059879be586"}, + {file = "aioprocessing-2.0.1.tar.gz", hash = "sha256:fe01c7b1a38c78168611d3040e73d93036c3b7c8a649d636dc9ed7a3bc9b1ba2"}, +] + +[package.extras] +dill = ["multiprocess"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "analytics-python" +version = "1.2.9" +description = "The hassle-free way to integrate analytics into any python application." +optional = false +python-versions = "*" +files = [ + {file = "analytics-python-1.2.9.tar.gz", hash = "sha256:f3d1ca27cb277da67c10d71a5c9c593d2a9ec99109e31409ab771b44821a86bf"}, + {file = "analytics_python-1.2.9-py2.py3-none-any.whl", hash = "sha256:69d88b2d3e2c350e6803487a1a802e0fd111e86665d4c9b16c3c6f5fbc6c445f"}, +] + +[package.dependencies] +python-dateutil = ">2.1" +requests = ">=2.7,<3.0" +six = ">=1.5" + +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + +[[package]] +name = "anyio" +version = "4.2.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"}, + {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + +[[package]] +name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = "*" +files = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] + +[[package]] +name = "appnope" +version = "0.1.4" +description = "Disable App Nap on macOS >= 10.9" +optional = false +python-versions = ">=3.6" +files = [ + {file = "appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c"}, + {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"}, +] + +[[package]] +name = "argon2-cffi" +version = "23.1.0" +description = "Argon2 for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"}, + {file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"}, +] + +[package.dependencies] +argon2-cffi-bindings = "*" + +[package.extras] +dev = ["argon2-cffi[tests,typing]", "tox (>4)"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-notfound-page"] +tests = ["hypothesis", "pytest"] +typing = ["mypy"] + +[[package]] +name = "argon2-cffi-bindings" +version = "21.2.0" +description = "Low-level CFFI bindings for Argon2" +optional = false +python-versions = ">=3.6" +files = [ + {file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f"}, + {file = "argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3e385d1c39c520c08b53d63300c3ecc28622f076f4c2b0e6d7e796e9f6502194"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3e3cc67fdb7d82c4718f19b4e7a87123caf8a93fde7e23cf66ac0337d3cb3f"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a22ad9800121b71099d0fb0a65323810a15f2e292f2ba450810a7316e128ee5"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:93f9bf70084f97245ba10ee36575f0c3f1e7d7724d67d8e5b08e61787c320ed7"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3b9ef65804859d335dc6b31582cad2c5166f0c3e7975f324d9ffaa34ee7e6583"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4966ef5848d820776f5f562a7d45fdd70c2f330c961d0d745b784034bd9f48d"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ef543a89dee4db46a1a6e206cd015360e5a75822f76df533845c3cbaf72670"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed2937d286e2ad0cc79a7087d3c272832865f779430e0cc2b4f3718d3159b0cb"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a"}, +] + +[package.dependencies] +cffi = ">=1.0.1" + +[package.extras] +dev = ["cogapp", "pre-commit", "pytest", "wheel"] +tests = ["pytest"] + +[[package]] +name = "arrow" +version = "1.3.0" +description = "Better dates & times for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80"}, + {file = "arrow-1.3.0.tar.gz", hash = "sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85"}, +] + +[package.dependencies] +python-dateutil = ">=2.7.0" +types-python-dateutil = ">=2.8.10" + +[package.extras] +doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] +test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (==3.*)"] + +[[package]] +name = "asgiref" +version = "3.7.2" +description = "ASGI specs, helper code, and adapters" +optional = false +python-versions = ">=3.7" +files = [ + {file = "asgiref-3.7.2-py3-none-any.whl", hash = "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e"}, + {file = "asgiref-3.7.2.tar.gz", hash = "sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4", markers = "python_version < \"3.11\""} + +[package.extras] +tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] + +[[package]] +name = "asttokens" +version = "2.4.1" +description = "Annotate AST trees with source code positions" +optional = false +python-versions = "*" +files = [ + {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, + {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, +] + +[package.dependencies] +six = ">=1.12.0" + +[package.extras] +astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"] +test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] + +[[package]] +name = "async-lru" +version = "2.0.4" +description = "Simple LRU cache for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "async-lru-2.0.4.tar.gz", hash = "sha256:b8a59a5df60805ff63220b2a0c5b5393da5521b113cd5465a44eb037d81a5627"}, + {file = "async_lru-2.0.4-py3-none-any.whl", hash = "sha256:ff02944ce3c288c5be660c42dbcca0742b32c3b279d6dceda655190240b99224"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "babel" +version = "2.14.0" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Babel-2.14.0-py3-none-any.whl", hash = "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"}, + {file = "Babel-2.14.0.tar.gz", hash = "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363"}, +] + +[package.extras] +dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bcrypt" +version = "4.1.2" +description = "Modern password hashing for your software and your servers" +optional = false +python-versions = ">=3.7" +files = [ + {file = "bcrypt-4.1.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0"}, + {file = "bcrypt-4.1.2-cp37-abi3-win32.whl", hash = "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369"}, + {file = "bcrypt-4.1.2-cp37-abi3-win_amd64.whl", hash = "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551"}, + {file = "bcrypt-4.1.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a"}, + {file = "bcrypt-4.1.2-cp39-abi3-win32.whl", hash = "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f"}, + {file = "bcrypt-4.1.2-cp39-abi3-win_amd64.whl", hash = "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42"}, + {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d75fc8cd0ba23f97bae88a6ec04e9e5351ff3c6ad06f38fe32ba50cbd0d11946"}, + {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:a97e07e83e3262599434816f631cc4c7ca2aa8e9c072c1b1a7fec2ae809a1d2d"}, + {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e51c42750b7585cee7892c2614be0d14107fad9581d1738d954a262556dd1aab"}, + {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba4e4cc26610581a6329b3937e02d319f5ad4b85b074846bf4fef8a8cf51e7bb"}, + {file = "bcrypt-4.1.2.tar.gz", hash = "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "black" +version = "24.1.1" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-24.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c"}, + {file = "black-24.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445"}, + {file = "black-24.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a"}, + {file = "black-24.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4"}, + {file = "black-24.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7"}, + {file = "black-24.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8"}, + {file = "black-24.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161"}, + {file = "black-24.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d"}, + {file = "black-24.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8"}, + {file = "black-24.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e"}, + {file = "black-24.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6"}, + {file = "black-24.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b"}, + {file = "black-24.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62"}, + {file = "black-24.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5"}, + {file = "black-24.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6"}, + {file = "black-24.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717"}, + {file = "black-24.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9"}, + {file = "black-24.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024"}, + {file = "black-24.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2"}, + {file = "black-24.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac"}, + {file = "black-24.1.1-py3-none-any.whl", hash = "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168"}, + {file = "black-24.1.1.tar.gz", hash = "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "bleach" +version = "6.1.0" +description = "An easy safelist-based HTML-sanitizing tool." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bleach-6.1.0-py3-none-any.whl", hash = "sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6"}, + {file = "bleach-6.1.0.tar.gz", hash = "sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe"}, +] + +[package.dependencies] +six = ">=1.9.0" +webencodings = "*" + +[package.extras] +css = ["tinycss2 (>=1.1.0,<1.3)"] + +[[package]] +name = "blinker" +version = "1.7.0" +description = "Fast, simple object-to-object and broadcast signaling" +optional = false +python-versions = ">=3.8" +files = [ + {file = "blinker-1.7.0-py3-none-any.whl", hash = "sha256:c3f865d4d54db7abc53758a01601cf343fe55b84c1de4e3fa910e420b438d5b9"}, + {file = "blinker-1.7.0.tar.gz", hash = "sha256:e6820ff6fa4e4d1d8e2747c2283749c3f547e4fee112b98555cdcdae32996182"}, +] + +[[package]] +name = "build" +version = "1.0.3" +description = "A simple, correct Python build frontend" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "build-1.0.3-py3-none-any.whl", hash = "sha256:589bf99a67df7c9cf07ec0ac0e5e2ea5d4b37ac63301c4986d1acb126aa83f8f"}, + {file = "build-1.0.3.tar.gz", hash = "sha256:538aab1b64f9828977f84bc63ae570b060a8ed1be419e7870b8b4fc5e6ea553b"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "os_name == \"nt\""} +packaging = ">=19.0" +pyproject_hooks = "*" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.08.17)", "sphinx (>=7.0,<8.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)", "sphinx-issues (>=3.0.0)"] +test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "setuptools (>=56.0.0)", "setuptools (>=67.8.0)", "wheel (>=0.36.0)"] +typing = ["importlib-metadata (>=5.1)", "mypy (>=1.5.0,<1.6.0)", "tomli", "typing-extensions (>=3.7.4.3)"] +virtualenv = ["virtualenv (>=20.0.35)"] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "chroma-hnswlib" +version = "0.7.3" +description = "Chromas fork of hnswlib" +optional = false +python-versions = "*" +files = [ + {file = "chroma-hnswlib-0.7.3.tar.gz", hash = "sha256:b6137bedde49fffda6af93b0297fe00429fc61e5a072b1ed9377f909ed95a932"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59d6a7c6f863c67aeb23e79a64001d537060b6995c3eca9a06e349ff7b0998ca"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d71a3f4f232f537b6152947006bd32bc1629a8686df22fd97777b70f416c127a"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c92dc1ebe062188e53970ba13f6b07e0ae32e64c9770eb7f7ffa83f149d4210"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49da700a6656fed8753f68d44b8cc8ae46efc99fc8a22a6d970dc1697f49b403"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-win_amd64.whl", hash = "sha256:108bc4c293d819b56476d8f7865803cb03afd6ca128a2a04d678fffc139af029"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:11e7ca93fb8192214ac2b9c0943641ac0daf8f9d4591bb7b73be808a83835667"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6f552e4d23edc06cdeb553cdc757d2fe190cdeb10d43093d6a3319f8d4bf1c6b"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f96f4d5699e486eb1fb95849fe35ab79ab0901265805be7e60f4eaa83ce263ec"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:368e57fe9ebae05ee5844840fa588028a023d1182b0cfdb1d13f607c9ea05756"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-win_amd64.whl", hash = "sha256:b7dca27b8896b494456db0fd705b689ac6b73af78e186eb6a42fea2de4f71c6f"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:70f897dc6218afa1d99f43a9ad5eb82f392df31f57ff514ccf4eeadecd62f544"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aef10b4952708f5a1381c124a29aead0c356f8d7d6e0b520b778aaa62a356f4"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee2d8d1529fca3898d512079144ec3e28a81d9c17e15e0ea4665697a7923253"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-win_amd64.whl", hash = "sha256:a4021a70e898783cd6f26e00008b494c6249a7babe8774e90ce4766dd288c8ba"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a8f61fa1d417fda848e3ba06c07671f14806a2585272b175ba47501b066fe6b1"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7563be58bc98e8f0866907368e22ae218d6060601b79c42f59af4eccbbd2e0a"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51b8d411486ee70d7b66ec08cc8b9b6620116b650df9c19076d2d8b6ce2ae914"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d706782b628e4f43f1b8a81e9120ac486837fbd9bcb8ced70fe0d9b95c72d77"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-win_amd64.whl", hash = "sha256:54f053dedc0e3ba657f05fec6e73dd541bc5db5b09aa8bc146466ffb734bdc86"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e607c5a71c610a73167a517062d302c0827ccdd6e259af6e4869a5c1306ffb5d"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2358a795870156af6761890f9eb5ca8cade57eb10c5f046fe94dae1faa04b9e"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cea425df2e6b8a5e201fff0d922a1cc1d165b3cfe762b1408075723c8892218"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:454df3dd3e97aa784fba7cf888ad191e0087eef0fd8c70daf28b753b3b591170"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:df587d15007ca701c6de0ee7d5585dd5e976b7edd2b30ac72bc376b3c3f85882"}, +] + +[package.dependencies] +numpy = "*" + +[[package]] +name = "chromadb" +version = "0.4.22" +description = "Chroma." +optional = false +python-versions = ">=3.8" +files = [ + {file = "chromadb-0.4.22-py3-none-any.whl", hash = "sha256:ad210b27b4cda2f09d15adc9c83c81bfa66b69f39648a27b637306e40de0680d"}, + {file = "chromadb-0.4.22.tar.gz", hash = "sha256:c793149e1c2bbbb52d77602c6c0594c5752f04cd9be12619250ddad2082af27a"}, +] + +[package.dependencies] +bcrypt = ">=4.0.1" +build = ">=1.0.3" +chroma-hnswlib = "0.7.3" +fastapi = ">=0.95.2" +grpcio = ">=1.58.0" +importlib-resources = "*" +kubernetes = ">=28.1.0" +mmh3 = ">=4.0.1" +numpy = ">=1.22.5" +onnxruntime = ">=1.14.1" +opentelemetry-api = ">=1.2.0" +opentelemetry-exporter-otlp-proto-grpc = ">=1.2.0" +opentelemetry-instrumentation-fastapi = ">=0.41b0" +opentelemetry-sdk = ">=1.2.0" +overrides = ">=7.3.1" +posthog = ">=2.4.0" +pulsar-client = ">=3.1.0" +pydantic = ">=1.9" +pypika = ">=0.48.9" +PyYAML = ">=6.0.0" +requests = ">=2.28" +tenacity = ">=8.2.3" +tokenizers = ">=0.13.2" +tqdm = ">=4.65.0" +typer = ">=0.9.0" +typing-extensions = ">=4.5.0" +uvicorn = {version = ">=0.18.3", extras = ["standard"]} + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "cohere" +version = "4.45" +description = "Python SDK for the Cohere API" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "cohere-4.45-py3-none-any.whl", hash = "sha256:bdaa2e5e1c64cf3b1d55caf9d483a33fa8eafed731a999fb0934ae12c0638b75"}, + {file = "cohere-4.45.tar.gz", hash = "sha256:63b21b2dc3abd718b18cae726a69d1b096a34eb59f3331c20469fd0df1672816"}, +] + +[package.dependencies] +aiohttp = ">=3.0,<4.0" +backoff = ">=2.0,<3.0" +fastavro = ">=1.8,<2.0" +importlib_metadata = ">=6.0,<7.0" +requests = ">=2.25.0,<3.0.0" +urllib3 = ">=1.26,<3" + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coloredlogs" +version = "15.0.1" +description = "Colored terminal output for Python's logging module" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"}, + {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, +] + +[package.dependencies] +humanfriendly = ">=9.1" + +[package.extras] +cron = ["capturer (>=2.4)"] + +[[package]] +name = "colorlog" +version = "6.8.2" +description = "Add colours to the output of Python's logging module." +optional = false +python-versions = ">=3.6" +files = [ + {file = "colorlog-6.8.2-py3-none-any.whl", hash = "sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33"}, + {file = "colorlog-6.8.2.tar.gz", hash = "sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +development = ["black", "flake8", "mypy", "pytest", "types-colorama"] + +[[package]] +name = "comm" +version = "0.2.1" +description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." +optional = false +python-versions = ">=3.8" +files = [ + {file = "comm-0.2.1-py3-none-any.whl", hash = "sha256:87928485c0dfc0e7976fd89fc1e187023cf587e7c353e4a9b417555b44adf021"}, + {file = "comm-0.2.1.tar.gz", hash = "sha256:0bc91edae1344d39d3661dcbc36937181fdaddb304790458f8b044dbc064b89a"}, +] + +[package.dependencies] +traitlets = ">=4" + +[package.extras] +test = ["pytest"] + +[[package]] +name = "dataclasses-json" +version = "0.6.4" +description = "Easily serialize dataclasses to and from JSON." +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "dataclasses_json-0.6.4-py3-none-any.whl", hash = "sha256:f90578b8a3177f7552f4e1a6e535e84293cd5da421fcce0642d49c0d7bdf8df2"}, + {file = "dataclasses_json-0.6.4.tar.gz", hash = "sha256:73696ebf24936560cca79a2430cbc4f3dd23ac7bf46ed17f38e5e5e7657a6377"}, +] + +[package.dependencies] +marshmallow = ">=3.18.0,<4.0.0" +typing-inspect = ">=0.4.0,<1" + +[[package]] +name = "dataclasses-json-speakeasy" +version = "0.5.11" +description = "Easily serialize dataclasses to and from JSON." +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "dataclasses_json_speakeasy-0.5.11-py3-none-any.whl", hash = "sha256:ac52a069a01e8521015d682f37849bfdf056c36fa3f81497055e201fec684104"}, + {file = "dataclasses_json_speakeasy-0.5.11.tar.gz", hash = "sha256:418a987cea2ccf4e4be662f39faa5cc79b47b147c9d1a69d6928d6a27e0c17e8"}, +] + +[package.dependencies] +marshmallow = ">=3.18.0,<4.0.0" +typing-inspect = ">=0.4.0,<1" + +[[package]] +name = "db-dtypes" +version = "1.2.0" +description = "Pandas Data Types for SQL systems (BigQuery, Spanner)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "db-dtypes-1.2.0.tar.gz", hash = "sha256:3531bb1fb8b5fbab33121fe243ccc2ade16ab2524f4c113b05cc702a1908e6ea"}, + {file = "db_dtypes-1.2.0-py2.py3-none-any.whl", hash = "sha256:6320bddd31d096447ef749224d64aab00972ed20e4392d86f7d8b81ad79f7ff0"}, +] + +[package.dependencies] +numpy = ">=1.16.6" +packaging = ">=17.0" +pandas = ">=0.24.2" +pyarrow = ">=3.0.0" + +[[package]] +name = "debugpy" +version = "1.8.0" +description = "An implementation of the Debug Adapter Protocol for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "debugpy-1.8.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:7fb95ca78f7ac43393cd0e0f2b6deda438ec7c5e47fa5d38553340897d2fbdfb"}, + {file = "debugpy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada"}, + {file = "debugpy-1.8.0-cp310-cp310-win32.whl", hash = "sha256:a8b7a2fd27cd9f3553ac112f356ad4ca93338feadd8910277aff71ab24d8775f"}, + {file = "debugpy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:5d9de202f5d42e62f932507ee8b21e30d49aae7e46d5b1dd5c908db1d7068637"}, + {file = "debugpy-1.8.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e"}, + {file = "debugpy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60009b132c91951354f54363f8ebdf7457aeb150e84abba5ae251b8e9f29a8a6"}, + {file = "debugpy-1.8.0-cp311-cp311-win32.whl", hash = "sha256:8cd0197141eb9e8a4566794550cfdcdb8b3db0818bdf8c49a8e8f8053e56e38b"}, + {file = "debugpy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a64093656c4c64dc6a438e11d59369875d200bd5abb8f9b26c1f5f723622e153"}, + {file = "debugpy-1.8.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:b05a6b503ed520ad58c8dc682749113d2fd9f41ffd45daec16e558ca884008cd"}, + {file = "debugpy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c6fb41c98ec51dd010d7ed650accfd07a87fe5e93eca9d5f584d0578f28f35f"}, + {file = "debugpy-1.8.0-cp38-cp38-win32.whl", hash = "sha256:46ab6780159eeabb43c1495d9c84cf85d62975e48b6ec21ee10c95767c0590aa"}, + {file = "debugpy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:bdc5ef99d14b9c0fcb35351b4fbfc06ac0ee576aeab6b2511702e5a648a2e595"}, + {file = "debugpy-1.8.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:61eab4a4c8b6125d41a34bad4e5fe3d2cc145caecd63c3fe953be4cc53e65bf8"}, + {file = "debugpy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:125b9a637e013f9faac0a3d6a82bd17c8b5d2c875fb6b7e2772c5aba6d082332"}, + {file = "debugpy-1.8.0-cp39-cp39-win32.whl", hash = "sha256:57161629133113c97b387382045649a2b985a348f0c9366e22217c87b68b73c6"}, + {file = "debugpy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:e3412f9faa9ade82aa64a50b602544efcba848c91384e9f93497a458767e6926"}, + {file = "debugpy-1.8.0-py2.py3-none-any.whl", hash = "sha256:9c9b0ac1ce2a42888199df1a1906e45e6f3c9555497643a85e0bf2406e3ffbc4"}, + {file = "debugpy-1.8.0.zip", hash = "sha256:12af2c55b419521e33d5fb21bd022df0b5eb267c3e178f1d374a63a2a6bdccd0"}, +] + +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +optional = false +python-versions = ">=3.5" +files = [ + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, + {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "discord" +version = "2.3.2" +description = "A mirror package for discord.py. Please install that instead." +optional = false +python-versions = "*" +files = [ + {file = "discord-2.3.2-py3-none-any.whl", hash = "sha256:d7959418799dd3b1e896685812d880169c193468b061b3431fa2a4664febd3da"}, + {file = "discord-2.3.2.tar.gz", hash = "sha256:cc1ee2dbe6df218ca51519af355b97e87309f8230f58c7f34885feb8e8a76145"}, +] + +[package.dependencies] +"discord.py" = ">=2.3.2" + +[[package]] +name = "discord-py" +version = "2.3.2" +description = "A Python wrapper for the Discord API" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "discord.py-2.3.2-py3-none-any.whl", hash = "sha256:9da4679fc3cb10c64b388284700dc998663e0e57328283bbfcfc2525ec5960a6"}, + {file = "discord.py-2.3.2.tar.gz", hash = "sha256:4560f70f2eddba7e83370ecebd237ac09fbb4980dc66507482b0c0e5b8f76b9c"}, +] + +[package.dependencies] +aiohttp = ">=3.7.4,<4" + +[package.extras] +docs = ["sphinx (==4.4.0)", "sphinxcontrib-trio (==1.1.2)", "sphinxcontrib-websupport", "typing-extensions (>=4.3,<5)"] +speed = ["Brotli", "aiodns (>=1.1)", "cchardet (==2.1.7)", "orjson (>=3.5.4)"] +test = ["coverage[toml]", "pytest", "pytest-asyncio", "pytest-cov", "pytest-mock", "typing-extensions (>=4.3,<5)"] +voice = ["PyNaCl (>=1.3.0,<1.6)"] + +[[package]] +name = "distro" +version = "1.9.0" +description = "Distro - an OS platform information API" +optional = false +python-versions = ">=3.6" +files = [ + {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, + {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, +] + +[[package]] +name = "docker-pycreds" +version = "0.4.0" +description = "Python bindings for the docker credentials store API" +optional = false +python-versions = "*" +files = [ + {file = "docker-pycreds-0.4.0.tar.gz", hash = "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4"}, + {file = "docker_pycreds-0.4.0-py2.py3-none-any.whl", hash = "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49"}, +] + +[package.dependencies] +six = ">=1.4.0" + +[[package]] +name = "emoji" +version = "2.10.1" +description = "Emoji for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "emoji-2.10.1-py2.py3-none-any.whl", hash = "sha256:11fb369ea79d20c14efa4362c732d67126df294a7959a2c98bfd7447c12a218e"}, + {file = "emoji-2.10.1.tar.gz", hash = "sha256:16287283518fb7141bde00198f9ffff4e1c1cb570efb68b2f1ec50975c3a581d"}, +] + +[package.extras] +dev = ["coverage", "coveralls", "pytest"] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "executing" +version = "2.0.1" +description = "Get the currently executing AST node of a frame, and other information" +optional = false +python-versions = ">=3.5" +files = [ + {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, + {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, +] + +[package.extras] +tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] + +[[package]] +name = "faiss-cpu" +version = "1.7.4" +description = "A library for efficient similarity search and clustering of dense vectors." +optional = false +python-versions = "*" +files = [ + {file = "faiss-cpu-1.7.4.tar.gz", hash = "sha256:265dc31b0c079bf4433303bf6010f73922490adff9188b915e2d3f5e9c82dd0a"}, + {file = "faiss_cpu-1.7.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50d4ebe7f1869483751c558558504f818980292a9b55be36f9a1ee1009d9a686"}, + {file = "faiss_cpu-1.7.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b1db7fae7bd8312aeedd0c41536bcd19a6e297229e1dce526bde3a73ab8c0b5"}, + {file = "faiss_cpu-1.7.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17b7fa7194a228a84929d9e6619d0e7dbf00cc0f717e3462253766f5e3d07de8"}, + {file = "faiss_cpu-1.7.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dca531952a2e3eac56f479ff22951af4715ee44788a3fe991d208d766d3f95f3"}, + {file = "faiss_cpu-1.7.4-cp310-cp310-win_amd64.whl", hash = "sha256:7173081d605e74766f950f2e3d6568a6f00c53f32fd9318063e96728c6c62821"}, + {file = "faiss_cpu-1.7.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0bbd6f55d7940cc0692f79e32a58c66106c3c950cee2341b05722de9da23ea3"}, + {file = "faiss_cpu-1.7.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e13c14280376100f143767d0efe47dcb32618f69e62bbd3ea5cd38c2e1755926"}, + {file = "faiss_cpu-1.7.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c521cb8462f3b00c0c7dfb11caff492bb67816528b947be28a3b76373952c41d"}, + {file = "faiss_cpu-1.7.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afdd9fe1141117fed85961fd36ee627c83fc3b9fd47bafb52d3c849cc2f088b7"}, + {file = "faiss_cpu-1.7.4-cp311-cp311-win_amd64.whl", hash = "sha256:2ff7f57889ea31d945e3b87275be3cad5d55b6261a4e3f51c7aba304d76b81fb"}, + {file = "faiss_cpu-1.7.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:eeaf92f27d76249fb53c1adafe617b0f217ab65837acf7b4ec818511caf6e3d8"}, + {file = "faiss_cpu-1.7.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:102b1bd763e9b0c281ac312590af3eaf1c8b663ccbc1145821fe6a9f92b8eaaf"}, + {file = "faiss_cpu-1.7.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5512da6707c967310c46ff712b00418b7ae28e93cb609726136e826e9f2f14fa"}, + {file = "faiss_cpu-1.7.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0c2e5b9d8c28c99f990e87379d5bbcc6c914da91ebb4250166864fd12db5755b"}, + {file = "faiss_cpu-1.7.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:43f67f325393145d360171cd98786fcea6120ce50397319afd3bb78be409fb8a"}, + {file = "faiss_cpu-1.7.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6a4e4af194b8fce74c4b770cad67ad1dd1b4673677fc169723e4c50ba5bd97a8"}, + {file = "faiss_cpu-1.7.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31bfb7b9cffc36897ae02a983e04c09fe3b8c053110a287134751a115334a1df"}, + {file = "faiss_cpu-1.7.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52d7de96abef2340c0d373c1f5cbc78026a3cebb0f8f3a5920920a00210ead1f"}, + {file = "faiss_cpu-1.7.4-cp38-cp38-win_amd64.whl", hash = "sha256:699feef85b23c2c729d794e26ca69bebc0bee920d676028c06fd0e0becc15c7e"}, + {file = "faiss_cpu-1.7.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:559a0133f5ed44422acb09ee1ac0acffd90c6666d1bc0d671c18f6e93ad603e2"}, + {file = "faiss_cpu-1.7.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1d71539fe3dc0f1bed41ef954ca701678776f231046bf0ca22ccea5cf5bef6"}, + {file = "faiss_cpu-1.7.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12d45e0157024eb3249842163162983a1ac8b458f1a8b17bbf86f01be4585a99"}, + {file = "faiss_cpu-1.7.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f0eab359e066d32c874f51a7d4bf6440edeec068b7fe47e6d803c73605a8b4c"}, + {file = "faiss_cpu-1.7.4-cp39-cp39-win_amd64.whl", hash = "sha256:98459ceeeb735b9df1a5b94572106ffe0a6ce740eb7e4626715dd218657bb4dc"}, +] + +[[package]] +name = "fastapi" +version = "0.109.2" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastapi-0.109.2-py3-none-any.whl", hash = "sha256:2c9bab24667293b501cad8dd388c05240c850b58ec5876ee3283c47d6e1e3a4d"}, + {file = "fastapi-0.109.2.tar.gz", hash = "sha256:f3817eac96fe4f65a2ebb4baa000f394e55f5fccdaf7f75250804bc58f354f73"}, +] + +[package.dependencies] +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" +starlette = ">=0.36.3,<0.37.0" +typing-extensions = ">=4.8.0" + +[package.extras] +all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.7)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] + +[[package]] +name = "fastavro" +version = "1.9.3" +description = "Fast read/write of AVRO files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastavro-1.9.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:5e9b2e1427fb84c0754bc34923d10cabcf2ed23230201208a1371ab7b6027674"}, + {file = "fastavro-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ef82f86ae276309abc0072598474b6be68105a0b28f8d7cc0398d1d353d7de"}, + {file = "fastavro-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:280ef7ab7232ecb2097038d6842416ec717d0e1c314b80ff245f85201f3396a4"}, + {file = "fastavro-1.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4a36cfc0421ed7576ecb1c22de7bd1dedcce62aebbffcc597379d59171e5d76e"}, + {file = "fastavro-1.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d80f2e20199140eb8c036b4393e9bc9eff325543311b958c72318999499d4279"}, + {file = "fastavro-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:a435f7edd7c5b52cee3f23ca950cd9373ab35cf2aa3d269b3d6aca7e2fc1372c"}, + {file = "fastavro-1.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a7053ed10194ec53754f5337b57b3273a74b48505edcd6edb79fe3c4cd259c0"}, + {file = "fastavro-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853e01f13534d1baa0a3d493a8573e665e93ffa35b4bf1d125e21764d343af8e"}, + {file = "fastavro-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5a279cda25d876e6f120950cadf184a307fd8998f9a22a90bb62e6749f88d1e"}, + {file = "fastavro-1.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:63d6f928840f3fb1f2e1fe20bc8b7d0e1a51ba4bb0e554ecb837a669fba31288"}, + {file = "fastavro-1.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8807046edc78f50b3ea5f55f6a534c87b2a13538e7c56fec3532ef802bcae333"}, + {file = "fastavro-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:e502579da4a51c5630eadbd811a1b3d262d6e783bf19998cfb33d2ea0cf6f516"}, + {file = "fastavro-1.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:6b665efe442061df8d9608c2fb692847df85d52ad825b776c441802f0dfa6571"}, + {file = "fastavro-1.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b8c96d81f0115633489d7f1133a03832922629a61ca81c1d47b482ddcda3b94"}, + {file = "fastavro-1.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338c7ec94dd2474c4679e44d2560a1922cb6fa99acbb7b18957264baf8eadfc7"}, + {file = "fastavro-1.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a509b34c9af71a109c633631ac2f6d2209830e13200d0048f7e9c057fd563f8f"}, + {file = "fastavro-1.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:967edefab470987c024cd5a1fcd04744a50a91e740c7bdf325181043a47f1083"}, + {file = "fastavro-1.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:033c15e8ed02f80f01d58be1cd880b09fd444faf277263d563a727711d47a98a"}, + {file = "fastavro-1.9.3-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:6b38723327603d77080aec56628e13a739415f8596ca0cc41a905615977c6d6b"}, + {file = "fastavro-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:046d75c4400941fd08f0a6855a34ae63bf02ea01f366b5b749942abe10640056"}, + {file = "fastavro-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ab312b8baf0e61ee717878d390022ee1b713d70b244d69efbf3325680f9749"}, + {file = "fastavro-1.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c562fcf8f5091a2446aafd0c2a0da590c24e0b53527a0100d33908e32f20eea8"}, + {file = "fastavro-1.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2aa0111e7ebd076d2a094862bbdf8ea175cebba148fcce6c89ff46b625e334b4"}, + {file = "fastavro-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:652072e0f455ca19a1ee502b527e603389783657c130d81f89df66775979d6f5"}, + {file = "fastavro-1.9.3-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:0a57cdd4edaee36d4216faf801ebc7f53f45e4e1518bdd9832d6f6f1d6e2d88f"}, + {file = "fastavro-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b46a18ebed61573b0823c28eda2716485d283258a83659c7fe6ad3aaeacfed4"}, + {file = "fastavro-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f756f0723f3bd97db20437d0a8e45712839e6ccd7c82f4d82469533be48b4c7"}, + {file = "fastavro-1.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d98d5a08063f5b6d7ac5016a0dfe0698b50d9987cb74686f7dfa8288b7b09e0b"}, + {file = "fastavro-1.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:00698e60db58a2d52cb709df882d451fb7664ebb2f8cb37d9171697e060dc767"}, + {file = "fastavro-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:d021bbc135023194688e88a7431fb0b5e3ce20e27153bf258f2ce08ee1a0106b"}, + {file = "fastavro-1.9.3.tar.gz", hash = "sha256:a30d3d2353f6d3b4f6dcd6a97ae937b3775faddd63f5856fe11ba3b0dbb1756a"}, +] + +[package.extras] +codecs = ["cramjam", "lz4", "zstandard"] +lz4 = ["lz4"] +snappy = ["cramjam"] +zstandard = ["zstandard"] + +[[package]] +name = "fastjsonschema" +version = "2.19.1" +description = "Fastest Python implementation of JSON schema" +optional = false +python-versions = "*" +files = [ + {file = "fastjsonschema-2.19.1-py3-none-any.whl", hash = "sha256:3672b47bc94178c9f23dbb654bf47440155d4db9df5f7bc47643315f9c405cd0"}, + {file = "fastjsonschema-2.19.1.tar.gz", hash = "sha256:e3126a94bdc4623d3de4485f8d468a12f02a67921315ddc87836d6e456dc789d"}, +] + +[package.extras] +devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benchmark", "pytest-cache", "validictory"] + +[[package]] +name = "filelock" +version = "3.13.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "filetype" +version = "1.2.0" +description = "Infer file type and MIME type of any file/buffer. No external dependencies." +optional = false +python-versions = "*" +files = [ + {file = "filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25"}, + {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, +] + +[[package]] +name = "flask" +version = "3.0.2" +description = "A simple framework for building complex web applications." +optional = false +python-versions = ">=3.8" +files = [ + {file = "flask-3.0.2-py3-none-any.whl", hash = "sha256:3232e0e9c850d781933cf0207523d1ece087eb8d87b23777ae38456e2fbe7c6e"}, + {file = "flask-3.0.2.tar.gz", hash = "sha256:822c03f4b799204250a7ee84b1eddc40665395333973dfb9deebfe425fefcb7d"}, +] + +[package.dependencies] +blinker = ">=1.6.2" +click = ">=8.1.3" +itsdangerous = ">=2.1.2" +Jinja2 = ">=3.1.2" +Werkzeug = ">=3.0.0" + +[package.extras] +async = ["asgiref (>=3.2)"] +dotenv = ["python-dotenv"] + +[[package]] +name = "flask-cors" +version = "4.0.0" +description = "A Flask extension adding a decorator for CORS support" +optional = false +python-versions = "*" +files = [ + {file = "Flask-Cors-4.0.0.tar.gz", hash = "sha256:f268522fcb2f73e2ecdde1ef45e2fd5c71cc48fe03cffb4b441c6d1b40684eb0"}, + {file = "Flask_Cors-4.0.0-py2.py3-none-any.whl", hash = "sha256:bc3492bfd6368d27cfe79c7821df5a8a319e1a6d5eab277a3794be19bdc51783"}, +] + +[package.dependencies] +Flask = ">=0.9" + +[[package]] +name = "flatbuffers" +version = "23.5.26" +description = "The FlatBuffers serialization format for Python" +optional = false +python-versions = "*" +files = [ + {file = "flatbuffers-23.5.26-py2.py3-none-any.whl", hash = "sha256:c0ff356da363087b915fde4b8b45bdda73432fc17cddb3c8157472eab1422ad1"}, + {file = "flatbuffers-23.5.26.tar.gz", hash = "sha256:9ea1144cac05ce5d86e2859f431c6cd5e66cd9c78c558317c7955fb8d4c78d89"}, +] + +[[package]] +name = "fqdn" +version = "1.5.1" +description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" +optional = false +python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" +files = [ + {file = "fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014"}, + {file = "fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f"}, +] + +[[package]] +name = "frozenlist" +version = "1.4.1" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"}, + {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"}, + {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, + {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, + {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"}, + {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"}, + {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"}, + {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"}, + {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"}, + {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"}, + {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"}, + {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, + {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, +] + +[[package]] +name = "fsspec" +version = "2024.2.0" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2024.2.0-py3-none-any.whl", hash = "sha256:817f969556fa5916bc682e02ca2045f96ff7f586d45110fcb76022063ad2c7d8"}, + {file = "fsspec-2024.2.0.tar.gz", hash = "sha256:b6ad1a679f760dda52b1168c859d01b7b80648ea6f7f7c7f5a8a91dc3f3ecb84"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "gitdb" +version = "4.0.11" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.41" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.41-py3-none-any.whl", hash = "sha256:c36b6634d069b3f719610175020a9aed919421c87552185b085e04fbbdb10b7c"}, + {file = "GitPython-3.1.41.tar.gz", hash = "sha256:ed66e624884f76df22c8e16066d567aaa5a37d5b5fa19db2c6df6f7156db9048"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[package.extras] +test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "sumtypes"] + +[[package]] +name = "giturlparse" +version = "0.12.0" +description = "A Git URL parsing module (supports parsing and rewriting)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "giturlparse-0.12.0-py2.py3-none-any.whl", hash = "sha256:412b74f2855f1da2fefa89fd8dde62df48476077a72fc19b62039554d27360eb"}, + {file = "giturlparse-0.12.0.tar.gz", hash = "sha256:c0fff7c21acc435491b1779566e038757a205c1ffdcb47e4f81ea52ad8c3859a"}, +] + +[[package]] +name = "google-api-core" +version = "2.16.2" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.16.2.tar.gz", hash = "sha256:032d37b45d1d6bdaf68fb11ff621e2593263a239fa9246e2e94325f9c47876d2"}, + {file = "google_api_core-2.16.2-py3-none-any.whl", hash = "sha256:449ca0e3f14c179b4165b664256066c7861610f70b6ffe54bb01a04e9b466929"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.27.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, + {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-cloud-bigquery" +version = "3.17.2" +description = "Google BigQuery API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-bigquery-3.17.2.tar.gz", hash = "sha256:6e1cf669a40e567ab3289c7b5f2056363da9fcb85d9a4736ee90240d4a7d84ea"}, + {file = "google_cloud_bigquery-3.17.2-py2.py3-none-any.whl", hash = "sha256:cdadf5283dca55a1a350bacf8c8a7466169d3cf46c5a0a3abc5e9aa0b0a51dee"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-cloud-core = ">=1.6.0,<3.0.0dev" +google-resumable-media = ">=0.6.0,<3.0dev" +packaging = ">=20.0.0" +python-dateutil = ">=2.7.2,<3.0dev" +requests = ">=2.21.0,<3.0.0dev" + +[package.extras] +all = ["Shapely (>=1.8.4,<3.0.0dev)", "db-dtypes (>=0.3.0,<2.0.0dev)", "geopandas (>=0.9.0,<1.0dev)", "google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "importlib-metadata (>=1.0.0)", "ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)", "ipywidgets (>=7.7.0)", "opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)", "pandas (>=1.1.0)", "proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)", "pyarrow (>=3.0.0)", "tqdm (>=4.7.4,<5.0.0dev)"] +bigquery-v2 = ["proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)"] +bqstorage = ["google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "pyarrow (>=3.0.0)"] +geopandas = ["Shapely (>=1.8.4,<3.0.0dev)", "geopandas (>=0.9.0,<1.0dev)"] +ipython = ["ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)"] +ipywidgets = ["ipykernel (>=6.0.0)", "ipywidgets (>=7.7.0)"] +opentelemetry = ["opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)"] +pandas = ["db-dtypes (>=0.3.0,<2.0.0dev)", "importlib-metadata (>=1.0.0)", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] +tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] + +[[package]] +name = "google-cloud-core" +version = "2.4.1" +description = "Google Cloud API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, + {file = "google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61"}, +] + +[package.dependencies] +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" + +[package.extras] +grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] + +[[package]] +name = "google-crc32c" +version = "1.5.0" +description = "A python wrapper of the C library 'Google CRC32C'" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-crc32c-1.5.0.tar.gz", hash = "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win32.whl", hash = "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win32.whl", hash = "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win32.whl", hash = "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win32.whl", hash = "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93"}, +] + +[package.extras] +testing = ["pytest"] + +[[package]] +name = "google-resumable-media" +version = "2.7.0" +description = "Utilities for Google Media Downloads and Resumable Uploads" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "google-resumable-media-2.7.0.tar.gz", hash = "sha256:5f18f5fa9836f4b083162064a1c2c98c17239bfda9ca50ad970ccf905f3e625b"}, + {file = "google_resumable_media-2.7.0-py2.py3-none-any.whl", hash = "sha256:79543cfe433b63fd81c0844b7803aba1bb8950b47bedf7d980c38fa123937e08"}, +] + +[package.dependencies] +google-crc32c = ">=1.0,<2.0dev" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.62.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "gql" +version = "3.5.0" +description = "GraphQL client for Python" +optional = false +python-versions = "*" +files = [ + {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, + {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, +] + +[package.dependencies] +anyio = ">=3.0,<5" +backoff = ">=1.11.1,<3.0" +graphql-core = ">=3.2,<3.3" +requests = {version = ">=2.26,<3", optional = true, markers = "extra == \"requests\""} +requests-toolbelt = {version = ">=1.0.0,<2", optional = true, markers = "extra == \"requests\""} +yarl = ">=1.6,<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] +all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] +botocore = ["botocore (>=1.21,<2)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +httpx = ["httpx (>=0.23.1,<1)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +websockets = ["websockets (>=10,<12)"] + +[[package]] +name = "graphql-core" +version = "3.2.3" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "graphql-core-3.2.3.tar.gz", hash = "sha256:06d2aad0ac723e35b1cb47885d3e5c45e956a53bc1b209a9fc5369007fe46676"}, + {file = "graphql_core-3.2.3-py3-none-any.whl", hash = "sha256:5766780452bd5ec8ba133f8bf287dc92713e3868ddd83aee4faab9fc3e303dc3"}, +] + +[[package]] +name = "greenlet" +version = "3.0.3" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, + {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, + {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, + {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, + {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, + {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, + {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, + {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, + {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, + {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, + {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, + {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, + {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, + {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, + {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, + {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil"] + +[[package]] +name = "grpcio" +version = "1.60.1" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpcio-1.60.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:14e8f2c84c0832773fb3958240c69def72357bc11392571f87b2d7b91e0bb092"}, + {file = "grpcio-1.60.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:33aed0a431f5befeffd9d346b0fa44b2c01aa4aeae5ea5b2c03d3e25e0071216"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:fead980fbc68512dfd4e0c7b1f5754c2a8e5015a04dea454b9cada54a8423525"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:082081e6a36b6eb5cf0fd9a897fe777dbb3802176ffd08e3ec6567edd85bc104"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55ccb7db5a665079d68b5c7c86359ebd5ebf31a19bc1a91c982fd622f1e31ff2"}, + {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b54577032d4f235452f77a83169b6527bf4b77d73aeada97d45b2aaf1bf5ce0"}, + {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7d142bcd604166417929b071cd396aa13c565749a4c840d6c702727a59d835eb"}, + {file = "grpcio-1.60.1-cp310-cp310-win32.whl", hash = "sha256:2a6087f234cb570008a6041c8ffd1b7d657b397fdd6d26e83d72283dae3527b1"}, + {file = "grpcio-1.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:f2212796593ad1d0235068c79836861f2201fc7137a99aa2fea7beeb3b101177"}, + {file = "grpcio-1.60.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:79ae0dc785504cb1e1788758c588c711f4e4a0195d70dff53db203c95a0bd303"}, + {file = "grpcio-1.60.1-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4eec8b8c1c2c9b7125508ff7c89d5701bf933c99d3910e446ed531cd16ad5d87"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8c9554ca8e26241dabe7951aa1fa03a1ba0856688ecd7e7bdbdd286ebc272e4c"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91422ba785a8e7a18725b1dc40fbd88f08a5bb4c7f1b3e8739cab24b04fa8a03"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cba6209c96828711cb7c8fcb45ecef8c8859238baf15119daa1bef0f6c84bfe7"}, + {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c71be3f86d67d8d1311c6076a4ba3b75ba5703c0b856b4e691c9097f9b1e8bd2"}, + {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5ef6cfaf0d023c00002ba25d0751e5995fa0e4c9eec6cd263c30352662cbce"}, + {file = "grpcio-1.60.1-cp311-cp311-win32.whl", hash = "sha256:a09506eb48fa5493c58f946c46754ef22f3ec0df64f2b5149373ff31fb67f3dd"}, + {file = "grpcio-1.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:49c9b6a510e3ed8df5f6f4f3c34d7fbf2d2cae048ee90a45cd7415abab72912c"}, + {file = "grpcio-1.60.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:b58b855d0071575ea9c7bc0d84a06d2edfbfccec52e9657864386381a7ce1ae9"}, + {file = "grpcio-1.60.1-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:a731ac5cffc34dac62053e0da90f0c0b8560396a19f69d9703e88240c8f05858"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:cf77f8cf2a651fbd869fbdcb4a1931464189cd210abc4cfad357f1cacc8642a6"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c557e94e91a983e5b1e9c60076a8fd79fea1e7e06848eb2e48d0ccfb30f6e073"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:069fe2aeee02dfd2135d562d0663fe70fbb69d5eed6eb3389042a7e963b54de8"}, + {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb0af13433dbbd1c806e671d81ec75bd324af6ef75171fd7815ca3074fe32bfe"}, + {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2f44c32aef186bbba254129cea1df08a20be414144ac3bdf0e84b24e3f3b2e05"}, + {file = "grpcio-1.60.1-cp312-cp312-win32.whl", hash = "sha256:a212e5dea1a4182e40cd3e4067ee46be9d10418092ce3627475e995cca95de21"}, + {file = "grpcio-1.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:6e490fa5f7f5326222cb9f0b78f207a2b218a14edf39602e083d5f617354306f"}, + {file = "grpcio-1.60.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:4216e67ad9a4769117433814956031cb300f85edc855252a645a9a724b3b6594"}, + {file = "grpcio-1.60.1-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:73e14acd3d4247169955fae8fb103a2b900cfad21d0c35f0dcd0fdd54cd60367"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6ecf21d20d02d1733e9c820fb5c114c749d888704a7ec824b545c12e78734d1c"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33bdea30dcfd4f87b045d404388469eb48a48c33a6195a043d116ed1b9a0196c"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b69e79d00f78c81eecfb38f4516080dc7f36a198b6b37b928f1c13b3c063e9"}, + {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:39aa848794b887120b1d35b1b994e445cc028ff602ef267f87c38122c1add50d"}, + {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:72153a0d2e425f45b884540a61c6639436ddafa1829a42056aa5764b84108b8e"}, + {file = "grpcio-1.60.1-cp37-cp37m-win_amd64.whl", hash = "sha256:50d56280b482875d1f9128ce596e59031a226a8b84bec88cb2bf76c289f5d0de"}, + {file = "grpcio-1.60.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:6d140bdeb26cad8b93c1455fa00573c05592793c32053d6e0016ce05ba267549"}, + {file = "grpcio-1.60.1-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:bc808924470643b82b14fe121923c30ec211d8c693e747eba8a7414bc4351a23"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:70c83bb530572917be20c21f3b6be92cd86b9aecb44b0c18b1d3b2cc3ae47df0"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b106bc52e7f28170e624ba61cc7dc6829566e535a6ec68528f8e1afbed1c41f"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e980cd6db1088c144b92fe376747328d5554bc7960ce583ec7b7d81cd47287"}, + {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0c5807e9152eff15f1d48f6b9ad3749196f79a4a050469d99eecb679be592acc"}, + {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f1c3dc536b3ee124e8b24feb7533e5c70b9f2ef833e3b2e5513b2897fd46763a"}, + {file = "grpcio-1.60.1-cp38-cp38-win32.whl", hash = "sha256:d7404cebcdb11bb5bd40bf94131faf7e9a7c10a6c60358580fe83913f360f929"}, + {file = "grpcio-1.60.1-cp38-cp38-win_amd64.whl", hash = "sha256:c8754c75f55781515a3005063d9a05878b2cfb3cb7e41d5401ad0cf19de14872"}, + {file = "grpcio-1.60.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:0250a7a70b14000fa311de04b169cc7480be6c1a769b190769d347939d3232a8"}, + {file = "grpcio-1.60.1-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:660fc6b9c2a9ea3bb2a7e64ba878c98339abaf1811edca904ac85e9e662f1d73"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76eaaba891083fcbe167aa0f03363311a9f12da975b025d30e94b93ac7a765fc"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d97c65ea7e097056f3d1ead77040ebc236feaf7f71489383d20f3b4c28412a"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb2a2911b028f01c8c64d126f6b632fcd8a9ac975aa1b3855766c94e4107180"}, + {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5a1ebbae7e2214f51b1f23b57bf98eeed2cf1ba84e4d523c48c36d5b2f8829ff"}, + {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a66f4d2a005bc78e61d805ed95dedfcb35efa84b7bba0403c6d60d13a3de2d6"}, + {file = "grpcio-1.60.1-cp39-cp39-win32.whl", hash = "sha256:8d488fbdbf04283f0d20742b64968d44825617aa6717b07c006168ed16488804"}, + {file = "grpcio-1.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:61b7199cd2a55e62e45bfb629a35b71fc2c0cb88f686a047f25b1112d3810904"}, + {file = "grpcio-1.60.1.tar.gz", hash = "sha256:dd1d3a8d1d2e50ad9b59e10aa7f07c7d1be2b367f3f2d33c5fade96ed5460962"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.60.1)"] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.2" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] + +[[package]] +name = "httptools" +version = "0.6.1" +description = "A collection of framework independent HTTP protocol utils." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d2f6c3c4cb1948d912538217838f6e9960bc4a521d7f9b323b3da579cd14532f"}, + {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:00d5d4b68a717765b1fabfd9ca755bd12bf44105eeb806c03d1962acd9b8e563"}, + {file = "httptools-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:639dc4f381a870c9ec860ce5c45921db50205a37cc3334e756269736ff0aac58"}, + {file = "httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e57997ac7fb7ee43140cc03664de5f268813a481dff6245e0075925adc6aa185"}, + {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0ac5a0ae3d9f4fe004318d64b8a854edd85ab76cffbf7ef5e32920faef62f142"}, + {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3f30d3ce413088a98b9db71c60a6ada2001a08945cb42dd65a9a9fe228627658"}, + {file = "httptools-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:1ed99a373e327f0107cb513b61820102ee4f3675656a37a50083eda05dc9541b"}, + {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7a7ea483c1a4485c71cb5f38be9db078f8b0e8b4c4dc0210f531cdd2ddac1ef1"}, + {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:85ed077c995e942b6f1b07583e4eb0a8d324d418954fc6af913d36db7c05a5a0"}, + {file = "httptools-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b0bb634338334385351a1600a73e558ce619af390c2b38386206ac6a27fecfc"}, + {file = "httptools-0.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d9ceb2c957320def533671fc9c715a80c47025139c8d1f3797477decbc6edd2"}, + {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4f0f8271c0a4db459f9dc807acd0eadd4839934a4b9b892f6f160e94da309837"}, + {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6a4f5ccead6d18ec072ac0b84420e95d27c1cdf5c9f1bc8fbd8daf86bd94f43d"}, + {file = "httptools-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:5cceac09f164bcba55c0500a18fe3c47df29b62353198e4f37bbcc5d591172c3"}, + {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75c8022dca7935cba14741a42744eee13ba05db00b27a4b940f0d646bd4d56d0"}, + {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:48ed8129cd9a0d62cf4d1575fcf90fb37e3ff7d5654d3a5814eb3d55f36478c2"}, + {file = "httptools-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f58e335a1402fb5a650e271e8c2d03cfa7cea46ae124649346d17bd30d59c90"}, + {file = "httptools-0.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ad80d7176aa5788902f207a4e79885f0576134695dfb0fefc15b7a4648d503"}, + {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9bb68d3a085c2174c2477eb3ffe84ae9fb4fde8792edb7bcd09a1d8467e30a84"}, + {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b512aa728bc02354e5ac086ce76c3ce635b62f5fbc32ab7082b5e582d27867bb"}, + {file = "httptools-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:97662ce7fb196c785344d00d638fc9ad69e18ee4bfb4000b35a52efe5adcc949"}, + {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8e216a038d2d52ea13fdd9b9c9c7459fb80d78302b257828285eca1c773b99b3"}, + {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3e802e0b2378ade99cd666b5bffb8b2a7cc8f3d28988685dc300469ea8dd86cb"}, + {file = "httptools-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd3e488b447046e386a30f07af05f9b38d3d368d1f7b4d8f7e10af85393db97"}, + {file = "httptools-0.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe467eb086d80217b7584e61313ebadc8d187a4d95bb62031b7bab4b205c3ba3"}, + {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3c3b214ce057c54675b00108ac42bacf2ab8f85c58e3f324a4e963bbc46424f4"}, + {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8ae5b97f690badd2ca27cbf668494ee1b6d34cf1c464271ef7bfa9ca6b83ffaf"}, + {file = "httptools-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:405784577ba6540fa7d6ff49e37daf104e04f4b4ff2d1ac0469eaa6a20fde084"}, + {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:95fb92dd3649f9cb139e9c56604cc2d7c7bf0fc2e7c8d7fbd58f96e35eddd2a3"}, + {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dcbab042cc3ef272adc11220517278519adf8f53fd3056d0e68f0a6f891ba94e"}, + {file = "httptools-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cf2372e98406efb42e93bfe10f2948e467edfd792b015f1b4ecd897903d3e8d"}, + {file = "httptools-0.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:678fcbae74477a17d103b7cae78b74800d795d702083867ce160fc202104d0da"}, + {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e0b281cf5a125c35f7f6722b65d8542d2e57331be573e9e88bc8b0115c4a7a81"}, + {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:95658c342529bba4e1d3d2b1a874db16c7cca435e8827422154c9da76ac4e13a"}, + {file = "httptools-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ebaec1bf683e4bf5e9fbb49b8cc36da482033596a415b3e4ebab5a4c0d7ec5e"}, + {file = "httptools-0.6.1.tar.gz", hash = "sha256:c6e26c30455600b95d94b1b836085138e82f177351454ee841c148f93a9bad5a"}, +] + +[package.extras] +test = ["Cython (>=0.29.24,<0.30.0)"] + +[[package]] +name = "httpx" +version = "0.26.0" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "huggingface-hub" +version = "0.20.3" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "huggingface_hub-0.20.3-py3-none-any.whl", hash = "sha256:d988ae4f00d3e307b0c80c6a05ca6dbb7edba8bba3079f74cda7d9c2e562a7b6"}, + {file = "huggingface_hub-0.20.3.tar.gz", hash = "sha256:94e7f8e074475fbc67d6a71957b678e1b4a74ff1b64a644fd6cbb83da962d05d"}, +] + +[package.dependencies] +filelock = "*" +fsspec = ">=2023.5.0" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "mypy (==1.5.1)", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.1.3)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "mypy (==1.5.1)", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.1.3)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +inference = ["aiohttp", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)"] +quality = ["mypy (==1.5.1)", "ruff (>=0.1.3)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +torch = ["torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] + +[[package]] +name = "humanfriendly" +version = "10.0" +description = "Human friendly output for text interfaces using Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, +] + +[package.dependencies] +pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "importlib-metadata" +version = "6.11.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-6.11.0-py3-none-any.whl", hash = "sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b"}, + {file = "importlib_metadata-6.11.0.tar.gz", hash = "sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] + +[[package]] +name = "importlib-resources" +version = "6.1.1" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_resources-6.1.1-py3-none-any.whl", hash = "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6"}, + {file = "importlib_resources-6.1.1.tar.gz", hash = "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff", "zipp (>=3.17)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "ipykernel" +version = "6.29.1" +description = "IPython Kernel for Jupyter" +optional = false +python-versions = ">=3.8" +files = [ + {file = "ipykernel-6.29.1-py3-none-any.whl", hash = "sha256:e5dfba210fc9da74a5dae8fa6c41f816e11bd18d10381b2517d9a0d57cc987c4"}, + {file = "ipykernel-6.29.1.tar.gz", hash = "sha256:1547352b32da95a2761011a8dac2af930c26a0703dfa07690d16b7d74dac0ba1"}, +] + +[package.dependencies] +appnope = {version = "*", markers = "platform_system == \"Darwin\""} +comm = ">=0.1.1" +debugpy = ">=1.6.5" +ipython = ">=7.23.1" +jupyter-client = ">=6.1.12" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +matplotlib-inline = ">=0.1" +nest-asyncio = "*" +packaging = "*" +psutil = "*" +pyzmq = ">=24" +tornado = ">=6.1" +traitlets = ">=5.4.0" + +[package.extras] +cov = ["coverage[toml]", "curio", "matplotlib", "pytest-cov", "trio"] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "trio"] +pyqt5 = ["pyqt5"] +pyside6 = ["pyside6"] +test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio (==0.23.4)", "pytest-cov", "pytest-timeout"] + +[[package]] +name = "ipynbname" +version = "2023.2.0.0" +description = "Simply returns either notebook filename or the full path to the notebook when run from Jupyter notebook in browser." +optional = false +python-versions = ">=3.4" +files = [ + {file = "ipynbname-2023.2.0.0-py3-none-any.whl", hash = "sha256:a88f4b1f840c4a12fe6e39862f936c1ccec70705fb034a8639f8e7752bf82637"}, + {file = "ipynbname-2023.2.0.0.tar.gz", hash = "sha256:462bbdd7956624876dc4ea81fa792845ab3870e444ca1f6b7acdaea37d8c9ebf"}, +] + +[package.dependencies] +ipykernel = "*" + +[[package]] +name = "ipython" +version = "8.21.0" +description = "IPython: Productive Interactive Computing" +optional = false +python-versions = ">=3.10" +files = [ + {file = "ipython-8.21.0-py3-none-any.whl", hash = "sha256:1050a3ab8473488d7eee163796b02e511d0735cf43a04ba2a8348bd0f2eaf8a5"}, + {file = "ipython-8.21.0.tar.gz", hash = "sha256:48fbc236fbe0e138b88773fa0437751f14c3645fb483f1d4c5dee58b37e5ce73"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +prompt-toolkit = ">=3.0.41,<3.1.0" +pygments = ">=2.4.0" +stack-data = "*" +traitlets = ">=5" + +[package.extras] +all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.23)", "pandas", "pickleshare", "pytest (<8)", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] +black = ["black"] +doc = ["docrepr", "exceptiongroup", "ipykernel", "matplotlib", "pickleshare", "pytest (<8)", "pytest-asyncio (<0.22)", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["ipywidgets", "notebook"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["pickleshare", "pytest (<8)", "pytest-asyncio (<0.22)", "testpath"] +test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.23)", "pandas", "pickleshare", "pytest (<8)", "pytest-asyncio (<0.22)", "testpath", "trio"] + +[[package]] +name = "isoduration" +version = "20.11.0" +description = "Operations with ISO 8601 durations" +optional = false +python-versions = ">=3.7" +files = [ + {file = "isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042"}, + {file = "isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9"}, +] + +[package.dependencies] +arrow = ">=0.15.0" + +[[package]] +name = "itsdangerous" +version = "2.1.2" +description = "Safely pass data to untrusted environments and back." +optional = false +python-versions = ">=3.7" +files = [ + {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, + {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, +] + +[[package]] +name = "janus" +version = "1.0.0" +description = "Mixed sync-async queue to interoperate between asyncio tasks and classic threads" +optional = false +python-versions = ">=3.7" +files = [ + {file = "janus-1.0.0-py3-none-any.whl", hash = "sha256:2596ea5482711c1ee3ef2df6c290aaf370a13c55a007826e8f7c32d696d1d00a"}, + {file = "janus-1.0.0.tar.gz", hash = "sha256:df976f2cdcfb034b147a2d51edfc34ff6bfb12d4e2643d3ad0e10de058cb1612"}, +] + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[[package]] +name = "jedi" +version = "0.19.1" +description = "An autocompletion tool for Python that can be used for text editors." +optional = false +python-versions = ">=3.6" +files = [ + {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"}, + {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, +] + +[package.dependencies] +parso = ">=0.8.3,<0.9.0" + +[package.extras] +docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "joblib" +version = "1.3.2" +description = "Lightweight pipelining with Python functions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "joblib-1.3.2-py3-none-any.whl", hash = "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"}, + {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, +] + +[[package]] +name = "json5" +version = "0.9.14" +description = "A Python implementation of the JSON5 data format." +optional = false +python-versions = "*" +files = [ + {file = "json5-0.9.14-py2.py3-none-any.whl", hash = "sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f"}, + {file = "json5-0.9.14.tar.gz", hash = "sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02"}, +] + +[package.extras] +dev = ["hypothesis"] + +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpath-python" +version = "1.0.6" +description = "A more powerful JSONPath implementation in modern python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "jsonpath-python-1.0.6.tar.gz", hash = "sha256:dd5be4a72d8a2995c3f583cf82bf3cd1a9544cfdabf2d22595b67aff07349666"}, + {file = "jsonpath_python-1.0.6-py3-none-any.whl", hash = "sha256:1e3b78df579f5efc23565293612decee04214609208a2335884b3ee3f786b575"}, +] + +[[package]] +name = "jsonpointer" +version = "2.4" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, + {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, +] + +[[package]] +name = "jsonschema" +version = "4.21.1" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema-4.21.1-py3-none-any.whl", hash = "sha256:7996507afae316306f9e2290407761157c6f78002dcf7419acb99822143d1c6f"}, + {file = "jsonschema-4.21.1.tar.gz", hash = "sha256:85727c00279f5fa6bedbe6238d2aa6403bedd8b4864ab11207d07df3cc1b2ee5"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +fqdn = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +idna = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +isoduration = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format-nongpl\""} +jsonschema-specifications = ">=2023.03.6" +referencing = ">=0.28.4" +rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +rfc3986-validator = {version = ">0.1.0", optional = true, markers = "extra == \"format-nongpl\""} +rpds-py = ">=0.7.1" +uri-template = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +webcolors = {version = ">=1.11", optional = true, markers = "extra == \"format-nongpl\""} + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "jsonschema-specifications" +version = "2023.12.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, + {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, +] + +[package.dependencies] +referencing = ">=0.31.0" + +[[package]] +name = "jupyter-client" +version = "8.6.0" +description = "Jupyter protocol implementation and client libraries" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter_client-8.6.0-py3-none-any.whl", hash = "sha256:909c474dbe62582ae62b758bca86d6518c85234bdee2d908c778db6d72f39d99"}, + {file = "jupyter_client-8.6.0.tar.gz", hash = "sha256:0642244bb83b4764ae60d07e010e15f0e2d275ec4e918a8f7b80fbbef3ca60c7"}, +] + +[package.dependencies] +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +python-dateutil = ">=2.8.2" +pyzmq = ">=23.0" +tornado = ">=6.2" +traitlets = ">=5.3" + +[package.extras] +docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] +test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] + +[[package]] +name = "jupyter-core" +version = "5.7.1" +description = "Jupyter core package. A base package on which Jupyter projects rely." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter_core-5.7.1-py3-none-any.whl", hash = "sha256:c65c82126453a723a2804aa52409930434598fd9d35091d63dfb919d2b765bb7"}, + {file = "jupyter_core-5.7.1.tar.gz", hash = "sha256:de61a9d7fc71240f688b2fb5ab659fbb56979458dc66a71decd098e03c79e218"}, +] + +[package.dependencies] +platformdirs = ">=2.5" +pywin32 = {version = ">=300", markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""} +traitlets = ">=5.3" + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] +test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] + +[[package]] +name = "jupyter-events" +version = "0.9.0" +description = "Jupyter Event System library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter_events-0.9.0-py3-none-any.whl", hash = "sha256:d853b3c10273ff9bc8bb8b30076d65e2c9685579db736873de6c2232dde148bf"}, + {file = "jupyter_events-0.9.0.tar.gz", hash = "sha256:81ad2e4bc710881ec274d31c6c50669d71bbaa5dd9d01e600b56faa85700d399"}, +] + +[package.dependencies] +jsonschema = {version = ">=4.18.0", extras = ["format-nongpl"]} +python-json-logger = ">=2.0.4" +pyyaml = ">=5.3" +referencing = "*" +rfc3339-validator = "*" +rfc3986-validator = ">=0.1.1" +traitlets = ">=5.3" + +[package.extras] +cli = ["click", "rich"] +docs = ["jupyterlite-sphinx", "myst-parser", "pydata-sphinx-theme", "sphinxcontrib-spelling"] +test = ["click", "pre-commit", "pytest (>=7.0)", "pytest-asyncio (>=0.19.0)", "pytest-console-scripts", "rich"] + +[[package]] +name = "jupyter-lsp" +version = "2.2.2" +description = "Multi-Language Server WebSocket proxy for Jupyter Notebook/Lab server" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter-lsp-2.2.2.tar.gz", hash = "sha256:256d24620542ae4bba04a50fc1f6ffe208093a07d8e697fea0a8d1b8ca1b7e5b"}, + {file = "jupyter_lsp-2.2.2-py3-none-any.whl", hash = "sha256:3b95229e4168355a8c91928057c1621ac3510ba98b2a925e82ebd77f078b1aa5"}, +] + +[package.dependencies] +jupyter-server = ">=1.1.2" + +[[package]] +name = "jupyter-server" +version = "2.12.5" +description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter_server-2.12.5-py3-none-any.whl", hash = "sha256:184a0f82809a8522777cfb6b760ab6f4b1bb398664c5860a27cec696cb884923"}, + {file = "jupyter_server-2.12.5.tar.gz", hash = "sha256:0edb626c94baa22809be1323f9770cf1c00a952b17097592e40d03e6a3951689"}, +] + +[package.dependencies] +anyio = ">=3.1.0" +argon2-cffi = "*" +jinja2 = "*" +jupyter-client = ">=7.4.4" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-events = ">=0.9.0" +jupyter-server-terminals = "*" +nbconvert = ">=6.4.4" +nbformat = ">=5.3.0" +overrides = "*" +packaging = "*" +prometheus-client = "*" +pywinpty = {version = "*", markers = "os_name == \"nt\""} +pyzmq = ">=24" +send2trash = ">=1.8.2" +terminado = ">=0.8.3" +tornado = ">=6.2.0" +traitlets = ">=5.6.0" +websocket-client = "*" + +[package.extras] +docs = ["ipykernel", "jinja2", "jupyter-client", "jupyter-server", "myst-parser", "nbformat", "prometheus-client", "pydata-sphinx-theme", "send2trash", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-openapi (>=0.8.0)", "sphinxcontrib-spelling", "sphinxemoji", "tornado", "typing-extensions"] +test = ["flaky", "ipykernel", "pre-commit", "pytest (>=7.0)", "pytest-console-scripts", "pytest-jupyter[server] (>=0.4)", "pytest-timeout", "requests"] + +[[package]] +name = "jupyter-server-terminals" +version = "0.5.2" +description = "A Jupyter Server Extension Providing Terminals." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter_server_terminals-0.5.2-py3-none-any.whl", hash = "sha256:1b80c12765da979513c42c90215481bbc39bd8ae7c0350b4f85bc3eb58d0fa80"}, + {file = "jupyter_server_terminals-0.5.2.tar.gz", hash = "sha256:396b5ccc0881e550bf0ee7012c6ef1b53edbde69e67cab1d56e89711b46052e8"}, +] + +[package.dependencies] +pywinpty = {version = ">=2.0.3", markers = "os_name == \"nt\""} +terminado = ">=0.8.3" + +[package.extras] +docs = ["jinja2", "jupyter-server", "mistune (<4.0)", "myst-parser", "nbformat", "packaging", "pydata-sphinx-theme", "sphinxcontrib-github-alt", "sphinxcontrib-openapi", "sphinxcontrib-spelling", "sphinxemoji", "tornado"] +test = ["jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-jupyter[server] (>=0.5.3)", "pytest-timeout"] + +[[package]] +name = "jupyterlab" +version = "4.1.0" +description = "JupyterLab computational environment" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyterlab-4.1.0-py3-none-any.whl", hash = "sha256:5380e85fb4f11a227ed2db13103e513cfea274d1011f6210e62d611e92e0369d"}, + {file = "jupyterlab-4.1.0.tar.gz", hash = "sha256:92cdfd86c53e163fb9e91e14497901153536c5a889c9225dade270f6107a077f"}, +] + +[package.dependencies] +async-lru = ">=1.0.0" +httpx = ">=0.25.0" +ipykernel = "*" +jinja2 = ">=3.0.3" +jupyter-core = "*" +jupyter-lsp = ">=2.0.0" +jupyter-server = ">=2.4.0,<3" +jupyterlab-server = ">=2.19.0,<3" +notebook-shim = ">=0.2" +packaging = "*" +tomli = {version = "*", markers = "python_version < \"3.11\""} +tornado = ">=6.2.0" +traitlets = "*" + +[package.extras] +dev = ["build", "bump2version", "coverage", "hatch", "pre-commit", "pytest-cov", "ruff (==0.1.15)"] +docs = ["jsx-lexer", "myst-parser", "pydata-sphinx-theme (>=0.13.0)", "pytest", "pytest-check-links", "pytest-jupyter", "sphinx (>=1.8,<7.3.0)", "sphinx-copybutton"] +docs-screenshots = ["altair (==5.2.0)", "ipython (==8.16.1)", "ipywidgets (==8.1.1)", "jupyterlab-geojson (==3.4.0)", "jupyterlab-language-pack-zh-cn (==4.0.post6)", "matplotlib (==3.8.2)", "nbconvert (>=7.0.0)", "pandas (==2.2.0)", "scipy (==1.12.0)", "vega-datasets (==0.9.0)"] +test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter (>=0.5.3)", "pytest-timeout", "pytest-tornasync", "requests", "requests-cache", "virtualenv"] + +[[package]] +name = "jupyterlab-pygments" +version = "0.3.0" +description = "Pygments theme using JupyterLab CSS variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyterlab_pygments-0.3.0-py3-none-any.whl", hash = "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780"}, + {file = "jupyterlab_pygments-0.3.0.tar.gz", hash = "sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d"}, +] + +[[package]] +name = "jupyterlab-server" +version = "2.25.2" +description = "A set of server components for JupyterLab and JupyterLab like applications." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyterlab_server-2.25.2-py3-none-any.whl", hash = "sha256:5b1798c9cc6a44f65c757de9f97fc06fc3d42535afbf47d2ace5e964ab447aaf"}, + {file = "jupyterlab_server-2.25.2.tar.gz", hash = "sha256:bd0ec7a99ebcedc8bcff939ef86e52c378e44c2707e053fcd81d046ce979ee63"}, +] + +[package.dependencies] +babel = ">=2.10" +jinja2 = ">=3.0.3" +json5 = ">=0.9.0" +jsonschema = ">=4.18.0" +jupyter-server = ">=1.21,<3" +packaging = ">=21.3" +requests = ">=2.31" + +[package.extras] +docs = ["autodoc-traits", "jinja2 (<3.2.0)", "mistune (<4)", "myst-parser", "pydata-sphinx-theme", "sphinx", "sphinx-copybutton", "sphinxcontrib-openapi (>0.8)"] +openapi = ["openapi-core (>=0.18.0,<0.19.0)", "ruamel-yaml"] +test = ["hatch", "ipykernel", "openapi-core (>=0.18.0,<0.19.0)", "openapi-spec-validator (>=0.6.0,<0.8.0)", "pytest (>=7.0)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter[server] (>=0.6.2)", "pytest-timeout", "requests-mock", "ruamel-yaml", "sphinxcontrib-spelling", "strict-rfc3339", "werkzeug"] + +[[package]] +name = "kubernetes" +version = "29.0.0" +description = "Kubernetes python client" +optional = false +python-versions = ">=3.6" +files = [ + {file = "kubernetes-29.0.0-py2.py3-none-any.whl", hash = "sha256:ab8cb0e0576ccdfb71886366efb102c6a20f268d817be065ce7f9909c631e43e"}, + {file = "kubernetes-29.0.0.tar.gz", hash = "sha256:c4812e227ae74d07d53c88293e564e54b850452715a59a927e7e1bc6b9a60459"}, +] + +[package.dependencies] +certifi = ">=14.05.14" +google-auth = ">=1.0.1" +oauthlib = ">=3.2.2" +python-dateutil = ">=2.5.3" +pyyaml = ">=5.4.1" +requests = "*" +requests-oauthlib = "*" +six = ">=1.9.0" +urllib3 = ">=1.24.2" +websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0" + +[package.extras] +adal = ["adal (>=1.0.2)"] + +[[package]] +name = "langchain" +version = "0.1.5" +description = "Building applications with LLMs through composability" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "langchain-0.1.5-py3-none-any.whl", hash = "sha256:4614118d4a95b2e7ba3611a0b6b21707a259a21652a04fbe3c31205bcf3fcd50"}, + {file = "langchain-0.1.5.tar.gz", hash = "sha256:69603a5bb21b044ddea69d38131dbbf47475afdf79728644faa67d1ad325d652"}, +] + +[package.dependencies] +aiohttp = ">=3.8.3,<4.0.0" +async-timeout = {version = ">=4.0.0,<5.0.0", markers = "python_version < \"3.11\""} +dataclasses-json = ">=0.5.7,<0.7" +jsonpatch = ">=1.33,<2.0" +langchain-community = ">=0.0.17,<0.1" +langchain-core = ">=0.1.16,<0.2" +langsmith = ">=0.0.83,<0.1" +numpy = ">=1,<2" +pydantic = ">=1,<3" +PyYAML = ">=5.3" +requests = ">=2,<3" +SQLAlchemy = ">=1.4,<3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +azure = ["azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-textanalytics (>=5.3.0,<6.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-core (>=1.26.4,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "azure-search-documents (==11.4.0b8)", "openai (<2)"] +clarifai = ["clarifai (>=9.1.0)"] +cli = ["typer (>=0.9.0,<0.10.0)"] +cohere = ["cohere (>=4,<5)"] +docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"] +embeddings = ["sentence-transformers (>=2,<3)"] +extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<5)", "couchbase (>=4.1.9,<5.0.0)", "dashvector (>=1.0.1,<2.0.0)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "langchain-openai (>=0.0.2,<0.1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "rdflib (==7.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] +javascript = ["esprima (>=4.0.1,<5.0.0)"] +llms = ["clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (<2)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"] +openai = ["openai (<2)", "tiktoken (>=0.3.2,<0.6.0)"] +qdrant = ["qdrant-client (>=1.3.1,<2.0.0)"] +text-helpers = ["chardet (>=5.1.0,<6.0.0)"] + +[[package]] +name = "langchain-community" +version = "0.0.18" +description = "Community contributed LangChain integrations." +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "langchain_community-0.0.18-py3-none-any.whl", hash = "sha256:b87e20c1fa3f37e9608d7ccc08b4d8ed86f875b8c1e735d0464ae986e41c5a71"}, + {file = "langchain_community-0.0.18.tar.gz", hash = "sha256:f044f331b418f16148b76929f27cc2107fce2d190ea3fae0cdaf155ceda9892f"}, +] + +[package.dependencies] +aiohttp = ">=3.8.3,<4.0.0" +dataclasses-json = ">=0.5.7,<0.7" +langchain-core = ">=0.1.19,<0.2" +langsmith = ">=0.0.83,<0.1" +numpy = ">=1,<2" +PyYAML = ">=5.3" +requests = ">=2,<3" +SQLAlchemy = ">=1.4,<3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +cli = ["typer (>=0.9.0,<0.10.0)"] +extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "azure-ai-documentintelligence (>=1.0.0b1,<2.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<5)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "elasticsearch (>=8.12.0,<9.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "gradientai (>=1.4.0,<2.0.0)", "hdbcli (>=2.19.21,<3.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "httpx (>=0.24.1,<0.25.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "nvidia-riva-client (>=2.14.0,<3.0.0)", "oci (>=2.119.1,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "oracle-ads (>=2.9.1,<3.0.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "rdflib (==7.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)", "zhipuai (>=1.0.7,<2.0.0)"] + +[[package]] +name = "langchain-core" +version = "0.1.19" +description = "Building applications with LLMs through composability" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "langchain_core-0.1.19-py3-none-any.whl", hash = "sha256:46b5fd54181df5aa6d3041d61beb2b91e5437b6742274e7924a97734ed62cf43"}, + {file = "langchain_core-0.1.19.tar.gz", hash = "sha256:30539190a63dff53e995f10aefb943b4f7e01aba4bf28fd1e13016b040c0e9da"}, +] + +[package.dependencies] +anyio = ">=3,<5" +jsonpatch = ">=1.33,<2.0" +langsmith = ">=0.0.83,<0.1" +packaging = ">=23.2,<24.0" +pydantic = ">=1,<3" +PyYAML = ">=5.3" +requests = ">=2,<3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +extended-testing = ["jinja2 (>=3,<4)"] + +[[package]] +name = "langchain-openai" +version = "0.0.5" +description = "An integration package connecting OpenAI and LangChain" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "langchain_openai-0.0.5-py3-none-any.whl", hash = "sha256:93b37dfac274adad65e46d5d6e71411e00c6984bcc5e10f1d6bb58e7944dc01b"}, + {file = "langchain_openai-0.0.5.tar.gz", hash = "sha256:f317fee5b652949ad96ad7edf8ef7a044a6a3f0cc71d1e12f9d5261789fd68c4"}, +] + +[package.dependencies] +langchain-core = ">=0.1.16,<0.2" +numpy = ">=1,<2" +openai = ">=1.10.0,<2.0.0" +tiktoken = ">=0.5.2,<0.6.0" + +[[package]] +name = "langdetect" +version = "1.0.9" +description = "Language detection library ported from Google's language-detection." +optional = false +python-versions = "*" +files = [ + {file = "langdetect-1.0.9-py2-none-any.whl", hash = "sha256:7cbc0746252f19e76f77c0b1690aadf01963be835ef0cd4b56dddf2a8f1dfc2a"}, + {file = "langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "langsmith" +version = "0.0.87" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "langsmith-0.0.87-py3-none-any.whl", hash = "sha256:8903d3811b9fc89eb18f5961c8e6935fbd2d0f119884fbf30dc70b8f8f4121fc"}, + {file = "langsmith-0.0.87.tar.gz", hash = "sha256:36c4cc47e5b54be57d038036a30fb19ce6e4c73048cd7a464b8f25b459694d34"}, +] + +[package.dependencies] +pydantic = ">=1,<3" +requests = ">=2,<3" + +[[package]] +name = "litellm" +version = "1.23.0" +description = "Library to easily interface with LLM API providers" +optional = false +python-versions = ">=3.8, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*" +files = [ + {file = "litellm-1.23.0-py3-none-any.whl", hash = "sha256:5d8718c72521c75e6efaf521f5ca0734989d7c9b2ab451d1aa0812fa2e34885e"}, + {file = "litellm-1.23.0.tar.gz", hash = "sha256:3ca4f88876c0e043879054750779a47128fc5ac1afcdd700be9a6dd934b1dca2"}, +] + +[package.dependencies] +aiohttp = "*" +click = "*" +importlib-metadata = ">=6.8.0" +jinja2 = ">=3.1.2,<4.0.0" +openai = ">=1.0.0" +python-dotenv = ">=0.2.0" +requests = ">=2.31.0,<3.0.0" +tiktoken = ">=0.4.0" +tokenizers = "*" + +[package.extras] +extra-proxy = ["streamlit (>=1.29.0,<2.0.0)"] +proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "fastapi (>=0.104.1,<0.105.0)", "fastapi-sso (>=0.10.0,<0.11.0)", "gunicorn (>=21.2.0,<22.0.0)", "orjson (>=3.9.7,<4.0.0)", "python-multipart (>=0.0.6,<0.0.7)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"] + +[[package]] +name = "llvmlite" +version = "0.42.0" +description = "lightweight wrapper around basic LLVM functionality" +optional = false +python-versions = ">=3.9" +files = [ + {file = "llvmlite-0.42.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3366938e1bf63d26c34fbfb4c8e8d2ded57d11e0567d5bb243d89aab1eb56098"}, + {file = "llvmlite-0.42.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c35da49666a21185d21b551fc3caf46a935d54d66969d32d72af109b5e7d2b6f"}, + {file = "llvmlite-0.42.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70f44ccc3c6220bd23e0ba698a63ec2a7d3205da0d848804807f37fc243e3f77"}, + {file = "llvmlite-0.42.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:763f8d8717a9073b9e0246998de89929071d15b47f254c10eef2310b9aac033d"}, + {file = "llvmlite-0.42.0-cp310-cp310-win_amd64.whl", hash = "sha256:8d90edf400b4ceb3a0e776b6c6e4656d05c7187c439587e06f86afceb66d2be5"}, + {file = "llvmlite-0.42.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ae511caed28beaf1252dbaf5f40e663f533b79ceb408c874c01754cafabb9cbf"}, + {file = "llvmlite-0.42.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81e674c2fe85576e6c4474e8c7e7aba7901ac0196e864fe7985492b737dbab65"}, + {file = "llvmlite-0.42.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb3975787f13eb97629052edb5017f6c170eebc1c14a0433e8089e5db43bcce6"}, + {file = "llvmlite-0.42.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5bece0cdf77f22379f19b1959ccd7aee518afa4afbd3656c6365865f84903f9"}, + {file = "llvmlite-0.42.0-cp311-cp311-win_amd64.whl", hash = "sha256:7e0c4c11c8c2aa9b0701f91b799cb9134a6a6de51444eff5a9087fc7c1384275"}, + {file = "llvmlite-0.42.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:08fa9ab02b0d0179c688a4216b8939138266519aaa0aa94f1195a8542faedb56"}, + {file = "llvmlite-0.42.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b2fce7d355068494d1e42202c7aff25d50c462584233013eb4470c33b995e3ee"}, + {file = "llvmlite-0.42.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebe66a86dc44634b59a3bc860c7b20d26d9aaffcd30364ebe8ba79161a9121f4"}, + {file = "llvmlite-0.42.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d47494552559e00d81bfb836cf1c4d5a5062e54102cc5767d5aa1e77ccd2505c"}, + {file = "llvmlite-0.42.0-cp312-cp312-win_amd64.whl", hash = "sha256:05cb7e9b6ce69165ce4d1b994fbdedca0c62492e537b0cc86141b6e2c78d5888"}, + {file = "llvmlite-0.42.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bdd3888544538a94d7ec99e7c62a0cdd8833609c85f0c23fcb6c5c591aec60ad"}, + {file = "llvmlite-0.42.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d0936c2067a67fb8816c908d5457d63eba3e2b17e515c5fe00e5ee2bace06040"}, + {file = "llvmlite-0.42.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a78ab89f1924fc11482209f6799a7a3fc74ddc80425a7a3e0e8174af0e9e2301"}, + {file = "llvmlite-0.42.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7599b65c7af7abbc978dbf345712c60fd596aa5670496561cc10e8a71cebfb2"}, + {file = "llvmlite-0.42.0-cp39-cp39-win_amd64.whl", hash = "sha256:43d65cc4e206c2e902c1004dd5418417c4efa6c1d04df05c6c5675a27e8ca90e"}, + {file = "llvmlite-0.42.0.tar.gz", hash = "sha256:f92b09243c0cc3f457da8b983f67bd8e1295d0f5b3746c7a1861d7a99403854a"}, +] + +[[package]] +name = "lxml" +version = "5.1.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e"}, + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da"}, + {file = "lxml-5.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2befa20a13f1a75c751f47e00929fb3433d67eb9923c2c0b364de449121f447c"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22b7ee4c35f374e2c20337a95502057964d7e35b996b1c667b5c65c567d2252a"}, + {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf8443781533b8d37b295016a4b53c1494fa9a03573c09ca5104550c138d5c05"}, + {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147"}, + {file = "lxml-5.1.0-cp310-cp310-win32.whl", hash = "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93"}, + {file = "lxml-5.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af8920ce4a55ff41167ddbc20077f5698c2e710ad3353d32a07d3264f3a2021e"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cfced4a069003d8913408e10ca8ed092c49a7f6cefee9bb74b6b3e860683b45"}, + {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9e5ac3437746189a9b4121db2a7b86056ac8786b12e88838696899328fc44bb2"}, + {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204"}, + {file = "lxml-5.1.0-cp311-cp311-win32.whl", hash = "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b"}, + {file = "lxml-5.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16dd953fb719f0ffc5bc067428fc9e88f599e15723a85618c45847c96f11f431"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16018f7099245157564d7148165132c70adb272fb5a17c048ba70d9cc542a1a1"}, + {file = "lxml-5.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:82cd34f1081ae4ea2ede3d52f71b7be313756e99b4b5f829f89b12da552d3aa3"}, + {file = "lxml-5.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:19a1bc898ae9f06bccb7c3e1dfd73897ecbbd2c96afe9095a6026016e5ca97b8"}, + {file = "lxml-5.1.0-cp312-cp312-win32.whl", hash = "sha256:13521a321a25c641b9ea127ef478b580b5ec82aa2e9fc076c86169d161798b01"}, + {file = "lxml-5.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:1ad17c20e3666c035db502c78b86e58ff6b5991906e55bdbef94977700c72623"}, + {file = "lxml-5.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:24ef5a4631c0b6cceaf2dbca21687e29725b7c4e171f33a8f8ce23c12558ded1"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d2900b7f5318bc7ad8631d3d40190b95ef2aa8cc59473b73b294e4a55e9f30f"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:601f4a75797d7a770daed8b42b97cd1bb1ba18bd51a9382077a6a247a12aa38d"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4b68c961b5cc402cbd99cca5eb2547e46ce77260eb705f4d117fd9c3f932b95"}, + {file = "lxml-5.1.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:afd825e30f8d1f521713a5669b63657bcfe5980a916c95855060048b88e1adb7"}, + {file = "lxml-5.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:262bc5f512a66b527d026518507e78c2f9c2bd9eb5c8aeeb9f0eb43fcb69dc67"}, + {file = "lxml-5.1.0-cp36-cp36m-win32.whl", hash = "sha256:e856c1c7255c739434489ec9c8aa9cdf5179785d10ff20add308b5d673bed5cd"}, + {file = "lxml-5.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c7257171bb8d4432fe9d6fdde4d55fdbe663a63636a17f7f9aaba9bcb3153ad7"}, + {file = "lxml-5.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b9e240ae0ba96477682aa87899d94ddec1cc7926f9df29b1dd57b39e797d5ab5"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a96f02ba1bcd330807fc060ed91d1f7a20853da6dd449e5da4b09bfcc08fdcf5"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3898ae2b58eeafedfe99e542a17859017d72d7f6a63de0f04f99c2cb125936"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61c5a7edbd7c695e54fca029ceb351fc45cd8860119a0f83e48be44e1c464862"}, + {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3aeca824b38ca78d9ee2ab82bd9883083d0492d9d17df065ba3b94e88e4d7ee6"}, + {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764"}, + {file = "lxml-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8"}, + {file = "lxml-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45"}, + {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:98f3f020a2b736566c707c8e034945c02aa94e124c24f77ca097c446f81b01f1"}, + {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e"}, + {file = "lxml-5.1.0-cp38-cp38-win32.whl", hash = "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a"}, + {file = "lxml-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f8b0c78e7aac24979ef09b7f50da871c2de2def043d468c4b41f512d831e912"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9bcf86dfc8ff3e992fed847c077bd875d9e0ba2fa25d859c3a0f0f76f07f0c8d"}, + {file = "lxml-5.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:49a9b4af45e8b925e1cd6f3b15bbba2c81e7dba6dce170c677c9cda547411e14"}, + {file = "lxml-5.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:280f3edf15c2a967d923bcfb1f8f15337ad36f93525828b40a0f9d6c2ad24890"}, + {file = "lxml-5.1.0-cp39-cp39-win32.whl", hash = "sha256:ed7326563024b6e91fef6b6c7a1a2ff0a71b97793ac33dbbcf38f6005e51ff6e"}, + {file = "lxml-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:8d7b4beebb178e9183138f552238f7e6613162a42164233e2bda00cb3afac58f"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9bd0ae7cc2b85320abd5e0abad5ccee5564ed5f0cc90245d2f9a8ef330a8deae"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8c1d679df4361408b628f42b26a5d62bd3e9ba7f0c0e7969f925021554755aa"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2ad3a8ce9e8a767131061a22cd28fdffa3cd2dc193f399ff7b81777f3520e372"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:304128394c9c22b6569eba2a6d98392b56fbdfbad58f83ea702530be80d0f9df"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d74fcaf87132ffc0447b3c685a9f862ffb5b43e70ea6beec2fb8057d5d2a1fea"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:8cf5877f7ed384dabfdcc37922c3191bf27e55b498fecece9fd5c2c7aaa34c33"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:877efb968c3d7eb2dad540b6cabf2f1d3c0fbf4b2d309a3c141f79c7e0061324"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f14a4fb1c1c402a22e6a341a24c1341b4a3def81b41cd354386dcb795f83897"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:25663d6e99659544ee8fe1b89b1a8c0aaa5e34b103fab124b17fa958c4a324a6"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8b9f19df998761babaa7f09e6bc169294eefafd6149aaa272081cbddc7ba4ca3"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e53d7e6a98b64fe54775d23a7c669763451340c3d44ad5e3a3b48a1efbdc96f"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c3cd1fc1dc7c376c54440aeaaa0dcc803d2126732ff5c6b68ccd619f2e64be4f"}, + {file = "lxml-5.1.0.tar.gz", hash = "sha256:3eea6ed6e6c918e468e693c41ef07f3c3acc310b70ddd9cc72d9ef84bc9564ca"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.7)"] + +[[package]] +name = "markdown" +version = "3.5.2" +description = "Python implementation of John Gruber's Markdown." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Markdown-3.5.2-py3-none-any.whl", hash = "sha256:d43323865d89fc0cb9b20c75fc8ad313af307cc087e84b657d9eec768eddeadd"}, + {file = "Markdown-3.5.2.tar.gz", hash = "sha256:e1ac7b3dc550ee80e602e71c1d168002f062e49f1b11e26a36264dafd4df2ef8"}, +] + +[package.extras] +docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.5)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] +testing = ["coverage", "pyyaml"] + +[[package]] +name = "markdownify" +version = "0.11.6" +description = "Convert HTML to markdown." +optional = false +python-versions = "*" +files = [ + {file = "markdownify-0.11.6-py3-none-any.whl", hash = "sha256:ba35fe289d5e9073bcd7d2cad629278fe25f1a93741fcdc0bfb4f009076d8324"}, + {file = "markdownify-0.11.6.tar.gz", hash = "sha256:009b240e0c9f4c8eaf1d085625dcd4011e12f0f8cec55dedf9ea6f7655e49bfe"}, +] + +[package.dependencies] +beautifulsoup4 = ">=4.9,<5" +six = ">=1.15,<2" + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "marshmallow" +version = "3.20.2" +description = "A lightweight library for converting complex datatypes to and from native Python datatypes." +optional = false +python-versions = ">=3.8" +files = [ + {file = "marshmallow-3.20.2-py3-none-any.whl", hash = "sha256:c21d4b98fee747c130e6bc8f45c4b3199ea66bc00c12ee1f639f0aeca034d5e9"}, + {file = "marshmallow-3.20.2.tar.gz", hash = "sha256:4c1daff273513dc5eb24b219a8035559dc573c8f322558ef85f5438ddd1236dd"}, +] + +[package.dependencies] +packaging = ">=17.0" + +[package.extras] +dev = ["pre-commit (>=2.4,<4.0)", "pytest", "pytz", "simplejson", "tox"] +docs = ["alabaster (==0.7.15)", "autodocsumm (==0.2.12)", "sphinx (==7.2.6)", "sphinx-issues (==3.0.1)", "sphinx-version-warning (==1.1.2)"] +lint = ["pre-commit (>=2.4,<4.0)"] +tests = ["pytest", "pytz", "simplejson"] + +[[package]] +name = "matplotlib-inline" +version = "0.1.6" +description = "Inline Matplotlib backend for Jupyter" +optional = false +python-versions = ">=3.5" +files = [ + {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, + {file = "matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, +] + +[package.dependencies] +traitlets = "*" + +[[package]] +name = "mistune" +version = "3.0.2" +description = "A sane and fast Markdown parser with useful plugins and renderers" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mistune-3.0.2-py3-none-any.whl", hash = "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205"}, + {file = "mistune-3.0.2.tar.gz", hash = "sha256:fc7f93ded930c92394ef2cb6f04a8aabab4117a91449e72dcc8dfa646a508be8"}, +] + +[[package]] +name = "mmh3" +version = "4.1.0" +description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +optional = false +python-versions = "*" +files = [ + {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be5ac76a8b0cd8095784e51e4c1c9c318c19edcd1709a06eb14979c8d850c31a"}, + {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98a49121afdfab67cd80e912b36404139d7deceb6773a83620137aaa0da5714c"}, + {file = "mmh3-4.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5259ac0535874366e7d1a5423ef746e0d36a9e3c14509ce6511614bdc5a7ef5b"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5950827ca0453a2be357696da509ab39646044e3fa15cad364eb65d78797437"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1dd0f652ae99585b9dd26de458e5f08571522f0402155809fd1dc8852a613a39"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d25548070942fab1e4a6f04d1626d67e66d0b81ed6571ecfca511f3edf07e6"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53db8d9bad3cb66c8f35cbc894f336273f63489ce4ac416634932e3cbe79eb5b"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75da0f615eb55295a437264cc0b736753f830b09d102aa4c2a7d719bc445ec05"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b926b07fd678ea84b3a2afc1fa22ce50aeb627839c44382f3d0291e945621e1a"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c5b053334f9b0af8559d6da9dc72cef0a65b325ebb3e630c680012323c950bb6"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bf33dc43cd6de2cb86e0aa73a1cc6530f557854bbbe5d59f41ef6de2e353d7b"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fa7eacd2b830727ba3dd65a365bed8a5c992ecd0c8348cf39a05cc77d22f4970"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:42dfd6742b9e3eec599f85270617debfa0bbb913c545bb980c8a4fa7b2d047da"}, + {file = "mmh3-4.1.0-cp310-cp310-win32.whl", hash = "sha256:2974ad343f0d39dcc88e93ee6afa96cedc35a9883bc067febd7ff736e207fa47"}, + {file = "mmh3-4.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:74699a8984ded645c1a24d6078351a056f5a5f1fe5838870412a68ac5e28d865"}, + {file = "mmh3-4.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:f0dc874cedc23d46fc488a987faa6ad08ffa79e44fb08e3cd4d4cf2877c00a00"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3280a463855b0eae64b681cd5b9ddd9464b73f81151e87bb7c91a811d25619e6"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:97ac57c6c3301769e757d444fa7c973ceb002cb66534b39cbab5e38de61cd896"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7b6502cdb4dbd880244818ab363c8770a48cdccecf6d729ade0241b736b5ec0"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52ba2da04671a9621580ddabf72f06f0e72c1c9c3b7b608849b58b11080d8f14"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a5fef4c4ecc782e6e43fbeab09cff1bac82c998a1773d3a5ee6a3605cde343e"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5135358a7e00991f73b88cdc8eda5203bf9de22120d10a834c5761dbeb07dd13"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cff9ae76a54f7c6fe0167c9c4028c12c1f6de52d68a31d11b6790bb2ae685560"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f02576a4d106d7830ca90278868bf0983554dd69183b7bbe09f2fcd51cf54f"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:073d57425a23721730d3ff5485e2da489dd3c90b04e86243dd7211f889898106"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:71e32ddec7f573a1a0feb8d2cf2af474c50ec21e7a8263026e8d3b4b629805db"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7cbb20b29d57e76a58b40fd8b13a9130db495a12d678d651b459bf61c0714cea"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a42ad267e131d7847076bb7e31050f6c4378cd38e8f1bf7a0edd32f30224d5c9"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a013979fc9390abadc445ea2527426a0e7a4495c19b74589204f9b71bcaafeb"}, + {file = "mmh3-4.1.0-cp311-cp311-win32.whl", hash = "sha256:1d3b1cdad7c71b7b88966301789a478af142bddcb3a2bee563f7a7d40519a00f"}, + {file = "mmh3-4.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0dc6dc32eb03727467da8e17deffe004fbb65e8b5ee2b502d36250d7a3f4e2ec"}, + {file = "mmh3-4.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:9ae3a5c1b32dda121c7dc26f9597ef7b01b4c56a98319a7fe86c35b8bc459ae6"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0033d60c7939168ef65ddc396611077a7268bde024f2c23bdc283a19123f9e9c"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d6af3e2287644b2b08b5924ed3a88c97b87b44ad08e79ca9f93d3470a54a41c5"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d82eb4defa245e02bb0b0dc4f1e7ee284f8d212633389c91f7fba99ba993f0a2"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba245e94b8d54765e14c2d7b6214e832557e7856d5183bc522e17884cab2f45d"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb04e2feeabaad6231e89cd43b3d01a4403579aa792c9ab6fdeef45cc58d4ec0"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e3b1a27def545ce11e36158ba5d5390cdbc300cfe456a942cc89d649cf7e3b2"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce0ab79ff736d7044e5e9b3bfe73958a55f79a4ae672e6213e92492ad5e734d5"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b02268be6e0a8eeb8a924d7db85f28e47344f35c438c1e149878bb1c47b1cd3"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:deb887f5fcdaf57cf646b1e062d56b06ef2f23421c80885fce18b37143cba828"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99dd564e9e2b512eb117bd0cbf0f79a50c45d961c2a02402787d581cec5448d5"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:08373082dfaa38fe97aa78753d1efd21a1969e51079056ff552e687764eafdfe"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:54b9c6a2ea571b714e4fe28d3e4e2db37abfd03c787a58074ea21ee9a8fd1740"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a7b1edf24c69e3513f879722b97ca85e52f9032f24a52284746877f6a7304086"}, + {file = "mmh3-4.1.0-cp312-cp312-win32.whl", hash = "sha256:411da64b951f635e1e2284b71d81a5a83580cea24994b328f8910d40bed67276"}, + {file = "mmh3-4.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bebc3ecb6ba18292e3d40c8712482b4477abd6981c2ebf0e60869bd90f8ac3a9"}, + {file = "mmh3-4.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:168473dd608ade6a8d2ba069600b35199a9af837d96177d3088ca91f2b3798e3"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:372f4b7e1dcde175507640679a2a8790185bb71f3640fc28a4690f73da986a3b"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:438584b97f6fe13e944faf590c90fc127682b57ae969f73334040d9fa1c7ffa5"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6e27931b232fc676675fac8641c6ec6b596daa64d82170e8597f5a5b8bdcd3b6"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:571a92bad859d7b0330e47cfd1850b76c39b615a8d8e7aa5853c1f971fd0c4b1"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a69d6afe3190fa08f9e3a58e5145549f71f1f3fff27bd0800313426929c7068"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afb127be0be946b7630220908dbea0cee0d9d3c583fa9114a07156f98566dc28"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940d86522f36348ef1a494cbf7248ab3f4a1638b84b59e6c9e90408bd11ad729"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3dcccc4935686619a8e3d1f7b6e97e3bd89a4a796247930ee97d35ea1a39341"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01bb9b90d61854dfc2407c5e5192bfb47222d74f29d140cb2dd2a69f2353f7cc"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bcb1b8b951a2c0b0fb8a5426c62a22557e2ffc52539e0a7cc46eb667b5d606a9"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6477a05d5e5ab3168e82e8b106e316210ac954134f46ec529356607900aea82a"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:da5892287e5bea6977364b15712a2573c16d134bc5fdcdd4cf460006cf849278"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:99180d7fd2327a6fffbaff270f760576839dc6ee66d045fa3a450f3490fda7f5"}, + {file = "mmh3-4.1.0-cp38-cp38-win32.whl", hash = "sha256:9b0d4f3949913a9f9a8fb1bb4cc6ecd52879730aab5ff8c5a3d8f5b593594b73"}, + {file = "mmh3-4.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:598c352da1d945108aee0c3c3cfdd0e9b3edef74108f53b49d481d3990402169"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:475d6d1445dd080f18f0f766277e1237fa2914e5fe3307a3b2a3044f30892103"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5ca07c41e6a2880991431ac717c2a049056fff497651a76e26fc22224e8b5732"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ebe052fef4bbe30c0548d12ee46d09f1b69035ca5208a7075e55adfe091be44"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaefd42e85afb70f2b855a011f7b4d8a3c7e19c3f2681fa13118e4d8627378c5"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0ae43caae5a47afe1b63a1ae3f0986dde54b5fb2d6c29786adbfb8edc9edfb"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6218666f74c8c013c221e7f5f8a693ac9cf68e5ac9a03f2373b32d77c48904de"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac59294a536ba447b5037f62d8367d7d93b696f80671c2c45645fa9f1109413c"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:086844830fcd1e5c84fec7017ea1ee8491487cfc877847d96f86f68881569d2e"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e42b38fad664f56f77f6fbca22d08450f2464baa68acdbf24841bf900eb98e87"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d08b790a63a9a1cde3b5d7d733ed97d4eb884bfbc92f075a091652d6bfd7709a"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:73ea4cc55e8aea28c86799ecacebca09e5f86500414870a8abaedfcbaf74d288"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f90938ff137130e47bcec8dc1f4ceb02f10178c766e2ef58a9f657ff1f62d124"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:aa1f13e94b8631c8cd53259250556edcf1de71738936b60febba95750d9632bd"}, + {file = "mmh3-4.1.0-cp39-cp39-win32.whl", hash = "sha256:a3b680b471c181490cf82da2142029edb4298e1bdfcb67c76922dedef789868d"}, + {file = "mmh3-4.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:fefef92e9c544a8dbc08f77a8d1b6d48006a750c4375bbcd5ff8199d761e263b"}, + {file = "mmh3-4.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:8e2c1f6a2b41723a4f82bd5a762a777836d29d664fc0095f17910bea0adfd4a6"}, + {file = "mmh3-4.1.0.tar.gz", hash = "sha256:a1cf25348b9acd229dda464a094d6170f47d2850a1fcb762a3b6172d2ce6ca4a"}, +] + +[package.extras] +test = ["mypy (>=1.0)", "pytest (>=7.0.0)"] + +[[package]] +name = "monotonic" +version = "1.6" +description = "An implementation of time.monotonic() for Python 2 & < 3.3" +optional = false +python-versions = "*" +files = [ + {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"}, + {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"}, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +optional = false +python-versions = "*" +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4)"] +tests = ["pytest (>=4.6)"] + +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "nbclient" +version = "0.9.0" +description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "nbclient-0.9.0-py3-none-any.whl", hash = "sha256:a3a1ddfb34d4a9d17fc744d655962714a866639acd30130e9be84191cd97cd15"}, + {file = "nbclient-0.9.0.tar.gz", hash = "sha256:4b28c207877cf33ef3a9838cdc7a54c5ceff981194a82eac59d558f05487295e"}, +] + +[package.dependencies] +jupyter-client = ">=6.1.12" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +nbformat = ">=5.1" +traitlets = ">=5.4" + +[package.extras] +dev = ["pre-commit"] +docs = ["autodoc-traits", "mock", "moto", "myst-parser", "nbclient[test]", "sphinx (>=1.7)", "sphinx-book-theme", "sphinxcontrib-spelling"] +test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>=7.0.0)", "pytest (>=7.0)", "pytest-asyncio", "pytest-cov (>=4.0)", "testpath", "xmltodict"] + +[[package]] +name = "nbconvert" +version = "7.15.0" +description = "Converting Jupyter Notebooks" +optional = false +python-versions = ">=3.8" +files = [ + {file = "nbconvert-7.15.0-py3-none-any.whl", hash = "sha256:0efd3ca74fd1525560e0312cec235e57dfbf3c5c775c7e61e04c532b28f8da6f"}, + {file = "nbconvert-7.15.0.tar.gz", hash = "sha256:ff3f54a1a5e1e024beb9fde8946d05b6d0bf68cd14b5f2f9dc5b545c8bc71055"}, +] + +[package.dependencies] +beautifulsoup4 = "*" +bleach = "!=5.0.0" +defusedxml = "*" +jinja2 = ">=3.0" +jupyter-core = ">=4.7" +jupyterlab-pygments = "*" +markupsafe = ">=2.0" +mistune = ">=2.0.3,<4" +nbclient = ">=0.5.0" +nbformat = ">=5.7" +packaging = "*" +pandocfilters = ">=1.4.1" +pygments = ">=2.4.1" +tinycss2 = "*" +traitlets = ">=5.1" + +[package.extras] +all = ["nbconvert[docs,qtpdf,serve,test,webpdf]"] +docs = ["ipykernel", "ipython", "myst-parser", "nbsphinx (>=0.2.12)", "pydata-sphinx-theme", "sphinx (==5.0.2)", "sphinxcontrib-spelling"] +qtpdf = ["nbconvert[qtpng]"] +qtpng = ["pyqtwebengine (>=5.15)"] +serve = ["tornado (>=6.1)"] +test = ["flaky", "ipykernel", "ipywidgets (>=7.5)", "pytest"] +webpdf = ["playwright"] + +[[package]] +name = "nbformat" +version = "5.9.2" +description = "The Jupyter Notebook format" +optional = false +python-versions = ">=3.8" +files = [ + {file = "nbformat-5.9.2-py3-none-any.whl", hash = "sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9"}, + {file = "nbformat-5.9.2.tar.gz", hash = "sha256:5f98b5ba1997dff175e77e0c17d5c10a96eaed2cbd1de3533d1fc35d5e111192"}, +] + +[package.dependencies] +fastjsonschema = "*" +jsonschema = ">=2.6" +jupyter-core = "*" +traitlets = ">=5.1" + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] +test = ["pep440", "pre-commit", "pytest", "testpath"] + +[[package]] +name = "nest-asyncio" +version = "1.6.0" +description = "Patch asyncio to allow nested event loops" +optional = false +python-versions = ">=3.5" +files = [ + {file = "nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c"}, + {file = "nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe"}, +] + +[[package]] +name = "nltk" +version = "3.8.1" +description = "Natural Language Toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "nltk-3.8.1-py3-none-any.whl", hash = "sha256:fd5c9109f976fa86bcadba8f91e47f5e9293bd034474752e92a520f81c93dda5"}, + {file = "nltk-3.8.1.zip", hash = "sha256:1834da3d0682cba4f2cede2f9aad6b0fafb6461ba451db0efb6f9c39798d64d3"}, +] + +[package.dependencies] +click = "*" +joblib = "*" +regex = ">=2021.8.3" +tqdm = "*" + +[package.extras] +all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] +corenlp = ["requests"] +machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] +plot = ["matplotlib"] +tgrep = ["pyparsing"] +twitter = ["twython"] + +[[package]] +name = "notebook" +version = "7.0.7" +description = "Jupyter Notebook - A web-based notebook environment for interactive computing" +optional = false +python-versions = ">=3.8" +files = [ + {file = "notebook-7.0.7-py3-none-any.whl", hash = "sha256:289b606d7e173f75a18beb1406ef411b43f97f7a9c55ba03efa3622905a62346"}, + {file = "notebook-7.0.7.tar.gz", hash = "sha256:3bcff00c17b3ac142ef5f436d50637d936b274cfa0b41f6ac0175363de9b4e09"}, +] + +[package.dependencies] +jupyter-server = ">=2.4.0,<3" +jupyterlab = ">=4.0.2,<5" +jupyterlab-server = ">=2.22.1,<3" +notebook-shim = ">=0.2,<0.3" +tornado = ">=6.2.0" + +[package.extras] +dev = ["hatch", "pre-commit"] +docs = ["myst-parser", "nbsphinx", "pydata-sphinx-theme", "sphinx (>=1.3.6)", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] +test = ["importlib-resources (>=5.0)", "ipykernel", "jupyter-server[test] (>=2.4.0,<3)", "jupyterlab-server[test] (>=2.22.1,<3)", "nbval", "pytest (>=7.0)", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", "requests"] + +[[package]] +name = "notebook-shim" +version = "0.2.3" +description = "A shim layer for notebook traits and config" +optional = false +python-versions = ">=3.7" +files = [ + {file = "notebook_shim-0.2.3-py3-none-any.whl", hash = "sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7"}, + {file = "notebook_shim-0.2.3.tar.gz", hash = "sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9"}, +] + +[package.dependencies] +jupyter-server = ">=1.8,<3" + +[package.extras] +test = ["pytest", "pytest-console-scripts", "pytest-jupyter", "pytest-tornasync"] + +[[package]] +name = "numba" +version = "0.59.0" +description = "compiling Python code using LLVM" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numba-0.59.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8d061d800473fb8fef76a455221f4ad649a53f5e0f96e3f6c8b8553ee6fa98fa"}, + {file = "numba-0.59.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c086a434e7d3891ce5dfd3d1e7ee8102ac1e733962098578b507864120559ceb"}, + {file = "numba-0.59.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9e20736bf62e61f8353fb71b0d3a1efba636c7a303d511600fc57648b55823ed"}, + {file = "numba-0.59.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e86e6786aec31d2002122199486e10bbc0dc40f78d76364cded375912b13614c"}, + {file = "numba-0.59.0-cp310-cp310-win_amd64.whl", hash = "sha256:0307ee91b24500bb7e64d8a109848baf3a3905df48ce142b8ac60aaa406a0400"}, + {file = "numba-0.59.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d540f69a8245fb714419c2209e9af6104e568eb97623adc8943642e61f5d6d8e"}, + {file = "numba-0.59.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1192d6b2906bf3ff72b1d97458724d98860ab86a91abdd4cfd9328432b661e31"}, + {file = "numba-0.59.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90efb436d3413809fcd15298c6d395cb7d98184350472588356ccf19db9e37c8"}, + {file = "numba-0.59.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cd3dac45e25d927dcb65d44fb3a973994f5add2b15add13337844afe669dd1ba"}, + {file = "numba-0.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:753dc601a159861808cc3207bad5c17724d3b69552fd22768fddbf302a817a4c"}, + {file = "numba-0.59.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ce62bc0e6dd5264e7ff7f34f41786889fa81a6b860662f824aa7532537a7bee0"}, + {file = "numba-0.59.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8cbef55b73741b5eea2dbaf1b0590b14977ca95a13a07d200b794f8f6833a01c"}, + {file = "numba-0.59.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:70d26ba589f764be45ea8c272caa467dbe882b9676f6749fe6f42678091f5f21"}, + {file = "numba-0.59.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e125f7d69968118c28ec0eed9fbedd75440e64214b8d2eac033c22c04db48492"}, + {file = "numba-0.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:4981659220b61a03c1e557654027d271f56f3087448967a55c79a0e5f926de62"}, + {file = "numba-0.59.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fe4d7562d1eed754a7511ed7ba962067f198f86909741c5c6e18c4f1819b1f47"}, + {file = "numba-0.59.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6feb1504bb432280f900deaf4b1dadcee68812209500ed3f81c375cbceab24dc"}, + {file = "numba-0.59.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:944faad25ee23ea9dda582bfb0189fb9f4fc232359a80ab2a028b94c14ce2b1d"}, + {file = "numba-0.59.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5516a469514bfae52a9d7989db4940653a5cbfac106f44cb9c50133b7ad6224b"}, + {file = "numba-0.59.0-cp39-cp39-win_amd64.whl", hash = "sha256:32bd0a41525ec0b1b853da244808f4e5333867df3c43c30c33f89cf20b9c2b63"}, + {file = "numba-0.59.0.tar.gz", hash = "sha256:12b9b064a3e4ad00e2371fc5212ef0396c80f41caec9b5ec391c8b04b6eaf2a8"}, +] + +[package.dependencies] +llvmlite = "==0.42.*" +numpy = ">=1.22,<1.27" + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "objgraph" +version = "3.6.0" +description = "Draws Python object reference graphs with graphviz" +optional = false +python-versions = ">=3.7" +files = [ + {file = "objgraph-3.6.0-py2.py3-none-any.whl", hash = "sha256:7c4aa57754c41bdd4ba67a8edfb44e45e248a1474444d0b9adca3cfd2717c485"}, + {file = "objgraph-3.6.0.tar.gz", hash = "sha256:369567c37b4f2f928160b6f6ededcbea8fc7e929831877fd1056c78a900c17d3"}, +] + +[package.extras] +ipython = ["graphviz"] + +[[package]] +name = "onnxruntime" +version = "1.17.0" +description = "ONNX Runtime is a runtime accelerator for Machine Learning models" +optional = false +python-versions = "*" +files = [ + {file = "onnxruntime-1.17.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:d2b22a25a94109cc983443116da8d9805ced0256eb215c5e6bc6dcbabefeab96"}, + {file = "onnxruntime-1.17.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4c87d83c6f58d1af2675fc99e3dc810f2dbdb844bcefd0c1b7573632661f6fc"}, + {file = "onnxruntime-1.17.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dba55723bf9b835e358f48c98a814b41692c393eb11f51e02ece0625c756b797"}, + {file = "onnxruntime-1.17.0-cp310-cp310-win32.whl", hash = "sha256:ee48422349cc500273beea7607e33c2237909f58468ae1d6cccfc4aecd158565"}, + {file = "onnxruntime-1.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:f34cc46553359293854e38bdae2ab1be59543aad78a6317e7746d30e311110c3"}, + {file = "onnxruntime-1.17.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:16d26badd092c8c257fa57c458bb600d96dc15282c647ccad0ed7b2732e6c03b"}, + {file = "onnxruntime-1.17.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6f1273bebcdb47ed932d076c85eb9488bc4768fcea16d5f2747ca692fad4f9d3"}, + {file = "onnxruntime-1.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cb60fd3c2c1acd684752eb9680e89ae223e9801a9b0e0dc7b28adabe45a2e380"}, + {file = "onnxruntime-1.17.0-cp311-cp311-win32.whl", hash = "sha256:4b038324586bc905299e435f7c00007e6242389c856b82fe9357fdc3b1ef2bdc"}, + {file = "onnxruntime-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:93d39b3fa1ee01f034f098e1c7769a811a21365b4883f05f96c14a2b60c6028b"}, + {file = "onnxruntime-1.17.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:90c0890e36f880281c6c698d9bc3de2afbeee2f76512725ec043665c25c67d21"}, + {file = "onnxruntime-1.17.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7466724e809a40e986b1637cba156ad9fc0d1952468bc00f79ef340bc0199552"}, + {file = "onnxruntime-1.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d47bee7557a8b99c8681b6882657a515a4199778d6d5e24e924d2aafcef55b0a"}, + {file = "onnxruntime-1.17.0-cp312-cp312-win32.whl", hash = "sha256:bb1bf1ee575c665b8bbc3813ab906e091a645a24ccc210be7932154b8260eca1"}, + {file = "onnxruntime-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:ac2f286da3494b29b4186ca193c7d4e6a2c1f770c4184c7192c5da142c3dec28"}, + {file = "onnxruntime-1.17.0-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1ec485643b93e0a3896c655eb2426decd63e18a278bb7ccebc133b340723624f"}, + {file = "onnxruntime-1.17.0-cp38-cp38-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83c35809cda898c5a11911c69ceac8a2ac3925911854c526f73bad884582f911"}, + {file = "onnxruntime-1.17.0-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fa464aa4d81df818375239e481887b656e261377d5b6b9a4692466f5f3261edc"}, + {file = "onnxruntime-1.17.0-cp38-cp38-win32.whl", hash = "sha256:b7b337cd0586f7836601623cbd30a443df9528ef23965860d11c753ceeb009f2"}, + {file = "onnxruntime-1.17.0-cp38-cp38-win_amd64.whl", hash = "sha256:fbb9faaf51d01aa2c147ef52524d9326744c852116d8005b9041809a71838878"}, + {file = "onnxruntime-1.17.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:5a06ab84eaa350bf64b1d747b33ccf10da64221ed1f38f7287f15eccbec81603"}, + {file = "onnxruntime-1.17.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d3d11db2c8242766212a68d0b139745157da7ce53bd96ba349a5c65e5a02357"}, + {file = "onnxruntime-1.17.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5632077c3ab8b0cd4f74b0af9c4e924be012b1a7bcd7daa845763c6c6bf14b7d"}, + {file = "onnxruntime-1.17.0-cp39-cp39-win32.whl", hash = "sha256:61a12732cba869b3ad2d4e29ab6cb62c7a96f61b8c213f7fcb961ba412b70b37"}, + {file = "onnxruntime-1.17.0-cp39-cp39-win_amd64.whl", hash = "sha256:461fa0fc7d9c392c352b6cccdedf44d818430f3d6eacd924bb804fdea2dcfd02"}, +] + +[package.dependencies] +coloredlogs = "*" +flatbuffers = "*" +numpy = ">=1.21.6" +packaging = "*" +protobuf = "*" +sympy = "*" + +[[package]] +name = "openai" +version = "1.11.1" +description = "The official Python library for the openai API" +optional = false +python-versions = ">=3.7.1" +files = [ + {file = "openai-1.11.1-py3-none-any.whl", hash = "sha256:e0f388ce499f53f58079d0c1f571f356f2b168b84d0d24a412506b6abc714980"}, + {file = "openai-1.11.1.tar.gz", hash = "sha256:f66b8fe431af43e09594147ef3cdcb79758285de72ebafd52be9700a2af41e99"}, +] + +[package.dependencies] +anyio = ">=3.5.0,<5" +distro = ">=1.7.0,<2" +httpx = ">=0.23.0,<1" +pydantic = ">=1.9.0,<3" +sniffio = "*" +tqdm = ">4" +typing-extensions = ">=4.7,<5" + +[package.extras] +datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] + +[[package]] +name = "opentelemetry-api" +version = "1.22.0" +description = "OpenTelemetry Python API" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_api-1.22.0-py3-none-any.whl", hash = "sha256:43621514301a7e9f5d06dd8013a1b450f30c2e9372b8e30aaeb4562abf2ce034"}, + {file = "opentelemetry_api-1.22.0.tar.gz", hash = "sha256:15ae4ca925ecf9cfdfb7a709250846fbb08072260fca08ade78056c502b86bed"}, +] + +[package.dependencies] +deprecated = ">=1.2.6" +importlib-metadata = ">=6.0,<7.0" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.22.0" +description = "OpenTelemetry Protobuf encoding" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_exporter_otlp_proto_common-1.22.0-py3-none-any.whl", hash = "sha256:3f2538bec5312587f8676c332b3747f54c89fe6364803a807e217af4603201fa"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.22.0.tar.gz", hash = "sha256:71ae2f81bc6d6fe408d06388826edc8933759b2ca3a97d24054507dc7cfce52d"}, +] + +[package.dependencies] +backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} +opentelemetry-proto = "1.22.0" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.22.0" +description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_exporter_otlp_proto_grpc-1.22.0-py3-none-any.whl", hash = "sha256:b5bcadc129272004316a455e9081216d3380c1fc2231a928ea6a70aa90e173fb"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.22.0.tar.gz", hash = "sha256:1e0e5aa4bbabc74942f06f268deffd94851d12a8dc30b02527472ef1729fe5b1"}, +] + +[package.dependencies] +backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} +deprecated = ">=1.2.6" +googleapis-common-protos = ">=1.52,<2.0" +grpcio = ">=1.0.0,<2.0.0" +opentelemetry-api = ">=1.15,<2.0" +opentelemetry-exporter-otlp-proto-common = "1.22.0" +opentelemetry-proto = "1.22.0" +opentelemetry-sdk = ">=1.22.0,<1.23.0" + +[package.extras] +test = ["pytest-grpc"] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.43b0" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_instrumentation-0.43b0-py3-none-any.whl", hash = "sha256:0ff1334d7e359e27640e9d420024efeb73eacae464309c2e14ede7ba6c93967e"}, + {file = "opentelemetry_instrumentation-0.43b0.tar.gz", hash = "sha256:c3755da6c4be8033be0216d0501e11f4832690f4e2eca5a3576fbf113498f0f6"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +setuptools = ">=16.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-asgi" +version = "0.43b0" +description = "ASGI instrumentation for OpenTelemetry" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_instrumentation_asgi-0.43b0-py3-none-any.whl", hash = "sha256:1f593829fa039e9367820736fb063e92acd15c25b53d7bcb5d319971b8e93fd7"}, + {file = "opentelemetry_instrumentation_asgi-0.43b0.tar.gz", hash = "sha256:3f6f19333dca31ef696672e4e36cb1c2613c71dc7e847c11ff36a37e1130dadc"}, +] + +[package.dependencies] +asgiref = ">=3.0,<4.0" +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.43b0" +opentelemetry-semantic-conventions = "0.43b0" +opentelemetry-util-http = "0.43b0" + +[package.extras] +instruments = ["asgiref (>=3.0,<4.0)"] +test = ["opentelemetry-instrumentation-asgi[instruments]", "opentelemetry-test-utils (==0.43b0)"] + +[[package]] +name = "opentelemetry-instrumentation-fastapi" +version = "0.43b0" +description = "OpenTelemetry FastAPI Instrumentation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_instrumentation_fastapi-0.43b0-py3-none-any.whl", hash = "sha256:b79c044df68a52e07b35fa12a424e7cc0dd27ff0a171c5fdcc41dea9de8fc938"}, + {file = "opentelemetry_instrumentation_fastapi-0.43b0.tar.gz", hash = "sha256:2afaaf470622e1a2732182c68f6d2431ffe5e026a7edacd0f83605632b66347f"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.43b0" +opentelemetry-instrumentation-asgi = "0.43b0" +opentelemetry-semantic-conventions = "0.43b0" +opentelemetry-util-http = "0.43b0" + +[package.extras] +instruments = ["fastapi (>=0.58,<1.0)"] +test = ["httpx (>=0.22,<1.0)", "opentelemetry-instrumentation-fastapi[instruments]", "opentelemetry-test-utils (==0.43b0)", "requests (>=2.23,<3.0)"] + +[[package]] +name = "opentelemetry-proto" +version = "1.22.0" +description = "OpenTelemetry Python Proto" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_proto-1.22.0-py3-none-any.whl", hash = "sha256:ce7188d22c75b6d0fe53e7fb58501613d0feade5139538e79dedd9420610fa0c"}, + {file = "opentelemetry_proto-1.22.0.tar.gz", hash = "sha256:9ec29169286029f17ca34ec1f3455802ffb90131642d2f545ece9a63e8f69003"}, +] + +[package.dependencies] +protobuf = ">=3.19,<5.0" + +[[package]] +name = "opentelemetry-sdk" +version = "1.22.0" +description = "OpenTelemetry Python SDK" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_sdk-1.22.0-py3-none-any.whl", hash = "sha256:a730555713d7c8931657612a88a141e3a4fe6eb5523d9e2d5a8b1e673d76efa6"}, + {file = "opentelemetry_sdk-1.22.0.tar.gz", hash = "sha256:45267ac1f38a431fc2eb5d6e0c0d83afc0b78de57ac345488aa58c28c17991d0"}, +] + +[package.dependencies] +opentelemetry-api = "1.22.0" +opentelemetry-semantic-conventions = "0.43b0" +typing-extensions = ">=3.7.4" + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.43b0" +description = "OpenTelemetry Semantic Conventions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_semantic_conventions-0.43b0-py3-none-any.whl", hash = "sha256:291284d7c1bf15fdaddf309b3bd6d3b7ce12a253cec6d27144439819a15d8445"}, + {file = "opentelemetry_semantic_conventions-0.43b0.tar.gz", hash = "sha256:b9576fb890df479626fa624e88dde42d3d60b8b6c8ae1152ad157a8b97358635"}, +] + +[[package]] +name = "opentelemetry-util-http" +version = "0.43b0" +description = "Web util for OpenTelemetry" +optional = false +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_util_http-0.43b0-py3-none-any.whl", hash = "sha256:f25a820784b030f6cb86b3d76e5676c769b75ed3f55a210bcdae0a5e175ebadb"}, + {file = "opentelemetry_util_http-0.43b0.tar.gz", hash = "sha256:3ff6ab361dbe99fc81200d625603c0fb890c055c6e416a3e6d661ddf47a6c7f7"}, +] + +[[package]] +name = "overrides" +version = "7.7.0" +description = "A decorator to automatically detect mismatch when overriding a method." +optional = false +python-versions = ">=3.6" +files = [ + {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, + {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.2.0" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8108ee1712bb4fa2c16981fba7e68b3f6ea330277f5ca34fa8d557e986a11670"}, + {file = "pandas-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:736da9ad4033aeab51d067fc3bd69a0ba36f5a60f66a527b3d72e2030e63280a"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e0b4fc3ddceb56ec8a287313bc22abe17ab0eb184069f08fc6a9352a769b18"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20404d2adefe92aed3b38da41d0847a143a09be982a31b85bc7dd565bdba0f4e"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ea3ee3f125032bfcade3a4cf85131ed064b4f8dd23e5ce6fa16473e48ebcaf5"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9670b3ac00a387620489dfc1bca66db47a787f4e55911f1293063a78b108df1"}, + {file = "pandas-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:5a946f210383c7e6d16312d30b238fd508d80d927014f3b33fb5b15c2f895430"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a1b438fa26b208005c997e78672f1aa8138f67002e833312e6230f3e57fa87d5"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8ce2fbc8d9bf303ce54a476116165220a1fedf15985b09656b4b4275300e920b"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2707514a7bec41a4ab81f2ccce8b382961a29fbe9492eab1305bb075b2b1ff4f"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85793cbdc2d5bc32620dc8ffa715423f0c680dacacf55056ba13454a5be5de88"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cfd6c2491dc821b10c716ad6776e7ab311f7df5d16038d0b7458bc0b67dc10f3"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a146b9dcacc3123aa2b399df1a284de5f46287a4ab4fbfc237eac98a92ebcb71"}, + {file = "pandas-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbc1b53c0e1fdf16388c33c3cca160f798d38aea2978004dd3f4d3dec56454c9"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a41d06f308a024981dcaa6c41f2f2be46a6b186b902c94c2674e8cb5c42985bc"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:159205c99d7a5ce89ecfc37cb08ed179de7783737cea403b295b5eda8e9c56d1"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1e1f3861ea9132b32f2133788f3b14911b68102d562715d71bd0013bc45440"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:761cb99b42a69005dec2b08854fb1d4888fdf7b05db23a8c5a099e4b886a2106"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a20628faaf444da122b2a64b1e5360cde100ee6283ae8effa0d8745153809a2e"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f5be5d03ea2073627e7111f61b9f1f0d9625dc3c4d8dda72cc827b0c58a1d042"}, + {file = "pandas-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:a626795722d893ed6aacb64d2401d017ddc8a2341b49e0384ab9bf7112bdec30"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9f66419d4a41132eb7e9a73dcec9486cf5019f52d90dd35547af11bc58f8637d"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:57abcaeda83fb80d447f28ab0cc7b32b13978f6f733875ebd1ed14f8fbc0f4ab"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60f1f7dba3c2d5ca159e18c46a34e7ca7247a73b5dd1a22b6d59707ed6b899a"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb61dc8567b798b969bcc1fc964788f5a68214d333cade8319c7ab33e2b5d88a"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:52826b5f4ed658fa2b729264d63f6732b8b29949c7fd234510d57c61dbeadfcd"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bde2bc699dbd80d7bc7f9cab1e23a95c4375de615860ca089f34e7c64f4a8de7"}, + {file = "pandas-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:3de918a754bbf2da2381e8a3dcc45eede8cd7775b047b923f9006d5f876802ae"}, + {file = "pandas-2.2.0.tar.gz", hash = "sha256:30b83f7c3eb217fb4d1b494a57a2fda5444f17834f5df2de6b2ffff68dc3c8e2"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pandocfilters" +version = "1.5.1" +description = "Utilities for writing pandoc filters in python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pandocfilters-1.5.1-py2.py3-none-any.whl", hash = "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc"}, + {file = "pandocfilters-1.5.1.tar.gz", hash = "sha256:002b4a555ee4ebc03f8b66307e287fa492e4a77b4ea14d3f934328297bb4939e"}, +] + +[[package]] +name = "parso" +version = "0.8.3" +description = "A Python Parser" +optional = false +python-versions = ">=3.6" +files = [ + {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, + {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, +] + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["docopt", "pytest (<6.0.0)"] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +description = "Pexpect allows easy control of interactive console applications." +optional = false +python-versions = "*" +files = [ + {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, + {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, +] + +[package.dependencies] +ptyprocess = ">=0.5" + +[[package]] +name = "pillow" +version = "10.2.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pillow-10.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0"}, + {file = "pillow-10.2.0-cp310-cp310-win32.whl", hash = "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023"}, + {file = "pillow-10.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72"}, + {file = "pillow-10.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757"}, + {file = "pillow-10.2.0-cp311-cp311-win32.whl", hash = "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068"}, + {file = "pillow-10.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56"}, + {file = "pillow-10.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb"}, + {file = "pillow-10.2.0-cp312-cp312-win32.whl", hash = "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f"}, + {file = "pillow-10.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9"}, + {file = "pillow-10.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe"}, + {file = "pillow-10.2.0-cp38-cp38-win32.whl", hash = "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e"}, + {file = "pillow-10.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591"}, + {file = "pillow-10.2.0-cp39-cp39-win32.whl", hash = "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516"}, + {file = "pillow-10.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8"}, + {file = "pillow-10.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6"}, + {file = "pillow-10.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"}, + {file = "pillow-10.2.0.tar.gz", hash = "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "posthog" +version = "3.4.0" +description = "Integrate PostHog into any python application." +optional = false +python-versions = "*" +files = [ + {file = "posthog-3.4.0-py2.py3-none-any.whl", hash = "sha256:922bdc1721a18f7275363130042b4be9bb49c88b3e9d1b30b0b3d33a664b3f10"}, + {file = "posthog-3.4.0.tar.gz", hash = "sha256:0b9fd6861353e67e289fad26fce72a78a1ec09abd9a3c725b9ab7e1edee3da37"}, +] + +[package.dependencies] +backoff = ">=1.10.0" +monotonic = ">=1.5" +python-dateutil = ">2.1" +requests = ">=2.7,<3.0" +six = ">=1.5" + +[package.extras] +dev = ["black", "flake8", "flake8-print", "isort", "pre-commit"] +sentry = ["django", "sentry-sdk"] +test = ["coverage", "flake8", "freezegun (==0.3.15)", "mock (>=2.0.0)", "pylint", "pytest", "pytest-timeout"] + +[[package]] +name = "prometheus-client" +version = "0.19.0" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.8" +files = [ + {file = "prometheus_client-0.19.0-py3-none-any.whl", hash = "sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92"}, + {file = "prometheus_client-0.19.0.tar.gz", hash = "sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1"}, +] + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "prompt-toolkit" +version = "3.0.43" +description = "Library for building powerful interactive command lines in Python" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, + {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, +] + +[package.dependencies] +wcwidth = "*" + +[[package]] +name = "protobuf" +version = "4.25.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, +] + +[[package]] +name = "psutil" +version = "5.9.8" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d"}, + {file = "psutil-5.9.8-cp27-none-win32.whl", hash = "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e"}, + {file = "psutil-5.9.8-cp27-none-win_amd64.whl", hash = "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631"}, + {file = "psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"}, + {file = "psutil-5.9.8-cp36-cp36m-win32.whl", hash = "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee"}, + {file = "psutil-5.9.8-cp36-cp36m-win_amd64.whl", hash = "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2"}, + {file = "psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"}, + {file = "psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"}, + {file = "psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"}, + {file = "psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"}, +] + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +optional = false +python-versions = "*" +files = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] + +[[package]] +name = "pulsar-client" +version = "3.4.0" +description = "Apache Pulsar Python client library" +optional = false +python-versions = "*" +files = [ + {file = "pulsar_client-3.4.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:ebf99db5244ff69479283b25621b070492acc4bb643d162d86b90387cb6fdb2a"}, + {file = "pulsar_client-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6cb5d8e1482a8aea758633be23717e0c4bb7dc53784e37915c0048c0382f134"}, + {file = "pulsar_client-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a7592e42c76034e9a8d64d42dd5bab361425f869de562e9ccad698e19cd88"}, + {file = "pulsar_client-3.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5963090a78a5644ba25f41da3a6d49ea3f00c972b095baff365916dc246426a"}, + {file = "pulsar_client-3.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:419cdcf577f755e3f31bf264300d9ba158325edb2ee9cee555d81ba1909c094e"}, + {file = "pulsar_client-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:4c93c35ee97307dae153e748b33dcd3d4f06da34bca373321aa2df73f1535705"}, + {file = "pulsar_client-3.4.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:11952fb022ee72debf53b169f4482f9dc5c890be0149ae98779864b3a21f1bd3"}, + {file = "pulsar_client-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8743c320aa96798d20cafa98ea97a68c4295fc4872c23acd5e012fd36cb06ba"}, + {file = "pulsar_client-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33571de99cd898349f17978ba62e2b839ea0275fb7067f31bf5f6ebfeae0987d"}, + {file = "pulsar_client-3.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a60c03c3e70f018538e7cd3fa84d95e283b610272b744166dbc48960a809fa07"}, + {file = "pulsar_client-3.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c47041267b5843ffec54352d842156c279945f3e976d7025ffa89875ff76390"}, + {file = "pulsar_client-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:49fe4ab04004b476c87ab3ad22fe87346fca564a3e3ca9c0ac58fee45a895d81"}, + {file = "pulsar_client-3.4.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:1e077a4839be3ead3de3f05b4c244269dca2df07f47cea0b90544c7e9dc1642f"}, + {file = "pulsar_client-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f202b84e1f683d64672dd1971114600ae2e5c3735587286ff9bfb431385f08e8"}, + {file = "pulsar_client-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c606c04f357341042fa6c75477de7d2204f7ae50aa29c2f74b24e54c85f47f96"}, + {file = "pulsar_client-3.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c67b25ede3a578f5a7dc30230e52609ef38191f74b47e5cbdbc98c42df556927"}, + {file = "pulsar_client-3.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b7f8211cc9460cdf4d06e4e1cb878689d2aa4a7e4027bd2a2f1419a79ade16a6"}, + {file = "pulsar_client-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:c5399e9780d6951c69808c0b6175311a966af82fb08addf6e741ae37b1bee7ef"}, + {file = "pulsar_client-3.4.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:a2d6c850b60106dc915d3476a490fba547c6748a5f742b68abd30d1a35355b82"}, + {file = "pulsar_client-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a52ea8294a9f30eb6f0a2db5dc16e3aad7ff2284f818c48ad3a6b601723be02b"}, + {file = "pulsar_client-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eeeede40108be12222e009285c971e5b8f6433d9f0f8ef934d6a131585921c4"}, + {file = "pulsar_client-3.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9409066c600f2b6f220552c5dfe08aeeabcf07fe0e76367aa5816b2e87a5cf72"}, + {file = "pulsar_client-3.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:58e2f886e6dab43e66c3ce990fe96209e55ab46350506829a637b77b74125fb9"}, + {file = "pulsar_client-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:b57dfa5063b0d9dc7664896c55605eac90753e35e80db5a959d3be2be0ab0d48"}, + {file = "pulsar_client-3.4.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:7704c664aa2c801af4c2d3a58e9d8ffaeef12ce8a0f71712e9187f9a96da856f"}, + {file = "pulsar_client-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0364db563e27442053bdbb8655e7ffb420f491690bc2c78da5a58bd35c658ad"}, + {file = "pulsar_client-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3e34de19e0744d8aa3538cb2172076bccd0761b3e94ebadb7bd59765ae3d1ed"}, + {file = "pulsar_client-3.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:dc8be41dec8cb052fb1837550f495e9b73a8b3cf85e07157904ec84832758a65"}, + {file = "pulsar_client-3.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b49d669bed15b7edb9c936704310d57808f1d01c511b94d866f54fe8ffe1752d"}, + {file = "pulsar_client-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:88c93e5fbfc349f3967e931f7a908d15fd4fd725ebdd842423ac9cd961fe293f"}, +] + +[package.dependencies] +certifi = "*" + +[package.extras] +all = ["apache-bookkeeper-client (>=4.16.1)", "fastavro (>=1.9.2)", "grpcio (>=1.60.0)", "prometheus-client", "protobuf (>=3.6.1,<=3.20.3)", "ratelimit"] +avro = ["fastavro (>=1.9.2)"] +functions = ["apache-bookkeeper-client (>=4.16.1)", "grpcio (>=1.60.0)", "prometheus-client", "protobuf (>=3.6.1,<=3.20.3)", "ratelimit"] + +[[package]] +name = "pure-eval" +version = "0.2.2" +description = "Safely evaluate AST nodes without side effects" +optional = false +python-versions = "*" +files = [ + {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, + {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, +] + +[package.extras] +tests = ["pytest"] + +[[package]] +name = "pyarrow" +version = "15.0.0" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyarrow-15.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:0a524532fd6dd482edaa563b686d754c70417c2f72742a8c990b322d4c03a15d"}, + {file = "pyarrow-15.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:60a6bdb314affa9c2e0d5dddf3d9cbb9ef4a8dddaa68669975287d47ece67642"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:66958fd1771a4d4b754cd385835e66a3ef6b12611e001d4e5edfcef5f30391e2"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f500956a49aadd907eaa21d4fff75f73954605eaa41f61cb94fb008cf2e00c6"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6f87d9c4f09e049c2cade559643424da84c43a35068f2a1c4653dc5b1408a929"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:85239b9f93278e130d86c0e6bb455dcb66fc3fd891398b9d45ace8799a871a1e"}, + {file = "pyarrow-15.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b8d43e31ca16aa6e12402fcb1e14352d0d809de70edd185c7650fe80e0769e3"}, + {file = "pyarrow-15.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:fa7cd198280dbd0c988df525e50e35b5d16873e2cdae2aaaa6363cdb64e3eec5"}, + {file = "pyarrow-15.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8780b1a29d3c8b21ba6b191305a2a607de2e30dab399776ff0aa09131e266340"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe0ec198ccc680f6c92723fadcb97b74f07c45ff3fdec9dd765deb04955ccf19"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:036a7209c235588c2f07477fe75c07e6caced9b7b61bb897c8d4e52c4b5f9555"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2bd8a0e5296797faf9a3294e9fa2dc67aa7f10ae2207920dbebb785c77e9dbe5"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e8ebed6053dbe76883a822d4e8da36860f479d55a762bd9e70d8494aed87113e"}, + {file = "pyarrow-15.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:17d53a9d1b2b5bd7d5e4cd84d018e2a45bc9baaa68f7e6e3ebed45649900ba99"}, + {file = "pyarrow-15.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9950a9c9df24090d3d558b43b97753b8f5867fb8e521f29876aa021c52fda351"}, + {file = "pyarrow-15.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:003d680b5e422d0204e7287bb3fa775b332b3fce2996aa69e9adea23f5c8f970"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f75fce89dad10c95f4bf590b765e3ae98bcc5ba9f6ce75adb828a334e26a3d40"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca9cb0039923bec49b4fe23803807e4ef39576a2bec59c32b11296464623dc2"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ed5a78ed29d171d0acc26a305a4b7f83c122d54ff5270810ac23c75813585e4"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6eda9e117f0402dfcd3cd6ec9bfee89ac5071c48fc83a84f3075b60efa96747f"}, + {file = "pyarrow-15.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a3a6180c0e8f2727e6f1b1c87c72d3254cac909e609f35f22532e4115461177"}, + {file = "pyarrow-15.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:19a8918045993349b207de72d4576af0191beef03ea655d8bdb13762f0cd6eac"}, + {file = "pyarrow-15.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d0ec076b32bacb6666e8813a22e6e5a7ef1314c8069d4ff345efa6246bc38593"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5db1769e5d0a77eb92344c7382d6543bea1164cca3704f84aa44e26c67e320fb"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2617e3bf9df2a00020dd1c1c6dce5cc343d979efe10bc401c0632b0eef6ef5b"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:d31c1d45060180131caf10f0f698e3a782db333a422038bf7fe01dace18b3a31"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:c8c287d1d479de8269398b34282e206844abb3208224dbdd7166d580804674b7"}, + {file = "pyarrow-15.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:07eb7f07dc9ecbb8dace0f58f009d3a29ee58682fcdc91337dfeb51ea618a75b"}, + {file = "pyarrow-15.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:47af7036f64fce990bb8a5948c04722e4e3ea3e13b1007ef52dfe0aa8f23cf7f"}, + {file = "pyarrow-15.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93768ccfff85cf044c418bfeeafce9a8bb0cee091bd8fd19011aff91e58de540"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6ee87fd6892700960d90abb7b17a72a5abb3b64ee0fe8db6c782bcc2d0dc0b4"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:001fca027738c5f6be0b7a3159cc7ba16a5c52486db18160909a0831b063c4e4"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:d1c48648f64aec09accf44140dccb92f4f94394b8d79976c426a5b79b11d4fa7"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:972a0141be402bb18e3201448c8ae62958c9c7923dfaa3b3d4530c835ac81aed"}, + {file = "pyarrow-15.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:f01fc5cf49081426429127aa2d427d9d98e1cb94a32cb961d583a70b7c4504e6"}, + {file = "pyarrow-15.0.0.tar.gz", hash = "sha256:876858f549d540898f927eba4ef77cd549ad8d24baa3207cf1b72e5788b50e83"}, +] + +[package.dependencies] +numpy = ">=1.16.6,<2" + +[[package]] +name = "pyasn1" +version = "0.5.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "2.6.1" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.6.1-py3-none-any.whl", hash = "sha256:0b6a909df3192245cb736509a92ff69e4fef76116feffec68e93a567347bae6f"}, + {file = "pydantic-2.6.1.tar.gz", hash = "sha256:4fd5c182a2488dc63e6d32737ff19937888001e2a6d86e94b3f233104a5d1fa9"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.16.2" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.16.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.16.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3fab4e75b8c525a4776e7630b9ee48aea50107fea6ca9f593c98da3f4d11bf7c"}, + {file = "pydantic_core-2.16.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8bde5b48c65b8e807409e6f20baee5d2cd880e0fad00b1a811ebc43e39a00ab2"}, + {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2924b89b16420712e9bb8192396026a8fbd6d8726224f918353ac19c4c043d2a"}, + {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:16aa02e7a0f539098e215fc193c8926c897175d64c7926d00a36188917717a05"}, + {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:936a787f83db1f2115ee829dd615c4f684ee48ac4de5779ab4300994d8af325b"}, + {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:459d6be6134ce3b38e0ef76f8a672924460c455d45f1ad8fdade36796df1ddc8"}, + {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9ee4febb249c591d07b2d4dd36ebcad0ccd128962aaa1801508320896575ef"}, + {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40a0bd0bed96dae5712dab2aba7d334a6c67cbcac2ddfca7dbcc4a8176445990"}, + {file = "pydantic_core-2.16.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:870dbfa94de9b8866b37b867a2cb37a60c401d9deb4a9ea392abf11a1f98037b"}, + {file = "pydantic_core-2.16.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:308974fdf98046db28440eb3377abba274808bf66262e042c412eb2adf852731"}, + {file = "pydantic_core-2.16.2-cp310-none-win32.whl", hash = "sha256:a477932664d9611d7a0816cc3c0eb1f8856f8a42435488280dfbf4395e141485"}, + {file = "pydantic_core-2.16.2-cp310-none-win_amd64.whl", hash = "sha256:8f9142a6ed83d90c94a3efd7af8873bf7cefed2d3d44387bf848888482e2d25f"}, + {file = "pydantic_core-2.16.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:406fac1d09edc613020ce9cf3f2ccf1a1b2f57ab00552b4c18e3d5276c67eb11"}, + {file = "pydantic_core-2.16.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce232a6170dd6532096cadbf6185271e4e8c70fc9217ebe105923ac105da9978"}, + {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a90fec23b4b05a09ad988e7a4f4e081711a90eb2a55b9c984d8b74597599180f"}, + {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8aafeedb6597a163a9c9727d8a8bd363a93277701b7bfd2749fbefee2396469e"}, + {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9957433c3a1b67bdd4c63717eaf174ebb749510d5ea612cd4e83f2d9142f3fc8"}, + {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0d7a9165167269758145756db43a133608a531b1e5bb6a626b9ee24bc38a8f7"}, + {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dffaf740fe2e147fedcb6b561353a16243e654f7fe8e701b1b9db148242e1272"}, + {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8ed79883b4328b7f0bd142733d99c8e6b22703e908ec63d930b06be3a0e7113"}, + {file = "pydantic_core-2.16.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cf903310a34e14651c9de056fcc12ce090560864d5a2bb0174b971685684e1d8"}, + {file = "pydantic_core-2.16.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:46b0d5520dbcafea9a8645a8164658777686c5c524d381d983317d29687cce97"}, + {file = "pydantic_core-2.16.2-cp311-none-win32.whl", hash = "sha256:70651ff6e663428cea902dac297066d5c6e5423fda345a4ca62430575364d62b"}, + {file = "pydantic_core-2.16.2-cp311-none-win_amd64.whl", hash = "sha256:98dc6f4f2095fc7ad277782a7c2c88296badcad92316b5a6e530930b1d475ebc"}, + {file = "pydantic_core-2.16.2-cp311-none-win_arm64.whl", hash = "sha256:ef6113cd31411eaf9b39fc5a8848e71c72656fd418882488598758b2c8c6dfa0"}, + {file = "pydantic_core-2.16.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:88646cae28eb1dd5cd1e09605680c2b043b64d7481cdad7f5003ebef401a3039"}, + {file = "pydantic_core-2.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7b883af50eaa6bb3299780651e5be921e88050ccf00e3e583b1e92020333304b"}, + {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bf26c2e2ea59d32807081ad51968133af3025c4ba5753e6a794683d2c91bf6e"}, + {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:99af961d72ac731aae2a1b55ccbdae0733d816f8bfb97b41909e143de735f522"}, + {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02906e7306cb8c5901a1feb61f9ab5e5c690dbbeaa04d84c1b9ae2a01ebe9379"}, + {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5362d099c244a2d2f9659fb3c9db7c735f0004765bbe06b99be69fbd87c3f15"}, + {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ac426704840877a285d03a445e162eb258924f014e2f074e209d9b4ff7bf380"}, + {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b94cbda27267423411c928208e89adddf2ea5dd5f74b9528513f0358bba019cb"}, + {file = "pydantic_core-2.16.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6db58c22ac6c81aeac33912fb1af0e930bc9774166cdd56eade913d5f2fff35e"}, + {file = "pydantic_core-2.16.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:396fdf88b1b503c9c59c84a08b6833ec0c3b5ad1a83230252a9e17b7dfb4cffc"}, + {file = "pydantic_core-2.16.2-cp312-none-win32.whl", hash = "sha256:7c31669e0c8cc68400ef0c730c3a1e11317ba76b892deeefaf52dcb41d56ed5d"}, + {file = "pydantic_core-2.16.2-cp312-none-win_amd64.whl", hash = "sha256:a3b7352b48fbc8b446b75f3069124e87f599d25afb8baa96a550256c031bb890"}, + {file = "pydantic_core-2.16.2-cp312-none-win_arm64.whl", hash = "sha256:a9e523474998fb33f7c1a4d55f5504c908d57add624599e095c20fa575b8d943"}, + {file = "pydantic_core-2.16.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:ae34418b6b389d601b31153b84dce480351a352e0bb763684a1b993d6be30f17"}, + {file = "pydantic_core-2.16.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:732bd062c9e5d9582a30e8751461c1917dd1ccbdd6cafb032f02c86b20d2e7ec"}, + {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b52776a2e3230f4854907a1e0946eec04d41b1fc64069ee774876bbe0eab55"}, + {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef551c053692b1e39e3f7950ce2296536728871110e7d75c4e7753fb30ca87f4"}, + {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ebb892ed8599b23fa8f1799e13a12c87a97a6c9d0f497525ce9858564c4575a4"}, + {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa6c8c582036275997a733427b88031a32ffa5dfc3124dc25a730658c47a572f"}, + {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4ba0884a91f1aecce75202473ab138724aa4fb26d7707f2e1fa6c3e68c84fbf"}, + {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7924e54f7ce5d253d6160090ddc6df25ed2feea25bfb3339b424a9dd591688bc"}, + {file = "pydantic_core-2.16.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69a7b96b59322a81c2203be537957313b07dd333105b73db0b69212c7d867b4b"}, + {file = "pydantic_core-2.16.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7e6231aa5bdacda78e96ad7b07d0c312f34ba35d717115f4b4bff6cb87224f0f"}, + {file = "pydantic_core-2.16.2-cp38-none-win32.whl", hash = "sha256:41dac3b9fce187a25c6253ec79a3f9e2a7e761eb08690e90415069ea4a68ff7a"}, + {file = "pydantic_core-2.16.2-cp38-none-win_amd64.whl", hash = "sha256:f685dbc1fdadb1dcd5b5e51e0a378d4685a891b2ddaf8e2bba89bd3a7144e44a"}, + {file = "pydantic_core-2.16.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:55749f745ebf154c0d63d46c8c58594d8894b161928aa41adbb0709c1fe78b77"}, + {file = "pydantic_core-2.16.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b30b0dd58a4509c3bd7eefddf6338565c4905406aee0c6e4a5293841411a1286"}, + {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18de31781cdc7e7b28678df7c2d7882f9692ad060bc6ee3c94eb15a5d733f8f7"}, + {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5864b0242f74b9dd0b78fd39db1768bc3f00d1ffc14e596fd3e3f2ce43436a33"}, + {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8f9186ca45aee030dc8234118b9c0784ad91a0bb27fc4e7d9d6608a5e3d386c"}, + {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc6f6c9be0ab6da37bc77c2dda5f14b1d532d5dbef00311ee6e13357a418e646"}, + {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa057095f621dad24a1e906747179a69780ef45cc8f69e97463692adbcdae878"}, + {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ad84731a26bcfb299f9eab56c7932d46f9cad51c52768cace09e92a19e4cf55"}, + {file = "pydantic_core-2.16.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3b052c753c4babf2d1edc034c97851f867c87d6f3ea63a12e2700f159f5c41c3"}, + {file = "pydantic_core-2.16.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e0f686549e32ccdb02ae6f25eee40cc33900910085de6aa3790effd391ae10c2"}, + {file = "pydantic_core-2.16.2-cp39-none-win32.whl", hash = "sha256:7afb844041e707ac9ad9acad2188a90bffce2c770e6dc2318be0c9916aef1469"}, + {file = "pydantic_core-2.16.2-cp39-none-win_amd64.whl", hash = "sha256:9da90d393a8227d717c19f5397688a38635afec89f2e2d7af0df037f3249c39a"}, + {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f60f920691a620b03082692c378661947d09415743e437a7478c309eb0e4f82"}, + {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:47924039e785a04d4a4fa49455e51b4eb3422d6eaacfde9fc9abf8fdef164e8a"}, + {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6294e76b0380bb7a61eb8a39273c40b20beb35e8c87ee101062834ced19c545"}, + {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe56851c3f1d6f5384b3051c536cc81b3a93a73faf931f404fef95217cf1e10d"}, + {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9d776d30cde7e541b8180103c3f294ef7c1862fd45d81738d156d00551005784"}, + {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:72f7919af5de5ecfaf1eba47bf9a5d8aa089a3340277276e5636d16ee97614d7"}, + {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:4bfcbde6e06c56b30668a0c872d75a7ef3025dc3c1823a13cf29a0e9b33f67e8"}, + {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ff7c97eb7a29aba230389a2661edf2e9e06ce616c7e35aa764879b6894a44b25"}, + {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9b5f13857da99325dcabe1cc4e9e6a3d7b2e2c726248ba5dd4be3e8e4a0b6d0e"}, + {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a7e41e3ada4cca5f22b478c08e973c930e5e6c7ba3588fb8e35f2398cdcc1545"}, + {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60eb8ceaa40a41540b9acae6ae7c1f0a67d233c40dc4359c256ad2ad85bdf5e5"}, + {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7beec26729d496a12fd23cf8da9944ee338c8b8a17035a560b585c36fe81af20"}, + {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22c5f022799f3cd6741e24f0443ead92ef42be93ffda0d29b2597208c94c3753"}, + {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:eca58e319f4fd6df004762419612122b2c7e7d95ffafc37e890252f869f3fb2a"}, + {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed957db4c33bc99895f3a1672eca7e80e8cda8bd1e29a80536b4ec2153fa9804"}, + {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:459c0d338cc55d099798618f714b21b7ece17eb1a87879f2da20a3ff4c7628e2"}, + {file = "pydantic_core-2.16.2.tar.gz", hash = "sha256:0ba503850d8b8dcc18391f10de896ae51d37fe5fe43dbfb6a35c5c5cad271a06"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pydantic-settings" +version = "2.1.0" +description = "Settings management using Pydantic" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_settings-2.1.0-py3-none-any.whl", hash = "sha256:7621c0cb5d90d1140d2f0ef557bdf03573aac7035948109adf2574770b77605a"}, + {file = "pydantic_settings-2.1.0.tar.gz", hash = "sha256:26b1492e0a24755626ac5e6d715e9077ab7ad4fb5f19a8b7ed7011d52f36141c"}, +] + +[package.dependencies] +pydantic = ">=2.3.0" +python-dotenv = ">=0.21.0" + +[[package]] +name = "pygments" +version = "2.17.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, +] + +[package.extras] +plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pymdown-extensions" +version = "10.7" +description = "Extension pack for Python Markdown." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pymdown_extensions-10.7-py3-none-any.whl", hash = "sha256:6ca215bc57bc12bf32b414887a68b810637d039124ed9b2e5bd3325cbb2c050c"}, + {file = "pymdown_extensions-10.7.tar.gz", hash = "sha256:c0d64d5cf62566f59e6b2b690a4095c931107c250a8c8e1351c1de5f6b036deb"}, +] + +[package.dependencies] +markdown = ">=3.5" +pyyaml = "*" + +[package.extras] +extra = ["pygments (>=2.12)"] + +[[package]] +name = "pynndescent" +version = "0.5.11" +description = "Nearest Neighbor Descent" +optional = false +python-versions = "*" +files = [ + {file = "pynndescent-0.5.11-py3-none-any.whl", hash = "sha256:a628f4fc8a67757c8fa15613449ac513fd056258a55b4084e47c06640ec90a8d"}, + {file = "pynndescent-0.5.11.tar.gz", hash = "sha256:6f44ced9d5a9da2c87d9b2fff30bb5308540c0657605e4d5cde7ed3275bbad50"}, +] + +[package.dependencies] +joblib = ">=0.11" +llvmlite = ">=0.30" +numba = ">=0.51.2" +scikit-learn = ">=0.18" +scipy = ">=1.0" + +[[package]] +name = "pypika" +version = "0.48.9" +description = "A SQL query builder API for Python" +optional = false +python-versions = "*" +files = [ + {file = "PyPika-0.48.9.tar.gz", hash = "sha256:838836a61747e7c8380cd1b7ff638694b7a7335345d0f559b04b2cd832ad5378"}, +] + +[[package]] +name = "pyproject-hooks" +version = "1.0.0" +description = "Wrappers to call pyproject.toml-based build backend hooks." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, + {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, +] + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "pyreadline3" +version = "3.4.1" +description = "A python implementation of GNU readline." +optional = false +python-versions = "*" +files = [ + {file = "pyreadline3-3.4.1-py3-none-any.whl", hash = "sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb"}, + {file = "pyreadline3-3.4.1.tar.gz", hash = "sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae"}, +] + +[[package]] +name = "pytest" +version = "8.0.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.0.0-py3-none-any.whl", hash = "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6"}, + {file = "pytest-8.0.0.tar.gz", hash = "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.3.0,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-asyncio" +version = "0.21.1" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-asyncio-0.21.1.tar.gz", hash = "sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d"}, + {file = "pytest_asyncio-0.21.1-py3-none-any.whl", hash = "sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "python-frontmatter" +version = "1.1.0" +description = "Parse and manage posts with YAML (or other) frontmatter" +optional = false +python-versions = "*" +files = [ + {file = "python-frontmatter-1.1.0.tar.gz", hash = "sha256:7118d2bd56af9149625745c58c9b51fb67e8d1294a0c76796dafdc72c36e5f6d"}, + {file = "python_frontmatter-1.1.0-py3-none-any.whl", hash = "sha256:335465556358d9d0e6c98bbeb69b1c969f2a4a21360587b9873bfc3b213407c1"}, +] + +[package.dependencies] +PyYAML = "*" + +[package.extras] +docs = ["sphinx"] +test = ["mypy", "pyaml", "pytest", "toml", "types-PyYAML", "types-toml"] + +[[package]] +name = "python-iso639" +version = "2024.1.2" +description = "Look-up utilities for ISO 639 language codes and names" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-iso639-2024.1.2.tar.gz", hash = "sha256:80d30382d3089bc8430144b2b19dd4afe165914c2f03942eb8d2f49e4adfe18b"}, + {file = "python_iso639-2024.1.2-py3-none-any.whl", hash = "sha256:a4c3fb99081138227bdc34c1abeac96b083bf287f29c01dc26ed57c8b04ac416"}, +] + +[package.extras] +dev = ["black (==23.12.1)", "build (==1.0.3)", "flake8 (==6.1.0)", "pytest (==7.4.4)", "twine (==4.0.2)"] + +[[package]] +name = "python-json-logger" +version = "2.0.7" +description = "A python library adding a json log formatter" +optional = false +python-versions = ">=3.6" +files = [ + {file = "python-json-logger-2.0.7.tar.gz", hash = "sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c"}, + {file = "python_json_logger-2.0.7-py3-none-any.whl", hash = "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd"}, +] + +[[package]] +name = "python-magic" +version = "0.4.27" +description = "File type identification using libmagic" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b"}, + {file = "python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3"}, +] + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + +[[package]] +name = "pywinpty" +version = "2.0.12" +description = "Pseudo terminal support for Windows from Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pywinpty-2.0.12-cp310-none-win_amd64.whl", hash = "sha256:21319cd1d7c8844fb2c970fb3a55a3db5543f112ff9cfcd623746b9c47501575"}, + {file = "pywinpty-2.0.12-cp311-none-win_amd64.whl", hash = "sha256:853985a8f48f4731a716653170cd735da36ffbdc79dcb4c7b7140bce11d8c722"}, + {file = "pywinpty-2.0.12-cp312-none-win_amd64.whl", hash = "sha256:1617b729999eb6713590e17665052b1a6ae0ad76ee31e60b444147c5b6a35dca"}, + {file = "pywinpty-2.0.12-cp38-none-win_amd64.whl", hash = "sha256:189380469ca143d06e19e19ff3fba0fcefe8b4a8cc942140a6b863aed7eebb2d"}, + {file = "pywinpty-2.0.12-cp39-none-win_amd64.whl", hash = "sha256:7520575b6546db23e693cbd865db2764097bd6d4ef5dc18c92555904cd62c3d4"}, + {file = "pywinpty-2.0.12.tar.gz", hash = "sha256:8197de460ae8ebb7f5d1701dfa1b5df45b157bb832e92acba316305e18ca00dd"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "pyzmq" +version = "25.1.2" +description = "Python bindings for 0MQ" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:e624c789359f1a16f83f35e2c705d07663ff2b4d4479bad35621178d8f0f6ea4"}, + {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49151b0efece79f6a79d41a461d78535356136ee70084a1c22532fc6383f4ad0"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9a5f194cf730f2b24d6af1f833c14c10f41023da46a7f736f48b6d35061e76e"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:faf79a302f834d9e8304fafdc11d0d042266667ac45209afa57e5efc998e3872"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f51a7b4ead28d3fca8dda53216314a553b0f7a91ee8fc46a72b402a78c3e43d"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0ddd6d71d4ef17ba5a87becf7ddf01b371eaba553c603477679ae817a8d84d75"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:246747b88917e4867e2367b005fc8eefbb4a54b7db363d6c92f89d69abfff4b6"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:00c48ae2fd81e2a50c3485de1b9d5c7c57cd85dc8ec55683eac16846e57ac979"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a68d491fc20762b630e5db2191dd07ff89834086740f70e978bb2ef2668be08"}, + {file = "pyzmq-25.1.2-cp310-cp310-win32.whl", hash = "sha256:09dfe949e83087da88c4a76767df04b22304a682d6154de2c572625c62ad6886"}, + {file = "pyzmq-25.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:fa99973d2ed20417744fca0073390ad65ce225b546febb0580358e36aa90dba6"}, + {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:82544e0e2d0c1811482d37eef297020a040c32e0687c1f6fc23a75b75db8062c"}, + {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:01171fc48542348cd1a360a4b6c3e7d8f46cdcf53a8d40f84db6707a6768acc1"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc69c96735ab501419c432110016329bf0dea8898ce16fab97c6d9106dc0b348"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3e124e6b1dd3dfbeb695435dff0e383256655bb18082e094a8dd1f6293114642"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7598d2ba821caa37a0f9d54c25164a4fa351ce019d64d0b44b45540950458840"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d1299d7e964c13607efd148ca1f07dcbf27c3ab9e125d1d0ae1d580a1682399d"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4e6f689880d5ad87918430957297c975203a082d9a036cc426648fcbedae769b"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cc69949484171cc961e6ecd4a8911b9ce7a0d1f738fcae717177c231bf77437b"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9880078f683466b7f567b8624bfc16cad65077be046b6e8abb53bed4eeb82dd3"}, + {file = "pyzmq-25.1.2-cp311-cp311-win32.whl", hash = "sha256:4e5837af3e5aaa99a091302df5ee001149baff06ad22b722d34e30df5f0d9097"}, + {file = "pyzmq-25.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:25c2dbb97d38b5ac9fd15586e048ec5eb1e38f3d47fe7d92167b0c77bb3584e9"}, + {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:11e70516688190e9c2db14fcf93c04192b02d457b582a1f6190b154691b4c93a"}, + {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:313c3794d650d1fccaaab2df942af9f2c01d6217c846177cfcbc693c7410839e"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b3cbba2f47062b85fe0ef9de5b987612140a9ba3a9c6d2543c6dec9f7c2ab27"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fc31baa0c32a2ca660784d5af3b9487e13b61b3032cb01a115fce6588e1bed30"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c9087b109070c5ab0b383079fa1b5f797f8d43e9a66c07a4b8b8bdecfd88ee"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f8429b17cbb746c3e043cb986328da023657e79d5ed258b711c06a70c2ea7537"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5074adeacede5f810b7ef39607ee59d94e948b4fd954495bdb072f8c54558181"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7ae8f354b895cbd85212da245f1a5ad8159e7840e37d78b476bb4f4c3f32a9fe"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b264bf2cc96b5bc43ce0e852be995e400376bd87ceb363822e2cb1964fcdc737"}, + {file = "pyzmq-25.1.2-cp312-cp312-win32.whl", hash = "sha256:02bbc1a87b76e04fd780b45e7f695471ae6de747769e540da909173d50ff8e2d"}, + {file = "pyzmq-25.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:ced111c2e81506abd1dc142e6cd7b68dd53747b3b7ae5edbea4578c5eeff96b7"}, + {file = "pyzmq-25.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7b6d09a8962a91151f0976008eb7b29b433a560fde056ec7a3db9ec8f1075438"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:967668420f36878a3c9ecb5ab33c9d0ff8d054f9c0233d995a6d25b0e95e1b6b"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5edac3f57c7ddaacdb4d40f6ef2f9e299471fc38d112f4bc6d60ab9365445fb0"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0dabfb10ef897f3b7e101cacba1437bd3a5032ee667b7ead32bbcdd1a8422fe7"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2c6441e0398c2baacfe5ba30c937d274cfc2dc5b55e82e3749e333aabffde561"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:16b726c1f6c2e7625706549f9dbe9b06004dfbec30dbed4bf50cbdfc73e5b32a"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:a86c2dd76ef71a773e70551a07318b8e52379f58dafa7ae1e0a4be78efd1ff16"}, + {file = "pyzmq-25.1.2-cp36-cp36m-win32.whl", hash = "sha256:359f7f74b5d3c65dae137f33eb2bcfa7ad9ebefd1cab85c935f063f1dbb245cc"}, + {file = "pyzmq-25.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:55875492f820d0eb3417b51d96fea549cde77893ae3790fd25491c5754ea2f68"}, + {file = "pyzmq-25.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8c8a419dfb02e91b453615c69568442e897aaf77561ee0064d789705ff37a92"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8807c87fa893527ae8a524c15fc505d9950d5e856f03dae5921b5e9aa3b8783b"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5e319ed7d6b8f5fad9b76daa0a68497bc6f129858ad956331a5835785761e003"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3c53687dde4d9d473c587ae80cc328e5b102b517447456184b485587ebd18b62"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9add2e5b33d2cd765ad96d5eb734a5e795a0755f7fc49aa04f76d7ddda73fd70"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e690145a8c0c273c28d3b89d6fb32c45e0d9605b2293c10e650265bf5c11cfec"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:00a06faa7165634f0cac1abb27e54d7a0b3b44eb9994530b8ec73cf52e15353b"}, + {file = "pyzmq-25.1.2-cp37-cp37m-win32.whl", hash = "sha256:0f97bc2f1f13cb16905a5f3e1fbdf100e712d841482b2237484360f8bc4cb3d7"}, + {file = "pyzmq-25.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6cc0020b74b2e410287e5942e1e10886ff81ac77789eb20bec13f7ae681f0fdd"}, + {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:bef02cfcbded83473bdd86dd8d3729cd82b2e569b75844fb4ea08fee3c26ae41"}, + {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e10a4b5a4b1192d74853cc71a5e9fd022594573926c2a3a4802020360aa719d8"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8c5f80e578427d4695adac6fdf4370c14a2feafdc8cb35549c219b90652536ae"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5dde6751e857910c1339890f3524de74007958557593b9e7e8c5f01cd919f8a7"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea1608dd169da230a0ad602d5b1ebd39807ac96cae1845c3ceed39af08a5c6df"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0f513130c4c361201da9bc69df25a086487250e16b5571ead521b31ff6b02220"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:019744b99da30330798bb37df33549d59d380c78e516e3bab9c9b84f87a9592f"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2e2713ef44be5d52dd8b8e2023d706bf66cb22072e97fc71b168e01d25192755"}, + {file = "pyzmq-25.1.2-cp38-cp38-win32.whl", hash = "sha256:07cd61a20a535524906595e09344505a9bd46f1da7a07e504b315d41cd42eb07"}, + {file = "pyzmq-25.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb7e49a17fb8c77d3119d41a4523e432eb0c6932187c37deb6fbb00cc3028088"}, + {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:94504ff66f278ab4b7e03e4cba7e7e400cb73bfa9d3d71f58d8972a8dc67e7a6"}, + {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6dd0d50bbf9dca1d0bdea219ae6b40f713a3fb477c06ca3714f208fd69e16fd8"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:004ff469d21e86f0ef0369717351073e0e577428e514c47c8480770d5e24a565"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c0b5ca88a8928147b7b1e2dfa09f3b6c256bc1135a1338536cbc9ea13d3b7add"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9a79f1d2495b167119d02be7448bfba57fad2a4207c4f68abc0bab4b92925b"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:518efd91c3d8ac9f9b4f7dd0e2b7b8bf1a4fe82a308009016b07eaa48681af82"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1ec23bd7b3a893ae676d0e54ad47d18064e6c5ae1fadc2f195143fb27373f7f6"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db36c27baed588a5a8346b971477b718fdc66cf5b80cbfbd914b4d6d355e44e2"}, + {file = "pyzmq-25.1.2-cp39-cp39-win32.whl", hash = "sha256:39b1067f13aba39d794a24761e385e2eddc26295826530a8c7b6c6c341584289"}, + {file = "pyzmq-25.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:8e9f3fabc445d0ce320ea2c59a75fe3ea591fdbdeebec5db6de530dd4b09412e"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a8c1d566344aee826b74e472e16edae0a02e2a044f14f7c24e123002dcff1c05"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:759cfd391a0996345ba94b6a5110fca9c557ad4166d86a6e81ea526c376a01e8"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c61e346ac34b74028ede1c6b4bcecf649d69b707b3ff9dc0fab453821b04d1e"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cb8fc1f8d69b411b8ec0b5f1ffbcaf14c1db95b6bccea21d83610987435f1a4"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3c00c9b7d1ca8165c610437ca0c92e7b5607b2f9076f4eb4b095c85d6e680a1d"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:df0c7a16ebb94452d2909b9a7b3337940e9a87a824c4fc1c7c36bb4404cb0cde"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:45999e7f7ed5c390f2e87ece7f6c56bf979fb213550229e711e45ecc7d42ccb8"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ac170e9e048b40c605358667aca3d94e98f604a18c44bdb4c102e67070f3ac9b"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1b604734bec94f05f81b360a272fc824334267426ae9905ff32dc2be433ab96"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:a793ac733e3d895d96f865f1806f160696422554e46d30105807fdc9841b9f7d"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0806175f2ae5ad4b835ecd87f5f85583316b69f17e97786f7443baaf54b9bb98"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ef12e259e7bc317c7597d4f6ef59b97b913e162d83b421dd0db3d6410f17a244"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea253b368eb41116011add00f8d5726762320b1bda892f744c91997b65754d73"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b9b1f2ad6498445a941d9a4fee096d387fee436e45cc660e72e768d3d8ee611"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8b14c75979ce932c53b79976a395cb2a8cd3aaf14aef75e8c2cb55a330b9b49d"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:889370d5174a741a62566c003ee8ddba4b04c3f09a97b8000092b7ca83ec9c49"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a18fff090441a40ffda8a7f4f18f03dc56ae73f148f1832e109f9bffa85df15"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99a6b36f95c98839ad98f8c553d8507644c880cf1e0a57fe5e3a3f3969040882"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4345c9a27f4310afbb9c01750e9461ff33d6fb74cd2456b107525bbeebcb5be3"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3516e0b6224cf6e43e341d56da15fd33bdc37fa0c06af4f029f7d7dfceceabbc"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:146b9b1f29ead41255387fb07be56dc29639262c0f7344f570eecdcd8d683314"}, + {file = "pyzmq-25.1.2.tar.gz", hash = "sha256:93f1aa311e8bb912e34f004cf186407a4e90eec4f0ecc0efd26056bf7eda0226"}, +] + +[package.dependencies] +cffi = {version = "*", markers = "implementation_name == \"pypy\""} + +[[package]] +name = "rapidfuzz" +version = "3.6.1" +description = "rapid fuzzy string matching" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ac434fc71edda30d45db4a92ba5e7a42c7405e1a54cb4ec01d03cc668c6dcd40"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a791168e119cfddf4b5a40470620c872812042f0621e6a293983a2d52372db0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5a2f3e9df346145c2be94e4d9eeffb82fab0cbfee85bd4a06810e834fe7c03fa"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23de71e7f05518b0bbeef55d67b5dbce3bcd3e2c81e7e533051a2e9401354eb0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d056e342989248d2bdd67f1955bb7c3b0ecfa239d8f67a8dfe6477b30872c607"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01835d02acd5d95c1071e1da1bb27fe213c84a013b899aba96380ca9962364bc"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed0f712e0bb5fea327e92aec8a937afd07ba8de4c529735d82e4c4124c10d5a0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96cd19934f76a1264e8ecfed9d9f5291fde04ecb667faef5f33bdbfd95fe2d1f"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e06c4242a1354cf9d48ee01f6f4e6e19c511d50bb1e8d7d20bcadbb83a2aea90"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d73dcfe789d37c6c8b108bf1e203e027714a239e50ad55572ced3c004424ed3b"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:06e98ff000e2619e7cfe552d086815671ed09b6899408c2c1b5103658261f6f3"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:08b6fb47dd889c69fbc0b915d782aaed43e025df6979b6b7f92084ba55edd526"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a1788ebb5f5b655a15777e654ea433d198f593230277e74d51a2a1e29a986283"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c65f92881753aa1098c77818e2b04a95048f30edbe9c3094dc3707d67df4598b"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:4243a9c35667a349788461aae6471efde8d8800175b7db5148a6ab929628047f"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win_arm64.whl", hash = "sha256:f59d19078cc332dbdf3b7b210852ba1f5db8c0a2cd8cc4c0ed84cc00c76e6802"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fbc07e2e4ac696497c5f66ec35c21ddab3fc7a406640bffed64c26ab2f7ce6d6"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40cced1a8852652813f30fb5d4b8f9b237112a0bbaeebb0f4cc3611502556764"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:82300e5f8945d601c2daaaac139d5524d7c1fdf719aa799a9439927739917460"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edf97c321fd641fea2793abce0e48fa4f91f3c202092672f8b5b4e781960b891"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7420e801b00dee4a344ae2ee10e837d603461eb180e41d063699fb7efe08faf0"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:060bd7277dc794279fa95522af355034a29c90b42adcb7aa1da358fc839cdb11"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7e3375e4f2bfec77f907680328e4cd16cc64e137c84b1886d547ab340ba6928"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a490cd645ef9d8524090551016f05f052e416c8adb2d8b85d35c9baa9d0428ab"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2e03038bfa66d2d7cffa05d81c2f18fd6acbb25e7e3c068d52bb7469e07ff382"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b19795b26b979c845dba407fe79d66975d520947b74a8ab6cee1d22686f7967"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:064c1d66c40b3a0f488db1f319a6e75616b2e5fe5430a59f93a9a5e40a656d15"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3c772d04fb0ebeece3109d91f6122b1503023086a9591a0b63d6ee7326bd73d9"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:841eafba6913c4dfd53045835545ba01a41e9644e60920c65b89c8f7e60c00a9"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win32.whl", hash = "sha256:266dd630f12696ea7119f31d8b8e4959ef45ee2cbedae54417d71ae6f47b9848"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:d79aec8aeee02ab55d0ddb33cea3ecd7b69813a48e423c966a26d7aab025cdfe"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win_arm64.whl", hash = "sha256:484759b5dbc5559e76fefaa9170147d1254468f555fd9649aea3bad46162a88b"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b2ef4c0fd3256e357b70591ffb9e8ed1d439fb1f481ba03016e751a55261d7c1"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:588c4b20fa2fae79d60a4e438cf7133d6773915df3cc0a7f1351da19eb90f720"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7142ee354e9c06e29a2636b9bbcb592bb00600a88f02aa5e70e4f230347b373e"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1dfc557c0454ad22382373ec1b7df530b4bbd974335efe97a04caec936f2956a"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:03f73b381bdeccb331a12c3c60f1e41943931461cdb52987f2ecf46bfc22f50d"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b0ccc2ec1781c7e5370d96aef0573dd1f97335343e4982bdb3a44c133e27786"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da3e8c9f7e64bb17faefda085ff6862ecb3ad8b79b0f618a6cf4452028aa2222"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fde9b14302a31af7bdafbf5cfbb100201ba21519be2b9dedcf4f1048e4fbe65d"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c1a23eee225dfb21c07f25c9fcf23eb055d0056b48e740fe241cbb4b22284379"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e49b9575d16c56c696bc7b06a06bf0c3d4ef01e89137b3ddd4e2ce709af9fe06"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:0a9fc714b8c290261669f22808913aad49553b686115ad0ee999d1cb3df0cd66"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:a3ee4f8f076aa92184e80308fc1a079ac356b99c39408fa422bbd00145be9854"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f056ba42fd2f32e06b2c2ba2443594873cfccc0c90c8b6327904fc2ddf6d5799"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win32.whl", hash = "sha256:5d82b9651e3d34b23e4e8e201ecd3477c2baa17b638979deeabbb585bcb8ba74"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:dad55a514868dae4543ca48c4e1fc0fac704ead038dafedf8f1fc0cc263746c1"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win_arm64.whl", hash = "sha256:3c84294f4470fcabd7830795d754d808133329e0a81d62fcc2e65886164be83b"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e19d519386e9db4a5335a4b29f25b8183a1c3f78cecb4c9c3112e7f86470e37f"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01eb03cd880a294d1bf1a583fdd00b87169b9cc9c9f52587411506658c864d73"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:be368573255f8fbb0125a78330a1a40c65e9ba3c5ad129a426ff4289099bfb41"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3e5af946f419c30f5cb98b69d40997fe8580efe78fc83c2f0f25b60d0e56efb"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f382f7ffe384ce34345e1c0b2065451267d3453cadde78946fbd99a59f0cc23c"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be156f51f3a4f369e758505ed4ae64ea88900dcb2f89d5aabb5752676d3f3d7e"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1936d134b6c513fbe934aeb668b0fee1ffd4729a3c9d8d373f3e404fbb0ce8a0"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12ff8eaf4a9399eb2bebd838f16e2d1ded0955230283b07376d68947bbc2d33d"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ae598a172e3a95df3383634589660d6b170cc1336fe7578115c584a99e0ba64d"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cd4ba4c18b149da11e7f1b3584813159f189dc20833709de5f3df8b1342a9759"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:0402f1629e91a4b2e4aee68043a30191e5e1b7cd2aa8dacf50b1a1bcf6b7d3ab"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:1e12319c6b304cd4c32d5db00b7a1e36bdc66179c44c5707f6faa5a889a317c0"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0bbfae35ce4de4c574b386c43c78a0be176eeddfdae148cb2136f4605bebab89"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-win32.whl", hash = "sha256:7fec74c234d3097612ea80f2a80c60720eec34947066d33d34dc07a3092e8105"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:a553cc1a80d97459d587529cc43a4c7c5ecf835f572b671107692fe9eddf3e24"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:757dfd7392ec6346bd004f8826afb3bf01d18a723c97cbe9958c733ab1a51791"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2963f4a3f763870a16ee076796be31a4a0958fbae133dbc43fc55c3968564cf5"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d2f0274595cc5b2b929c80d4e71b35041104b577e118cf789b3fe0a77b37a4c5"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f211e366e026de110a4246801d43a907cd1a10948082f47e8a4e6da76fef52"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a59472b43879012b90989603aa5a6937a869a72723b1bf2ff1a0d1edee2cc8e6"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a03863714fa6936f90caa7b4b50ea59ea32bb498cc91f74dc25485b3f8fccfe9"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5dd95b6b7bfb1584f806db89e1e0c8dbb9d25a30a4683880c195cc7f197eaf0c"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7183157edf0c982c0b8592686535c8b3e107f13904b36d85219c77be5cefd0d8"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ad9d74ef7c619b5b0577e909582a1928d93e07d271af18ba43e428dc3512c2a1"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b53137d81e770c82189e07a8f32722d9e4260f13a0aec9914029206ead38cac3"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:49b9ed2472394d306d5dc967a7de48b0aab599016aa4477127b20c2ed982dbf9"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:dec307b57ec2d5054d77d03ee4f654afcd2c18aee00c48014cb70bfed79597d6"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4381023fa1ff32fd5076f5d8321249a9aa62128eb3f21d7ee6a55373e672b261"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win32.whl", hash = "sha256:8d7a072f10ee57c8413c8ab9593086d42aaff6ee65df4aa6663eecdb7c398dca"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:ebcfb5bfd0a733514352cfc94224faad8791e576a80ffe2fd40b2177bf0e7198"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win_arm64.whl", hash = "sha256:1c47d592e447738744905c18dda47ed155620204714e6df20eb1941bb1ba315e"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:eef8b346ab331bec12bbc83ac75641249e6167fab3d84d8f5ca37fd8e6c7a08c"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53251e256017e2b87f7000aee0353ba42392c442ae0bafd0f6b948593d3f68c6"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6dede83a6b903e3ebcd7e8137e7ff46907ce9316e9d7e7f917d7e7cdc570ee05"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4da90e4c2b444d0a171d7444ea10152e07e95972bb40b834a13bdd6de1110c"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:ca3dfcf74f2b6962f411c33dd95b0adf3901266e770da6281bc96bb5a8b20de9"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bcc957c0a8bde8007f1a8a413a632a1a409890f31f73fe764ef4eac55f59ca87"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:692c9a50bea7a8537442834f9bc6b7d29d8729a5b6379df17c31b6ab4df948c2"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c23ceaea27e790ddd35ef88b84cf9d721806ca366199a76fd47cfc0457a81b"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b155e67fff215c09f130555002e42f7517d0ea72cbd58050abb83cb7c880cec"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3028ee8ecc48250607fa8a0adce37b56275ec3b1acaccd84aee1f68487c8557b"}, + {file = "rapidfuzz-3.6.1.tar.gz", hash = "sha256:35660bee3ce1204872574fa041c7ad7ec5175b3053a4cb6e181463fc07013de7"}, +] + +[package.extras] +full = ["numpy"] + +[[package]] +name = "referencing" +version = "0.33.0" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.33.0-py3-none-any.whl", hash = "sha256:39240f2ecc770258f28b642dd47fd74bc8b02484de54e1882b74b35ebd779bd5"}, + {file = "referencing-0.33.0.tar.gz", hash = "sha256:c775fedf74bc0f9189c2a3be1c12fd03e8c23f4d371dce795df44e06c5b412f7"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + +[[package]] +name = "regex" +version = "2023.12.25" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.7" +files = [ + {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0694219a1d54336fd0445ea382d49d36882415c0134ee1e8332afd1529f0baa5"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b014333bd0217ad3d54c143de9d4b9a3ca1c5a29a6d0d554952ea071cff0f1f8"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d865984b3f71f6d0af64d0d88f5733521698f6c16f445bb09ce746c92c97c586"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e0eabac536b4cc7f57a5f3d095bfa557860ab912f25965e08fe1545e2ed8b4c"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c25a8ad70e716f96e13a637802813f65d8a6760ef48672aa3502f4c24ea8b400"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9b6d73353f777630626f403b0652055ebfe8ff142a44ec2cf18ae470395766e"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9cc99d6946d750eb75827cb53c4371b8b0fe89c733a94b1573c9dd16ea6c9e4"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88d1f7bef20c721359d8675f7d9f8e414ec5003d8f642fdfd8087777ff7f94b5"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cb3fe77aec8f1995611f966d0c656fdce398317f850d0e6e7aebdfe61f40e1cd"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7aa47c2e9ea33a4a2a05f40fcd3ea36d73853a2aae7b4feab6fc85f8bf2c9704"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:df26481f0c7a3f8739fecb3e81bc9da3fcfae34d6c094563b9d4670b047312e1"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c40281f7d70baf6e0db0c2f7472b31609f5bc2748fe7275ea65a0b4601d9b392"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:d94a1db462d5690ebf6ae86d11c5e420042b9898af5dcf278bd97d6bda065423"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba1b30765a55acf15dce3f364e4928b80858fa8f979ad41f862358939bdd1f2f"}, + {file = "regex-2023.12.25-cp310-cp310-win32.whl", hash = "sha256:150c39f5b964e4d7dba46a7962a088fbc91f06e606f023ce57bb347a3b2d4630"}, + {file = "regex-2023.12.25-cp310-cp310-win_amd64.whl", hash = "sha256:09da66917262d9481c719599116c7dc0c321ffcec4b1f510c4f8a066f8768105"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b9d811f72210fa9306aeb88385b8f8bcef0dfbf3873410413c00aa94c56c2b6"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d902a43085a308cef32c0d3aea962524b725403fd9373dea18110904003bac97"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d166eafc19f4718df38887b2bbe1467a4f74a9830e8605089ea7a30dd4da8887"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7ad32824b7f02bb3c9f80306d405a1d9b7bb89362d68b3c5a9be53836caebdb"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:636ba0a77de609d6510235b7f0e77ec494d2657108f777e8765efc060094c98c"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fda75704357805eb953a3ee15a2b240694a9a514548cd49b3c5124b4e2ad01b"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f72cbae7f6b01591f90814250e636065850c5926751af02bb48da94dfced7baa"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db2a0b1857f18b11e3b0e54ddfefc96af46b0896fb678c85f63fb8c37518b3e7"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7502534e55c7c36c0978c91ba6f61703faf7ce733715ca48f499d3dbbd7657e0"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e8c7e08bb566de4faaf11984af13f6bcf6a08f327b13631d41d62592681d24fe"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:283fc8eed679758de38fe493b7d7d84a198b558942b03f017b1f94dda8efae80"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f44dd4d68697559d007462b0a3a1d9acd61d97072b71f6d1968daef26bc744bd"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:67d3ccfc590e5e7197750fcb3a2915b416a53e2de847a728cfa60141054123d4"}, + {file = "regex-2023.12.25-cp311-cp311-win32.whl", hash = "sha256:68191f80a9bad283432385961d9efe09d783bcd36ed35a60fb1ff3f1ec2efe87"}, + {file = "regex-2023.12.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d2af3f6b8419661a0c421584cfe8aaec1c0e435ce7e47ee2a97e344b98f794f"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8a0ccf52bb37d1a700375a6b395bff5dd15c50acb745f7db30415bae3c2b0715"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c3c4a78615b7762740531c27cf46e2f388d8d727d0c0c739e72048beb26c8a9d"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad83e7545b4ab69216cef4cc47e344d19622e28aabec61574b20257c65466d6a"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7a635871143661feccce3979e1727c4e094f2bdfd3ec4b90dfd4f16f571a87a"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d498eea3f581fbe1b34b59c697512a8baef88212f92e4c7830fcc1499f5b45a5"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43f7cd5754d02a56ae4ebb91b33461dc67be8e3e0153f593c509e21d219c5060"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51f4b32f793812714fd5307222a7f77e739b9bc566dc94a18126aba3b92b98a3"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba99d8077424501b9616b43a2d208095746fb1284fc5ba490139651f971d39d9"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4bfc2b16e3ba8850e0e262467275dd4d62f0d045e0e9eda2bc65078c0110a11f"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8c2c19dae8a3eb0ea45a8448356ed561be843b13cbc34b840922ddf565498c1c"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:60080bb3d8617d96f0fb7e19796384cc2467447ef1c491694850ebd3670bc457"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b77e27b79448e34c2c51c09836033056a0547aa360c45eeeb67803da7b0eedaf"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:518440c991f514331f4850a63560321f833979d145d7d81186dbe2f19e27ae3d"}, + {file = "regex-2023.12.25-cp312-cp312-win32.whl", hash = "sha256:e2610e9406d3b0073636a3a2e80db05a02f0c3169b5632022b4e81c0364bcda5"}, + {file = "regex-2023.12.25-cp312-cp312-win_amd64.whl", hash = "sha256:cc37b9aeebab425f11f27e5e9e6cf580be7206c6582a64467a14dda211abc232"}, + {file = "regex-2023.12.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:da695d75ac97cb1cd725adac136d25ca687da4536154cdc2815f576e4da11c69"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d126361607b33c4eb7b36debc173bf25d7805847346dd4d99b5499e1fef52bc7"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4719bb05094d7d8563a450cf8738d2e1061420f79cfcc1fa7f0a44744c4d8f73"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5dd58946bce44b53b06d94aa95560d0b243eb2fe64227cba50017a8d8b3cd3e2"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22a86d9fff2009302c440b9d799ef2fe322416d2d58fc124b926aa89365ec482"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2aae8101919e8aa05ecfe6322b278f41ce2994c4a430303c4cd163fef746e04f"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e692296c4cc2873967771345a876bcfc1c547e8dd695c6b89342488b0ea55cd8"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:263ef5cc10979837f243950637fffb06e8daed7f1ac1e39d5910fd29929e489a"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d6f7e255e5fa94642a0724e35406e6cb7001c09d476ab5fce002f652b36d0c39"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:88ad44e220e22b63b0f8f81f007e8abbb92874d8ced66f32571ef8beb0643b2b"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:3a17d3ede18f9cedcbe23d2daa8a2cd6f59fe2bf082c567e43083bba3fb00347"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d15b274f9e15b1a0b7a45d2ac86d1f634d983ca40d6b886721626c47a400bf39"}, + {file = "regex-2023.12.25-cp37-cp37m-win32.whl", hash = "sha256:ed19b3a05ae0c97dd8f75a5d8f21f7723a8c33bbc555da6bbe1f96c470139d3c"}, + {file = "regex-2023.12.25-cp37-cp37m-win_amd64.whl", hash = "sha256:a6d1047952c0b8104a1d371f88f4ab62e6275567d4458c1e26e9627ad489b445"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b43523d7bc2abd757119dbfb38af91b5735eea45537ec6ec3a5ec3f9562a1c53"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:efb2d82f33b2212898f1659fb1c2e9ac30493ac41e4d53123da374c3b5541e64"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b7fca9205b59c1a3d5031f7e64ed627a1074730a51c2a80e97653e3e9fa0d415"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086dd15e9435b393ae06f96ab69ab2d333f5d65cbe65ca5a3ef0ec9564dfe770"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e81469f7d01efed9b53740aedd26085f20d49da65f9c1f41e822a33992cb1590"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:34e4af5b27232f68042aa40a91c3b9bb4da0eeb31b7632e0091afc4310afe6cb"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9852b76ab558e45b20bf1893b59af64a28bd3820b0c2efc80e0a70a4a3ea51c1"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff100b203092af77d1a5a7abe085b3506b7eaaf9abf65b73b7d6905b6cb76988"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cc038b2d8b1470364b1888a98fd22d616fba2b6309c5b5f181ad4483e0017861"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:094ba386bb5c01e54e14434d4caabf6583334090865b23ef58e0424a6286d3dc"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5cd05d0f57846d8ba4b71d9c00f6f37d6b97d5e5ef8b3c3840426a475c8f70f4"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:9aa1a67bbf0f957bbe096375887b2505f5d8ae16bf04488e8b0f334c36e31360"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:98a2636994f943b871786c9e82bfe7883ecdaba2ef5df54e1450fa9869d1f756"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37f8e93a81fc5e5bd8db7e10e62dc64261bcd88f8d7e6640aaebe9bc180d9ce2"}, + {file = "regex-2023.12.25-cp38-cp38-win32.whl", hash = "sha256:d78bd484930c1da2b9679290a41cdb25cc127d783768a0369d6b449e72f88beb"}, + {file = "regex-2023.12.25-cp38-cp38-win_amd64.whl", hash = "sha256:b521dcecebc5b978b447f0f69b5b7f3840eac454862270406a39837ffae4e697"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f7bc09bc9c29ebead055bcba136a67378f03d66bf359e87d0f7c759d6d4ffa31"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e14b73607d6231f3cc4622809c196b540a6a44e903bcfad940779c80dffa7be7"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9eda5f7a50141291beda3edd00abc2d4a5b16c29c92daf8d5bd76934150f3edc"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc6bb9aa69aacf0f6032c307da718f61a40cf970849e471254e0e91c56ffca95"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:298dc6354d414bc921581be85695d18912bea163a8b23cac9a2562bbcd5088b1"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f4e475a80ecbd15896a976aa0b386c5525d0ed34d5c600b6d3ebac0a67c7ddf"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:531ac6cf22b53e0696f8e1d56ce2396311254eb806111ddd3922c9d937151dae"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22f3470f7524b6da61e2020672df2f3063676aff444db1daa283c2ea4ed259d6"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:89723d2112697feaa320c9d351e5f5e7b841e83f8b143dba8e2d2b5f04e10923"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0ecf44ddf9171cd7566ef1768047f6e66975788258b1c6c6ca78098b95cf9a3d"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:905466ad1702ed4acfd67a902af50b8db1feeb9781436372261808df7a2a7bca"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:4558410b7a5607a645e9804a3e9dd509af12fb72b9825b13791a37cd417d73a5"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:7e316026cc1095f2a3e8cc012822c99f413b702eaa2ca5408a513609488cb62f"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3b1de218d5375cd6ac4b5493e0b9f3df2be331e86520f23382f216c137913d20"}, + {file = "regex-2023.12.25-cp39-cp39-win32.whl", hash = "sha256:11a963f8e25ab5c61348d090bf1b07f1953929c13bd2309a0662e9ff680763c9"}, + {file = "regex-2023.12.25-cp39-cp39-win_amd64.whl", hash = "sha256:e693e233ac92ba83a87024e1d32b5f9ab15ca55ddd916d878146f4e3406b5c91"}, + {file = "regex-2023.12.25.tar.gz", hash = "sha256:29171aa128da69afdf4bde412d5bedc335f2ca8fcfe4489038577d05f16181e5"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +description = "A utility belt for advanced users of python-requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, + {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, +] + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +description = "A pure python RFC3339 validator" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"}, + {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "rfc3986-validator" +version = "0.1.1" +description = "Pure python rfc3986 validator" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9"}, + {file = "rfc3986_validator-0.1.1.tar.gz", hash = "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055"}, +] + +[[package]] +name = "rpds-py" +version = "0.17.1" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.17.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4128980a14ed805e1b91a7ed551250282a8ddf8201a4e9f8f5b7e6225f54170d"}, + {file = "rpds_py-0.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ff1dcb8e8bc2261a088821b2595ef031c91d499a0c1b031c152d43fe0a6ecec8"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d65e6b4f1443048eb7e833c2accb4fa7ee67cc7d54f31b4f0555b474758bee55"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a71169d505af63bb4d20d23a8fbd4c6ce272e7bce6cc31f617152aa784436f29"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:436474f17733c7dca0fbf096d36ae65277e8645039df12a0fa52445ca494729d"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10162fe3f5f47c37ebf6d8ff5a2368508fe22007e3077bf25b9c7d803454d921"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:720215373a280f78a1814becb1312d4e4d1077b1202a56d2b0815e95ccb99ce9"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:70fcc6c2906cfa5c6a552ba7ae2ce64b6c32f437d8f3f8eea49925b278a61453"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e5a8200e65aaac342a791272c564dffcf1281abd635d304d6c4e6b495f29dc"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:99f567dae93e10be2daaa896e07513dd4bf9c2ecf0576e0533ac36ba3b1d5394"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24e4900a6643f87058a27320f81336d527ccfe503984528edde4bb660c8c8d59"}, + {file = "rpds_py-0.17.1-cp310-none-win32.whl", hash = "sha256:0bfb09bf41fe7c51413f563373e5f537eaa653d7adc4830399d4e9bdc199959d"}, + {file = "rpds_py-0.17.1-cp310-none-win_amd64.whl", hash = "sha256:20de7b7179e2031a04042e85dc463a93a82bc177eeba5ddd13ff746325558aa6"}, + {file = "rpds_py-0.17.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:65dcf105c1943cba45d19207ef51b8bc46d232a381e94dd38719d52d3980015b"}, + {file = "rpds_py-0.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:01f58a7306b64e0a4fe042047dd2b7d411ee82e54240284bab63e325762c1147"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:071bc28c589b86bc6351a339114fb7a029f5cddbaca34103aa573eba7b482382"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae35e8e6801c5ab071b992cb2da958eee76340e6926ec693b5ff7d6381441745"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149c5cd24f729e3567b56e1795f74577aa3126c14c11e457bec1b1c90d212e38"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e796051f2070f47230c745d0a77a91088fbee2cc0502e9b796b9c6471983718c"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e820ee1004327609b28db8307acc27f5f2e9a0b185b2064c5f23e815f248f8"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1957a2ab607f9added64478a6982742eb29f109d89d065fa44e01691a20fc20a"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8587fd64c2a91c33cdc39d0cebdaf30e79491cc029a37fcd458ba863f8815383"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4dc889a9d8a34758d0fcc9ac86adb97bab3fb7f0c4d29794357eb147536483fd"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2953937f83820376b5979318840f3ee47477d94c17b940fe31d9458d79ae7eea"}, + {file = "rpds_py-0.17.1-cp311-none-win32.whl", hash = "sha256:1bfcad3109c1e5ba3cbe2f421614e70439f72897515a96c462ea657261b96518"}, + {file = "rpds_py-0.17.1-cp311-none-win_amd64.whl", hash = "sha256:99da0a4686ada4ed0f778120a0ea8d066de1a0a92ab0d13ae68492a437db78bf"}, + {file = "rpds_py-0.17.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1dc29db3900cb1bb40353772417800f29c3d078dbc8024fd64655a04ee3c4bdf"}, + {file = "rpds_py-0.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82ada4a8ed9e82e443fcef87e22a3eed3654dd3adf6e3b3a0deb70f03e86142a"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d36b2b59e8cc6e576f8f7b671e32f2ff43153f0ad6d0201250a7c07f25d570e"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3677fcca7fb728c86a78660c7fb1b07b69b281964673f486ae72860e13f512ad"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:516fb8c77805159e97a689e2f1c80655c7658f5af601c34ffdb916605598cda2"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df3b6f45ba4515632c5064e35ca7f31d51d13d1479673185ba8f9fefbbed58b9"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a967dd6afda7715d911c25a6ba1517975acd8d1092b2f326718725461a3d33f9"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dbbb95e6fc91ea3102505d111b327004d1c4ce98d56a4a02e82cd451f9f57140"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02866e060219514940342a1f84303a1ef7a1dad0ac311792fbbe19b521b489d2"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2528ff96d09f12e638695f3a2e0c609c7b84c6df7c5ae9bfeb9252b6fa686253"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd345a13ce06e94c753dab52f8e71e5252aec1e4f8022d24d56decd31e1b9b23"}, + {file = "rpds_py-0.17.1-cp312-none-win32.whl", hash = "sha256:2a792b2e1d3038daa83fa474d559acfd6dc1e3650ee93b2662ddc17dbff20ad1"}, + {file = "rpds_py-0.17.1-cp312-none-win_amd64.whl", hash = "sha256:292f7344a3301802e7c25c53792fae7d1593cb0e50964e7bcdcc5cf533d634e3"}, + {file = "rpds_py-0.17.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:8ffe53e1d8ef2520ebcf0c9fec15bb721da59e8ef283b6ff3079613b1e30513d"}, + {file = "rpds_py-0.17.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4341bd7579611cf50e7b20bb8c2e23512a3dc79de987a1f411cb458ab670eb90"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4eb548daf4836e3b2c662033bfbfc551db58d30fd8fe660314f86bf8510b93"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b686f25377f9c006acbac63f61614416a6317133ab7fafe5de5f7dc8a06d42eb"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e21b76075c01d65d0f0f34302b5a7457d95721d5e0667aea65e5bb3ab415c25"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b86b21b348f7e5485fae740d845c65a880f5d1eda1e063bc59bef92d1f7d0c55"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f175e95a197f6a4059b50757a3dca33b32b61691bdbd22c29e8a8d21d3914cae"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1701fc54460ae2e5efc1dd6350eafd7a760f516df8dbe51d4a1c79d69472fbd4"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:9051e3d2af8f55b42061603e29e744724cb5f65b128a491446cc029b3e2ea896"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7450dbd659fed6dd41d1a7d47ed767e893ba402af8ae664c157c255ec6067fde"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5a024fa96d541fd7edaa0e9d904601c6445e95a729a2900c5aec6555fe921ed6"}, + {file = "rpds_py-0.17.1-cp38-none-win32.whl", hash = "sha256:da1ead63368c04a9bded7904757dfcae01eba0e0f9bc41d3d7f57ebf1c04015a"}, + {file = "rpds_py-0.17.1-cp38-none-win_amd64.whl", hash = "sha256:841320e1841bb53fada91c9725e766bb25009cfd4144e92298db296fb6c894fb"}, + {file = "rpds_py-0.17.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:f6c43b6f97209e370124baf2bf40bb1e8edc25311a158867eb1c3a5d449ebc7a"}, + {file = "rpds_py-0.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7d63ec01fe7c76c2dbb7e972fece45acbb8836e72682bde138e7e039906e2c"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81038ff87a4e04c22e1d81f947c6ac46f122e0c80460b9006e6517c4d842a6ec"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:810685321f4a304b2b55577c915bece4c4a06dfe38f6e62d9cc1d6ca8ee86b99"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:25f071737dae674ca8937a73d0f43f5a52e92c2d178330b4c0bb6ab05586ffa6"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa5bfb13f1e89151ade0eb812f7b0d7a4d643406caaad65ce1cbabe0a66d695f"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfe07308b311a8293a0d5ef4e61411c5c20f682db6b5e73de6c7c8824272c256"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a000133a90eea274a6f28adc3084643263b1e7c1a5a66eb0a0a7a36aa757ed74"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d0e8a6434a3fbf77d11448c9c25b2f25244226cfbec1a5159947cac5b8c5fa4"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:efa767c220d94aa4ac3a6dd3aeb986e9f229eaf5bce92d8b1b3018d06bed3772"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:dbc56680ecf585a384fbd93cd42bc82668b77cb525343170a2d86dafaed2a84b"}, + {file = "rpds_py-0.17.1-cp39-none-win32.whl", hash = "sha256:270987bc22e7e5a962b1094953ae901395e8c1e1e83ad016c5cfcfff75a15a3f"}, + {file = "rpds_py-0.17.1-cp39-none-win_amd64.whl", hash = "sha256:2a7b2f2f56a16a6d62e55354dd329d929560442bd92e87397b7a9586a32e3e76"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a3264e3e858de4fc601741498215835ff324ff2482fd4e4af61b46512dd7fc83"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f2f3b28b40fddcb6c1f1f6c88c6f3769cd933fa493ceb79da45968a21dccc920"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9584f8f52010295a4a417221861df9bea4c72d9632562b6e59b3c7b87a1522b7"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c64602e8be701c6cfe42064b71c84ce62ce66ddc6422c15463fd8127db3d8066"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:060f412230d5f19fc8c8b75f315931b408d8ebf56aec33ef4168d1b9e54200b1"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9412abdf0ba70faa6e2ee6c0cc62a8defb772e78860cef419865917d86c7342"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9737bdaa0ad33d34c0efc718741abaafce62fadae72c8b251df9b0c823c63b22"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9f0e4dc0f17dcea4ab9d13ac5c666b6b5337042b4d8f27e01b70fae41dd65c57"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1db228102ab9d1ff4c64148c96320d0be7044fa28bd865a9ce628ce98da5973d"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:d8bbd8e56f3ba25a7d0cf980fc42b34028848a53a0e36c9918550e0280b9d0b6"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:be22ae34d68544df293152b7e50895ba70d2a833ad9566932d750d3625918b82"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bf046179d011e6114daf12a534d874958b039342b347348a78b7cdf0dd9d6041"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:1a746a6d49665058a5896000e8d9d2f1a6acba8a03b389c1e4c06e11e0b7f40d"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0b8bf5b8db49d8fd40f54772a1dcf262e8be0ad2ab0206b5a2ec109c176c0a4"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f7f4cb1f173385e8a39c29510dd11a78bf44e360fb75610594973f5ea141028b"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7fbd70cb8b54fe745301921b0816c08b6d917593429dfc437fd024b5ba713c58"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bdf1303df671179eaf2cb41e8515a07fc78d9d00f111eadbe3e14262f59c3d0"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad059a4bd14c45776600d223ec194e77db6c20255578bb5bcdd7c18fd169361"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3664d126d3388a887db44c2e293f87d500c4184ec43d5d14d2d2babdb4c64cad"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:698ea95a60c8b16b58be9d854c9f993c639f5c214cf9ba782eca53a8789d6b19"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:c3d2010656999b63e628a3c694f23020322b4178c450dc478558a2b6ef3cb9bb"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:938eab7323a736533f015e6069a7d53ef2dcc841e4e533b782c2bfb9fb12d84b"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e626b365293a2142a62b9a614e1f8e331b28f3ca57b9f05ebbf4cf2a0f0bdc5"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:380e0df2e9d5d5d339803cfc6d183a5442ad7ab3c63c2a0982e8c824566c5ccc"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b760a56e080a826c2e5af09002c1a037382ed21d03134eb6294812dda268c811"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5576ee2f3a309d2bb403ec292d5958ce03953b0e57a11d224c1f134feaf8c40f"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3c3461ebb4c4f1bbc70b15d20b565759f97a5aaf13af811fcefc892e9197ba"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:637b802f3f069a64436d432117a7e58fab414b4e27a7e81049817ae94de45d8d"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffee088ea9b593cc6160518ba9bd319b5475e5f3e578e4552d63818773c6f56a"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3ac732390d529d8469b831949c78085b034bff67f584559340008d0f6041a049"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:93432e747fb07fa567ad9cc7aaadd6e29710e515aabf939dfbed8046041346c6"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:7b7d9ca34542099b4e185b3c2a2b2eda2e318a7dbde0b0d83357a6d4421b5296"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:0387ce69ba06e43df54e43968090f3626e231e4bc9150e4c3246947567695f68"}, + {file = "rpds_py-0.17.1.tar.gz", hash = "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7"}, +] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "scikit-learn" +version = "1.4.0" +description = "A set of python modules for machine learning and data mining" +optional = false +python-versions = ">=3.9" +files = [ + {file = "scikit-learn-1.4.0.tar.gz", hash = "sha256:d4373c984eba20e393216edd51a3e3eede56cbe93d4247516d205643c3b93121"}, + {file = "scikit_learn-1.4.0-1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fce93a7473e2f4ee4cc280210968288d6a7d7ad8dc6fa7bb7892145e407085f9"}, + {file = "scikit_learn-1.4.0-1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d77df3d1e15fc37a9329999979fa7868ba8655dbab21fe97fc7ddabac9e08cc7"}, + {file = "scikit_learn-1.4.0-1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2404659fedec40eeafa310cd14d613e564d13dbf8f3c752d31c095195ec05de6"}, + {file = "scikit_learn-1.4.0-1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e98632da8f6410e6fb6bf66937712c949b4010600ccd3f22a5388a83e610cc3c"}, + {file = "scikit_learn-1.4.0-1-cp310-cp310-win_amd64.whl", hash = "sha256:11b3b140f70fbc9f6a08884631ae8dd60a4bb2d7d6d1de92738ea42b740d8992"}, + {file = "scikit_learn-1.4.0-1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8341eabdc754d5ab91641a7763243845e96b6d68e03e472531e88a4f1b09f21"}, + {file = "scikit_learn-1.4.0-1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:d1f6bce875ac2bb6b52514f67c185c564ccd299a05b65b7bab091a4c13dde12d"}, + {file = "scikit_learn-1.4.0-1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c408b46b2fd61952d519ea1af2f8f0a7a703e1433923ab1704c4131520b2083b"}, + {file = "scikit_learn-1.4.0-1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b465dd1dcd237b7b1dcd1a9048ccbf70a98c659474324fa708464c3a2533fad"}, + {file = "scikit_learn-1.4.0-1-cp311-cp311-win_amd64.whl", hash = "sha256:0db8e22c42f7980fe5eb22069b1f84c48966f3e0d23a01afde5999e3987a2501"}, + {file = "scikit_learn-1.4.0-1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e7eef6ea2ed289af40e88c0be9f7704ca8b5de18508a06897c3fe21e0905efdf"}, + {file = "scikit_learn-1.4.0-1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:349669b01435bc4dbf25c6410b0892073befdaec52637d1a1d1ff53865dc8db3"}, + {file = "scikit_learn-1.4.0-1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d439c584e58434d0350701bd33f6c10b309e851fccaf41c121aed55f6851d8cf"}, + {file = "scikit_learn-1.4.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0e2427d9ef46477625ab9b55c1882844fe6fc500f418c3f8e650200182457bc"}, + {file = "scikit_learn-1.4.0-1-cp312-cp312-win_amd64.whl", hash = "sha256:d3d75343940e7bf9b85c830c93d34039fa015eeb341c5c0b4cd7a90dadfe00d4"}, + {file = "scikit_learn-1.4.0-1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:76986d22e884ab062b1beecdd92379656e9d3789ecc1f9870923c178de55f9fe"}, + {file = "scikit_learn-1.4.0-1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:e22446ad89f1cb7657f0d849dcdc345b48e2d10afa3daf2925fdb740f85b714c"}, + {file = "scikit_learn-1.4.0-1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74812c9eabb265be69d738a8ea8d4884917a59637fcbf88a5f0e9020498bc6b3"}, + {file = "scikit_learn-1.4.0-1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad2a63e0dd386b92da3270887a29b308af4d7c750d8c4995dfd9a4798691bcc"}, + {file = "scikit_learn-1.4.0-1-cp39-cp39-win_amd64.whl", hash = "sha256:53b9e29177897c37e2ff9d4ba6ca12fdb156e22523e463db05def303f5c72b5c"}, + {file = "scikit_learn-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cb8f044a8f5962613ce1feb4351d66f8d784bd072d36393582f351859b065f7d"}, + {file = "scikit_learn-1.4.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:a6372c90bbf302387792108379f1ec77719c1618d88496d0df30cb8e370b4661"}, + {file = "scikit_learn-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:785ce3c352bf697adfda357c3922c94517a9376002971bc5ea50896144bc8916"}, + {file = "scikit_learn-1.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0aba2a20d89936d6e72d95d05e3bf1db55bca5c5920926ad7b92c34f5e7d3bbe"}, + {file = "scikit_learn-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:2bac5d56b992f8f06816f2cd321eb86071c6f6d44bb4b1cb3d626525820d754b"}, + {file = "scikit_learn-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:27ae4b0f1b2c77107c096a7e05b33458354107b47775428d1f11b23e30a73e8a"}, + {file = "scikit_learn-1.4.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5c5c62ffb52c3ffb755eb21fa74cc2cbf2c521bd53f5c04eaa10011dbecf5f80"}, + {file = "scikit_learn-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f0d2018ac6fa055dab65fe8a485967990d33c672d55bc254c56c35287b02fab"}, + {file = "scikit_learn-1.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91a8918c415c4b4bf1d60c38d32958849a9191c2428ab35d30b78354085c7c7a"}, + {file = "scikit_learn-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:80a21de63275f8bcd7877b3e781679d2ff1eddfed515a599f95b2502a3283d42"}, + {file = "scikit_learn-1.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0f33bbafb310c26b81c4d41ecaebdbc1f63498a3f13461d50ed9a2e8f24d28e4"}, + {file = "scikit_learn-1.4.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:8b6ac1442ec714b4911e5aef8afd82c691b5c88b525ea58299d455acc4e8dcec"}, + {file = "scikit_learn-1.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05fc5915b716c6cc60a438c250108e9a9445b522975ed37e416d5ea4f9a63381"}, + {file = "scikit_learn-1.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:842b7d6989f3c574685e18da6f91223eb32301d0f93903dd399894250835a6f7"}, + {file = "scikit_learn-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:88bcb586fdff865372df1bc6be88bb7e6f9e0aa080dab9f54f5cac7eca8e2b6b"}, + {file = "scikit_learn-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f77674647dd31f56cb12ed13ed25b6ed43a056fffef051715022d2ebffd7a7d1"}, + {file = "scikit_learn-1.4.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:833999872e2920ce00f3a50839946bdac7539454e200eb6db54898a41f4bfd43"}, + {file = "scikit_learn-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:970ec697accaef10fb4f51763f3a7b1250f9f0553cf05514d0e94905322a0172"}, + {file = "scikit_learn-1.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:923d778f378ebacca2c672ab1740e5a413e437fb45ab45ab02578f8b689e5d43"}, + {file = "scikit_learn-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:1d041bc95006b545b59e458399e3175ab11ca7a03dc9a74a573ac891f5df1489"}, +] + +[package.dependencies] +joblib = ">=1.2.0" +numpy = ">=1.19.5" +scipy = ">=1.6.0" +threadpoolctl = ">=2.0.0" + +[package.extras] +benchmark = ["matplotlib (>=3.3.4)", "memory-profiler (>=0.57.0)", "pandas (>=1.1.5)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory-profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=6.0.0)", "sphinx-copybutton (>=0.5.2)", "sphinx-gallery (>=0.15.0)", "sphinx-prompt (>=1.3.0)", "sphinxext-opengraph (>=0.4.2)"] +examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] +tests = ["black (>=23.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.3)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.19.12)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.0.272)", "scikit-image (>=0.17.2)"] + +[[package]] +name = "scipy" +version = "1.12.0" +description = "Fundamental algorithms for scientific computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "scipy-1.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:78e4402e140879387187f7f25d91cc592b3501a2e51dfb320f48dfb73565f10b"}, + {file = "scipy-1.12.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f5f00ebaf8de24d14b8449981a2842d404152774c1a1d880c901bf454cb8e2a1"}, + {file = "scipy-1.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e53958531a7c695ff66c2e7bb7b79560ffdc562e2051644c5576c39ff8efb563"}, + {file = "scipy-1.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e32847e08da8d895ce09d108a494d9eb78974cf6de23063f93306a3e419960c"}, + {file = "scipy-1.12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4c1020cad92772bf44b8e4cdabc1df5d87376cb219742549ef69fc9fd86282dd"}, + {file = "scipy-1.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:75ea2a144096b5e39402e2ff53a36fecfd3b960d786b7efd3c180e29c39e53f2"}, + {file = "scipy-1.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:408c68423f9de16cb9e602528be4ce0d6312b05001f3de61fe9ec8b1263cad08"}, + {file = "scipy-1.12.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5adfad5dbf0163397beb4aca679187d24aec085343755fcdbdeb32b3679f254c"}, + {file = "scipy-1.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3003652496f6e7c387b1cf63f4bb720951cfa18907e998ea551e6de51a04467"}, + {file = "scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b8066bce124ee5531d12a74b617d9ac0ea59245246410e19bca549656d9a40a"}, + {file = "scipy-1.12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8bee4993817e204d761dba10dbab0774ba5a8612e57e81319ea04d84945375ba"}, + {file = "scipy-1.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:a24024d45ce9a675c1fb8494e8e5244efea1c7a09c60beb1eeb80373d0fecc70"}, + {file = "scipy-1.12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e7e76cc48638228212c747ada851ef355c2bb5e7f939e10952bc504c11f4e372"}, + {file = "scipy-1.12.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f7ce148dffcd64ade37b2df9315541f9adad6efcaa86866ee7dd5db0c8f041c3"}, + {file = "scipy-1.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c39f92041f490422924dfdb782527a4abddf4707616e07b021de33467f917bc"}, + {file = "scipy-1.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7ebda398f86e56178c2fa94cad15bf457a218a54a35c2a7b4490b9f9cb2676c"}, + {file = "scipy-1.12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:95e5c750d55cf518c398a8240571b0e0782c2d5a703250872f36eaf737751338"}, + {file = "scipy-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:e646d8571804a304e1da01040d21577685ce8e2db08ac58e543eaca063453e1c"}, + {file = "scipy-1.12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:913d6e7956c3a671de3b05ccb66b11bc293f56bfdef040583a7221d9e22a2e35"}, + {file = "scipy-1.12.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba1b0c7256ad75401c73e4b3cf09d1f176e9bd4248f0d3112170fb2ec4db067"}, + {file = "scipy-1.12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:730badef9b827b368f351eacae2e82da414e13cf8bd5051b4bdfd720271a5371"}, + {file = "scipy-1.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6546dc2c11a9df6926afcbdd8a3edec28566e4e785b915e849348c6dd9f3f490"}, + {file = "scipy-1.12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:196ebad3a4882081f62a5bf4aeb7326aa34b110e533aab23e4374fcccb0890dc"}, + {file = "scipy-1.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:b360f1b6b2f742781299514e99ff560d1fe9bd1bff2712894b52abe528d1fd1e"}, + {file = "scipy-1.12.0.tar.gz", hash = "sha256:4bf5abab8a36d20193c698b0f1fc282c1d083c94723902c447e5d2f1780936a3"}, +] + +[package.dependencies] +numpy = ">=1.22.4,<1.29.0" + +[package.extras] +dev = ["click", "cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] +doc = ["jupytext", "matplotlib (>2)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] +test = ["asv", "gmpy2", "hypothesis", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + +[[package]] +name = "send2trash" +version = "1.8.2" +description = "Send file to trash natively under Mac OS X, Windows and Linux" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ + {file = "Send2Trash-1.8.2-py3-none-any.whl", hash = "sha256:a384719d99c07ce1eefd6905d2decb6f8b7ed054025bb0e618919f945de4f679"}, + {file = "Send2Trash-1.8.2.tar.gz", hash = "sha256:c132d59fa44b9ca2b1699af5c86f57ce9f4c5eb56629d5d55fbb7a35f84e2312"}, +] + +[package.extras] +nativelib = ["pyobjc-framework-Cocoa", "pywin32"] +objc = ["pyobjc-framework-Cocoa"] +win32 = ["pywin32"] + +[[package]] +name = "sentry-sdk" +version = "1.28.1" +description = "Python client for Sentry (https://sentry.io)" +optional = false +python-versions = "*" +files = [ + {file = "sentry-sdk-1.28.1.tar.gz", hash = "sha256:dcd88c68aa64dae715311b5ede6502fd684f70d00a7cd4858118f0ba3153a3ae"}, + {file = "sentry_sdk-1.28.1-py2.py3-none-any.whl", hash = "sha256:6bdb25bd9092478d3a817cb0d01fa99e296aea34d404eac3ca0037faa5c2aa0a"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""} + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +arq = ["arq (>=0.23)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +chalice = ["chalice (>=1.16.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +fastapi = ["fastapi (>=0.79.0)"] +flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"] +grpcio = ["grpcio (>=1.21.1)"] +httpx = ["httpx (>=0.16.0)"] +huey = ["huey (>=2)"] +loguru = ["loguru (>=0.5)"] +opentelemetry = ["opentelemetry-distro (>=0.35b0)"] +pure-eval = ["asttokens", "executing", "pure-eval"] +pymongo = ["pymongo (>=3.1)"] +pyspark = ["pyspark (>=2.4.4)"] +quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +starlette = ["starlette (>=0.19.1)"] +starlite = ["starlite (>=1.48)"] +tornado = ["tornado (>=5)"] + +[[package]] +name = "setproctitle" +version = "1.3.3" +description = "A Python module to customize the process title" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setproctitle-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:897a73208da48db41e687225f355ce993167079eda1260ba5e13c4e53be7f754"}, + {file = "setproctitle-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c331e91a14ba4076f88c29c777ad6b58639530ed5b24b5564b5ed2fd7a95452"}, + {file = "setproctitle-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbbd6c7de0771c84b4aa30e70b409565eb1fc13627a723ca6be774ed6b9d9fa3"}, + {file = "setproctitle-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c05ac48ef16ee013b8a326c63e4610e2430dbec037ec5c5b58fcced550382b74"}, + {file = "setproctitle-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1342f4fdb37f89d3e3c1c0a59d6ddbedbde838fff5c51178a7982993d238fe4f"}, + {file = "setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc74e84fdfa96821580fb5e9c0b0777c1c4779434ce16d3d62a9c4d8c710df39"}, + {file = "setproctitle-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9617b676b95adb412bb69645d5b077d664b6882bb0d37bfdafbbb1b999568d85"}, + {file = "setproctitle-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6a249415f5bb88b5e9e8c4db47f609e0bf0e20a75e8d744ea787f3092ba1f2d0"}, + {file = "setproctitle-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:38da436a0aaace9add67b999eb6abe4b84397edf4a78ec28f264e5b4c9d53cd5"}, + {file = "setproctitle-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:da0d57edd4c95bf221b2ebbaa061e65b1788f1544977288bdf95831b6e44e44d"}, + {file = "setproctitle-1.3.3-cp310-cp310-win32.whl", hash = "sha256:a1fcac43918b836ace25f69b1dca8c9395253ad8152b625064415b1d2f9be4fb"}, + {file = "setproctitle-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:200620c3b15388d7f3f97e0ae26599c0c378fdf07ae9ac5a13616e933cbd2086"}, + {file = "setproctitle-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:334f7ed39895d692f753a443102dd5fed180c571eb6a48b2a5b7f5b3564908c8"}, + {file = "setproctitle-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:950f6476d56ff7817a8fed4ab207727fc5260af83481b2a4b125f32844df513a"}, + {file = "setproctitle-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:195c961f54a09eb2acabbfc90c413955cf16c6e2f8caa2adbf2237d1019c7dd8"}, + {file = "setproctitle-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f05e66746bf9fe6a3397ec246fe481096664a9c97eb3fea6004735a4daf867fd"}, + {file = "setproctitle-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b5901a31012a40ec913265b64e48c2a4059278d9f4e6be628441482dd13fb8b5"}, + {file = "setproctitle-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64286f8a995f2cd934082b398fc63fca7d5ffe31f0e27e75b3ca6b4efda4e353"}, + {file = "setproctitle-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:184239903bbc6b813b1a8fc86394dc6ca7d20e2ebe6f69f716bec301e4b0199d"}, + {file = "setproctitle-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:664698ae0013f986118064b6676d7dcd28fefd0d7d5a5ae9497cbc10cba48fa5"}, + {file = "setproctitle-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e5119a211c2e98ff18b9908ba62a3bd0e3fabb02a29277a7232a6fb4b2560aa0"}, + {file = "setproctitle-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:417de6b2e214e837827067048f61841f5d7fc27926f2e43954567094051aff18"}, + {file = "setproctitle-1.3.3-cp311-cp311-win32.whl", hash = "sha256:6a143b31d758296dc2f440175f6c8e0b5301ced3b0f477b84ca43cdcf7f2f476"}, + {file = "setproctitle-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a680d62c399fa4b44899094027ec9a1bdaf6f31c650e44183b50d4c4d0ccc085"}, + {file = "setproctitle-1.3.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d4460795a8a7a391e3567b902ec5bdf6c60a47d791c3b1d27080fc203d11c9dc"}, + {file = "setproctitle-1.3.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bdfd7254745bb737ca1384dee57e6523651892f0ea2a7344490e9caefcc35e64"}, + {file = "setproctitle-1.3.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:477d3da48e216d7fc04bddab67b0dcde633e19f484a146fd2a34bb0e9dbb4a1e"}, + {file = "setproctitle-1.3.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ab2900d111e93aff5df9fddc64cf51ca4ef2c9f98702ce26524f1acc5a786ae7"}, + {file = "setproctitle-1.3.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:088b9efc62d5aa5d6edf6cba1cf0c81f4488b5ce1c0342a8b67ae39d64001120"}, + {file = "setproctitle-1.3.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6d50252377db62d6a0bb82cc898089916457f2db2041e1d03ce7fadd4a07381"}, + {file = "setproctitle-1.3.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:87e668f9561fd3a457ba189edfc9e37709261287b52293c115ae3487a24b92f6"}, + {file = "setproctitle-1.3.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:287490eb90e7a0ddd22e74c89a92cc922389daa95babc833c08cf80c84c4df0a"}, + {file = "setproctitle-1.3.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:4fe1c49486109f72d502f8be569972e27f385fe632bd8895f4730df3c87d5ac8"}, + {file = "setproctitle-1.3.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4a6ba2494a6449b1f477bd3e67935c2b7b0274f2f6dcd0f7c6aceae10c6c6ba3"}, + {file = "setproctitle-1.3.3-cp312-cp312-win32.whl", hash = "sha256:2df2b67e4b1d7498632e18c56722851ba4db5d6a0c91aaf0fd395111e51cdcf4"}, + {file = "setproctitle-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:f38d48abc121263f3b62943f84cbaede05749047e428409c2c199664feb6abc7"}, + {file = "setproctitle-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:816330675e3504ae4d9a2185c46b573105d2310c20b19ea2b4596a9460a4f674"}, + {file = "setproctitle-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68f960bc22d8d8e4ac886d1e2e21ccbd283adcf3c43136161c1ba0fa509088e0"}, + {file = "setproctitle-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00e6e7adff74796ef12753ff399491b8827f84f6c77659d71bd0b35870a17d8f"}, + {file = "setproctitle-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53bc0d2358507596c22b02db079618451f3bd720755d88e3cccd840bafb4c41c"}, + {file = "setproctitle-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad6d20f9541f5f6ac63df553b6d7a04f313947f550eab6a61aa758b45f0d5657"}, + {file = "setproctitle-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c1c84beab776b0becaa368254801e57692ed749d935469ac10e2b9b825dbdd8e"}, + {file = "setproctitle-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:507e8dc2891021350eaea40a44ddd887c9f006e6b599af8d64a505c0f718f170"}, + {file = "setproctitle-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b1067647ac7aba0b44b591936118a22847bda3c507b0a42d74272256a7a798e9"}, + {file = "setproctitle-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2e71f6365744bf53714e8bd2522b3c9c1d83f52ffa6324bd7cbb4da707312cd8"}, + {file = "setproctitle-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:7f1d36a1e15a46e8ede4e953abb104fdbc0845a266ec0e99cc0492a4364f8c44"}, + {file = "setproctitle-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9a402881ec269d0cc9c354b149fc29f9ec1a1939a777f1c858cdb09c7a261df"}, + {file = "setproctitle-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ff814dea1e5c492a4980e3e7d094286077054e7ea116cbeda138819db194b2cd"}, + {file = "setproctitle-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:accb66d7b3ccb00d5cd11d8c6e07055a4568a24c95cf86109894dcc0c134cc89"}, + {file = "setproctitle-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554eae5a5b28f02705b83a230e9d163d645c9a08914c0ad921df363a07cf39b1"}, + {file = "setproctitle-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a911b26264dbe9e8066c7531c0591cfab27b464459c74385b276fe487ca91c12"}, + {file = "setproctitle-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2982efe7640c4835f7355fdb4da313ad37fb3b40f5c69069912f8048f77b28c8"}, + {file = "setproctitle-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df3f4274b80709d8bcab2f9a862973d453b308b97a0b423a501bcd93582852e3"}, + {file = "setproctitle-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:af2c67ae4c795d1674a8d3ac1988676fa306bcfa1e23fddb5e0bd5f5635309ca"}, + {file = "setproctitle-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:af4061f67fd7ec01624c5e3c21f6b7af2ef0e6bab7fbb43f209e6506c9ce0092"}, + {file = "setproctitle-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:37a62cbe16d4c6294e84670b59cf7adcc73faafe6af07f8cb9adaf1f0e775b19"}, + {file = "setproctitle-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a83ca086fbb017f0d87f240a8f9bbcf0809f3b754ee01cec928fff926542c450"}, + {file = "setproctitle-1.3.3-cp38-cp38-win32.whl", hash = "sha256:059f4ce86f8cc92e5860abfc43a1dceb21137b26a02373618d88f6b4b86ba9b2"}, + {file = "setproctitle-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ab92e51cd4a218208efee4c6d37db7368fdf182f6e7ff148fb295ecddf264287"}, + {file = "setproctitle-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c7951820b77abe03d88b114b998867c0f99da03859e5ab2623d94690848d3e45"}, + {file = "setproctitle-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc94cf128676e8fac6503b37763adb378e2b6be1249d207630f83fc325d9b11"}, + {file = "setproctitle-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f5d9027eeda64d353cf21a3ceb74bb1760bd534526c9214e19f052424b37e42"}, + {file = "setproctitle-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e4a8104db15d3462e29d9946f26bed817a5b1d7a47eabca2d9dc2b995991503"}, + {file = "setproctitle-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c32c41ace41f344d317399efff4cffb133e709cec2ef09c99e7a13e9f3b9483c"}, + {file = "setproctitle-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbf16381c7bf7f963b58fb4daaa65684e10966ee14d26f5cc90f07049bfd8c1e"}, + {file = "setproctitle-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e18b7bd0898398cc97ce2dfc83bb192a13a087ef6b2d5a8a36460311cb09e775"}, + {file = "setproctitle-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:69d565d20efe527bd8a9b92e7f299ae5e73b6c0470f3719bd66f3cd821e0d5bd"}, + {file = "setproctitle-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ddedd300cd690a3b06e7eac90ed4452348b1348635777ce23d460d913b5b63c3"}, + {file = "setproctitle-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:415bfcfd01d1fbf5cbd75004599ef167a533395955305f42220a585f64036081"}, + {file = "setproctitle-1.3.3-cp39-cp39-win32.whl", hash = "sha256:21112fcd2195d48f25760f0eafa7a76510871bbb3b750219310cf88b04456ae3"}, + {file = "setproctitle-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:5a740f05d0968a5a17da3d676ce6afefebeeeb5ce137510901bf6306ba8ee002"}, + {file = "setproctitle-1.3.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6b9e62ddb3db4b5205c0321dd69a406d8af9ee1693529d144e86bd43bcb4b6c0"}, + {file = "setproctitle-1.3.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e3b99b338598de0bd6b2643bf8c343cf5ff70db3627af3ca427a5e1a1a90dd9"}, + {file = "setproctitle-1.3.3-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ae9a02766dad331deb06855fb7a6ca15daea333b3967e214de12cfae8f0ef5"}, + {file = "setproctitle-1.3.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:200ede6fd11233085ba9b764eb055a2a191fb4ffb950c68675ac53c874c22e20"}, + {file = "setproctitle-1.3.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0d3a953c50776751e80fe755a380a64cb14d61e8762bd43041ab3f8cc436092f"}, + {file = "setproctitle-1.3.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5e08e232b78ba3ac6bc0d23ce9e2bee8fad2be391b7e2da834fc9a45129eb87"}, + {file = "setproctitle-1.3.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1da82c3e11284da4fcbf54957dafbf0655d2389cd3d54e4eaba636faf6d117a"}, + {file = "setproctitle-1.3.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:aeaa71fb9568ebe9b911ddb490c644fbd2006e8c940f21cb9a1e9425bd709574"}, + {file = "setproctitle-1.3.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:59335d000c6250c35989394661eb6287187854e94ac79ea22315469ee4f4c244"}, + {file = "setproctitle-1.3.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3ba57029c9c50ecaf0c92bb127224cc2ea9fda057b5d99d3f348c9ec2855ad3"}, + {file = "setproctitle-1.3.3-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d876d355c53d975c2ef9c4f2487c8f83dad6aeaaee1b6571453cb0ee992f55f6"}, + {file = "setproctitle-1.3.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:224602f0939e6fb9d5dd881be1229d485f3257b540f8a900d4271a2c2aa4e5f4"}, + {file = "setproctitle-1.3.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d7f27e0268af2d7503386e0e6be87fb9b6657afd96f5726b733837121146750d"}, + {file = "setproctitle-1.3.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5e7266498cd31a4572378c61920af9f6b4676a73c299fce8ba93afd694f8ae7"}, + {file = "setproctitle-1.3.3-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33c5609ad51cd99d388e55651b19148ea99727516132fb44680e1f28dd0d1de9"}, + {file = "setproctitle-1.3.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:eae8988e78192fd1a3245a6f4f382390b61bce6cfcc93f3809726e4c885fa68d"}, + {file = "setproctitle-1.3.3.tar.gz", hash = "sha256:c913e151e7ea01567837ff037a23ca8740192880198b7fbb90b16d181607caae"}, +] + +[package.extras] +test = ["pytest"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "slack-bolt" +version = "1.18.1" +description = "The Bolt Framework for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "slack_bolt-1.18.1-py2.py3-none-any.whl", hash = "sha256:2509e5bb43898a593667bf37965057a9b9a41008b29628e3b57a6136b650b90e"}, + {file = "slack_bolt-1.18.1.tar.gz", hash = "sha256:694f84a81ba1c4c428ba7daa01d599d3e9fba7a54ad10c11008aa22573b23ff0"}, +] + +[package.dependencies] +slack-sdk = ">=3.25.0,<4" + +[package.extras] +adapter = ["CherryPy (>=18,<19)", "Django (>=3,<5)", "Flask (>=1,<3)", "Werkzeug (>=2,<3)", "boto3 (<=2)", "bottle (>=0.12,<1)", "chalice (>=1.28,<2)", "falcon (>=2,<4)", "fastapi (>=0.70.0,<1)", "gunicorn (>=20,<21)", "pyramid (>=1,<3)", "sanic (>=22,<23)", "starlette (>=0.14,<1)", "tornado (>=6,<7)", "uvicorn (<1)", "websocket-client (>=1.2.3,<2)"] +adapter-testing = ["Flask (>=1,<2)", "Werkzeug (>=1,<2)", "boddle (>=0.2,<0.3)", "docker (>=5,<6)", "moto (>=3,<4)", "requests (>=2,<3)", "sanic-testing (>=0.7)"] +async = ["aiohttp (>=3,<4)", "websockets (>=10,<11)"] +testing = ["Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werkzeug (>=1,<2)", "aiohttp (>=3,<4)", "black (==22.8.0)", "click (<=8.0.4)", "itsdangerous (==2.0.1)", "pytest (>=6.2.5,<7)", "pytest-asyncio (>=0.18.2,<1)", "pytest-cov (>=3,<4)"] +testing-without-asyncio = ["Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werkzeug (>=1,<2)", "black (==22.8.0)", "click (<=8.0.4)", "itsdangerous (==2.0.1)", "pytest (>=6.2.5,<7)", "pytest-cov (>=3,<4)"] + +[[package]] +name = "slack-sdk" +version = "3.26.2" +description = "The Slack API Platform SDK for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "slack_sdk-3.26.2-py2.py3-none-any.whl", hash = "sha256:a10e8ee69ca17d274989d0c2bbecb875f19898da3052d8d57de0898a00b1ab52"}, + {file = "slack_sdk-3.26.2.tar.gz", hash = "sha256:bcdac5e688fa50e9357ecd00b803b6a8bad766aa614d35d8dc0636f40adc48bf"}, +] + +[[package]] +name = "smmap" +version = "5.0.1" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +files = [ + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, +] + +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.25" +description = "Database Abstraction Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-2.0.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4344d059265cc8b1b1be351bfb88749294b87a8b2bbe21dfbe066c4199541ebd"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f9e2e59cbcc6ba1488404aad43de005d05ca56e069477b33ff74e91b6319735"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84daa0a2055df9ca0f148a64fdde12ac635e30edbca80e87df9b3aaf419e144a"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc8b7dabe8e67c4832891a5d322cec6d44ef02f432b4588390017f5cec186a84"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f5693145220517b5f42393e07a6898acdfe820e136c98663b971906120549da5"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db854730a25db7c956423bb9fb4bdd1216c839a689bf9cc15fada0a7fb2f4570"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-win32.whl", hash = "sha256:14a6f68e8fc96e5e8f5647ef6cda6250c780612a573d99e4d881581432ef1669"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-win_amd64.whl", hash = "sha256:87f6e732bccd7dcf1741c00f1ecf33797383128bd1c90144ac8adc02cbb98643"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:342d365988ba88ada8af320d43df4e0b13a694dbd75951f537b2d5e4cb5cd002"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f37c0caf14b9e9b9e8f6dbc81bc56db06acb4363eba5a633167781a48ef036ed"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa9373708763ef46782d10e950b49d0235bfe58facebd76917d3f5cbf5971aed"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d24f571990c05f6b36a396218f251f3e0dda916e0c687ef6fdca5072743208f5"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75432b5b14dc2fff43c50435e248b45c7cdadef73388e5610852b95280ffd0e9"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:884272dcd3ad97f47702965a0e902b540541890f468d24bd1d98bcfe41c3f018"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-win32.whl", hash = "sha256:e607cdd99cbf9bb80391f54446b86e16eea6ad309361942bf88318bcd452363c"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d505815ac340568fd03f719446a589162d55c52f08abd77ba8964fbb7eb5b5f"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0dacf67aee53b16f365c589ce72e766efaabd2b145f9de7c917777b575e3659d"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b801154027107461ee992ff4b5c09aa7cc6ec91ddfe50d02bca344918c3265c6"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59a21853f5daeb50412d459cfb13cb82c089ad4c04ec208cd14dddd99fc23b39"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29049e2c299b5ace92cbed0c1610a7a236f3baf4c6b66eb9547c01179f638ec5"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b64b183d610b424a160b0d4d880995e935208fc043d0302dd29fee32d1ee3f95"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f7a7d7fcc675d3d85fbf3b3828ecd5990b8d61bd6de3f1b260080b3beccf215"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-win32.whl", hash = "sha256:cf18ff7fc9941b8fc23437cc3e68ed4ebeff3599eec6ef5eebf305f3d2e9a7c2"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-win_amd64.whl", hash = "sha256:91f7d9d1c4dd1f4f6e092874c128c11165eafcf7c963128f79e28f8445de82d5"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bb209a73b8307f8fe4fe46f6ad5979649be01607f11af1eb94aa9e8a3aaf77f0"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:798f717ae7c806d67145f6ae94dc7c342d3222d3b9a311a784f371a4333212c7"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fdd402169aa00df3142149940b3bf9ce7dde075928c1886d9a1df63d4b8de62"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0d3cab3076af2e4aa5693f89622bef7fa770c6fec967143e4da7508b3dceb9b9"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:74b080c897563f81062b74e44f5a72fa44c2b373741a9ade701d5f789a10ba23"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-win32.whl", hash = "sha256:87d91043ea0dc65ee583026cb18e1b458d8ec5fc0a93637126b5fc0bc3ea68c4"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-win_amd64.whl", hash = "sha256:75f99202324383d613ddd1f7455ac908dca9c2dd729ec8584c9541dd41822a2c"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:420362338681eec03f53467804541a854617faed7272fe71a1bfdb07336a381e"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c88f0c7dcc5f99bdb34b4fd9b69b93c89f893f454f40219fe923a3a2fd11625"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3be4987e3ee9d9a380b66393b77a4cd6d742480c951a1c56a23c335caca4ce3"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a159111a0f58fb034c93eeba211b4141137ec4b0a6e75789ab7a3ef3c7e7e3"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8b8cb63d3ea63b29074dcd29da4dc6a97ad1349151f2d2949495418fd6e48db9"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:736ea78cd06de6c21ecba7416499e7236a22374561493b456a1f7ffbe3f6cdb4"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-win32.whl", hash = "sha256:10331f129982a19df4284ceac6fe87353ca3ca6b4ca77ff7d697209ae0a5915e"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-win_amd64.whl", hash = "sha256:c55731c116806836a5d678a70c84cb13f2cedba920212ba7dcad53260997666d"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:605b6b059f4b57b277f75ace81cc5bc6335efcbcc4ccb9066695e515dbdb3900"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:665f0a3954635b5b777a55111ababf44b4fc12b1f3ba0a435b602b6387ffd7cf"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecf6d4cda1f9f6cb0b45803a01ea7f034e2f1aed9475e883410812d9f9e3cfcf"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c51db269513917394faec5e5c00d6f83829742ba62e2ac4fa5c98d58be91662f"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:790f533fa5c8901a62b6fef5811d48980adeb2f51f1290ade8b5e7ba990ba3de"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1b1180cda6df7af84fe72e4530f192231b1f29a7496951db4ff38dac1687202d"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-win32.whl", hash = "sha256:555651adbb503ac7f4cb35834c5e4ae0819aab2cd24857a123370764dc7d7e24"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-win_amd64.whl", hash = "sha256:dc55990143cbd853a5d038c05e79284baedf3e299661389654551bd02a6a68d7"}, + {file = "SQLAlchemy-2.0.25-py3-none-any.whl", hash = "sha256:a86b4240e67d4753dc3092d9511886795b3c2852abe599cffe108952f7af7ac3"}, + {file = "SQLAlchemy-2.0.25.tar.gz", hash = "sha256:a2c69a7664fb2d54b8682dd774c3b54f67f84fa123cf84dda2a5f40dcaa04e08"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +typing-extensions = ">=4.6.0" + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3_binary"] + +[[package]] +name = "stack-data" +version = "0.6.3" +description = "Extract data from python stack frames and tracebacks for informative displays" +optional = false +python-versions = "*" +files = [ + {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, + {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, +] + +[package.dependencies] +asttokens = ">=2.1.0" +executing = ">=1.2.0" +pure-eval = "*" + +[package.extras] +tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] + +[[package]] +name = "starlette" +version = "0.36.3" +description = "The little ASGI library that shines." +optional = false +python-versions = ">=3.8" +files = [ + {file = "starlette-0.36.3-py3-none-any.whl", hash = "sha256:13d429aa93a61dc40bf503e8c801db1f1bca3dc706b10ef2434a36123568f044"}, + {file = "starlette-0.36.3.tar.gz", hash = "sha256:90a671733cfb35771d8cc605e0b679d23b992f8dcfad48cc60b38cb29aeb7080"}, +] + +[package.dependencies] +anyio = ">=3.4.0,<5" + +[package.extras] +full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7)", "pyyaml"] + +[[package]] +name = "sympy" +version = "1.12" +description = "Computer algebra system (CAS) in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, + {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, +] + +[package.dependencies] +mpmath = ">=0.19" + +[[package]] +name = "tabulate" +version = "0.9.0" +description = "Pretty-print tabular data" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, + {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, +] + +[package.extras] +widechars = ["wcwidth"] + +[[package]] +name = "tenacity" +version = "8.2.3" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tenacity-8.2.3-py3-none-any.whl", hash = "sha256:ce510e327a630c9e1beaf17d42e6ffacc88185044ad85cf74c0a8887c6a0f88c"}, + {file = "tenacity-8.2.3.tar.gz", hash = "sha256:5398ef0d78e63f40007c1fb4c0bff96e1911394d2fa8d194f77619c05ff6cc8a"}, +] + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + +[[package]] +name = "terminado" +version = "0.18.0" +description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "terminado-0.18.0-py3-none-any.whl", hash = "sha256:87b0d96642d0fe5f5abd7783857b9cab167f221a39ff98e3b9619a788a3c0f2e"}, + {file = "terminado-0.18.0.tar.gz", hash = "sha256:1ea08a89b835dd1b8c0c900d92848147cef2537243361b2e3f4dc15df9b6fded"}, +] + +[package.dependencies] +ptyprocess = {version = "*", markers = "os_name != \"nt\""} +pywinpty = {version = ">=1.1.0", markers = "os_name == \"nt\""} +tornado = ">=6.1.0" + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] +test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] +typing = ["mypy (>=1.6,<2.0)", "traitlets (>=5.11.1)"] + +[[package]] +name = "threadpoolctl" +version = "3.2.0" +description = "threadpoolctl" +optional = false +python-versions = ">=3.8" +files = [ + {file = "threadpoolctl-3.2.0-py3-none-any.whl", hash = "sha256:2b7818516e423bdaebb97c723f86a7c6b0a83d3f3b0970328d66f4d9104dc032"}, + {file = "threadpoolctl-3.2.0.tar.gz", hash = "sha256:c96a0ba3bdddeaca37dc4cc7344aafad41cdb8c313f74fdfe387a867bba93355"}, +] + +[[package]] +name = "tiktoken" +version = "0.5.2" +description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tiktoken-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c4e654282ef05ec1bd06ead22141a9a1687991cef2c6a81bdd1284301abc71d"}, + {file = "tiktoken-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b3134aa24319f42c27718c6967f3c1916a38a715a0fa73d33717ba121231307"}, + {file = "tiktoken-0.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6092e6e77730929c8c6a51bb0d7cfdf1b72b63c4d033d6258d1f2ee81052e9e5"}, + {file = "tiktoken-0.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72ad8ae2a747622efae75837abba59be6c15a8f31b4ac3c6156bc56ec7a8e631"}, + {file = "tiktoken-0.5.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:51cba7c8711afa0b885445f0637f0fcc366740798c40b981f08c5f984e02c9d1"}, + {file = "tiktoken-0.5.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3d8c7d2c9313f8e92e987d585ee2ba0f7c40a0de84f4805b093b634f792124f5"}, + {file = "tiktoken-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:692eca18c5fd8d1e0dde767f895c17686faaa102f37640e884eecb6854e7cca7"}, + {file = "tiktoken-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:138d173abbf1ec75863ad68ca289d4da30caa3245f3c8d4bfb274c4d629a2f77"}, + {file = "tiktoken-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7388fdd684690973fdc450b47dfd24d7f0cbe658f58a576169baef5ae4658607"}, + {file = "tiktoken-0.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a114391790113bcff670c70c24e166a841f7ea8f47ee2fe0e71e08b49d0bf2d4"}, + {file = "tiktoken-0.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca96f001e69f6859dd52926d950cfcc610480e920e576183497ab954e645e6ac"}, + {file = "tiktoken-0.5.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:15fed1dd88e30dfadcdd8e53a8927f04e1f6f81ad08a5ca824858a593ab476c7"}, + {file = "tiktoken-0.5.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:93f8e692db5756f7ea8cb0cfca34638316dcf0841fb8469de8ed7f6a015ba0b0"}, + {file = "tiktoken-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:bcae1c4c92df2ffc4fe9f475bf8148dbb0ee2404743168bbeb9dcc4b79dc1fdd"}, + {file = "tiktoken-0.5.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b76a1e17d4eb4357d00f0622d9a48ffbb23401dcf36f9716d9bd9c8e79d421aa"}, + {file = "tiktoken-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:01d8b171bb5df4035580bc26d4f5339a6fd58d06f069091899d4a798ea279d3e"}, + {file = "tiktoken-0.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42adf7d4fb1ed8de6e0ff2e794a6a15005f056a0d83d22d1d6755a39bffd9e7f"}, + {file = "tiktoken-0.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3f894dbe0adb44609f3d532b8ea10820d61fdcb288b325a458dfc60fefb7db"}, + {file = "tiktoken-0.5.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:58ccfddb4e62f0df974e8f7e34a667981d9bb553a811256e617731bf1d007d19"}, + {file = "tiktoken-0.5.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58902a8bad2de4268c2a701f1c844d22bfa3cbcc485b10e8e3e28a050179330b"}, + {file = "tiktoken-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e39257826d0647fcac403d8fa0a474b30d02ec8ffc012cfaf13083e9b5e82c5"}, + {file = "tiktoken-0.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8bde3b0fbf09a23072d39c1ede0e0821f759b4fa254a5f00078909158e90ae1f"}, + {file = "tiktoken-0.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2ddee082dcf1231ccf3a591d234935e6acf3e82ee28521fe99af9630bc8d2a60"}, + {file = "tiktoken-0.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35c057a6a4e777b5966a7540481a75a31429fc1cb4c9da87b71c8b75b5143037"}, + {file = "tiktoken-0.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c4a049b87e28f1dc60509f8eb7790bc8d11f9a70d99b9dd18dfdd81a084ffe6"}, + {file = "tiktoken-0.5.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5bf5ce759089f4f6521ea6ed89d8f988f7b396e9f4afb503b945f5c949c6bec2"}, + {file = "tiktoken-0.5.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0c964f554af1a96884e01188f480dad3fc224c4bbcf7af75d4b74c4b74ae0125"}, + {file = "tiktoken-0.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:368dd5726d2e8788e47ea04f32e20f72a2012a8a67af5b0b003d1e059f1d30a3"}, + {file = "tiktoken-0.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a2deef9115b8cd55536c0a02c0203512f8deb2447f41585e6d929a0b878a0dd2"}, + {file = "tiktoken-0.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2ed7d380195affbf886e2f8b92b14edfe13f4768ff5fc8de315adba5b773815e"}, + {file = "tiktoken-0.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c76fce01309c8140ffe15eb34ded2bb94789614b7d1d09e206838fc173776a18"}, + {file = "tiktoken-0.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60a5654d6a2e2d152637dd9a880b4482267dfc8a86ccf3ab1cec31a8c76bfae8"}, + {file = "tiktoken-0.5.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:41d4d3228e051b779245a8ddd21d4336f8975563e92375662f42d05a19bdff41"}, + {file = "tiktoken-0.5.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c1cdec2c92fcde8c17a50814b525ae6a88e8e5b02030dc120b76e11db93f13"}, + {file = "tiktoken-0.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:84ddb36faedb448a50b246e13d1b6ee3437f60b7169b723a4b2abad75e914f3e"}, + {file = "tiktoken-0.5.2.tar.gz", hash = "sha256:f54c581f134a8ea96ce2023ab221d4d4d81ab614efa0b2fbce926387deb56c80"}, +] + +[package.dependencies] +regex = ">=2022.1.18" +requests = ">=2.26.0" + +[package.extras] +blobfile = ["blobfile (>=2)"] + +[[package]] +name = "tinycss2" +version = "1.2.1" +description = "A tiny CSS parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tinycss2-1.2.1-py3-none-any.whl", hash = "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847"}, + {file = "tinycss2-1.2.1.tar.gz", hash = "sha256:8cff3a8f066c2ec677c06dbc7b45619804a6938478d9d73c284b29d14ecb0627"}, +] + +[package.dependencies] +webencodings = ">=0.4" + +[package.extras] +doc = ["sphinx", "sphinx_rtd_theme"] +test = ["flake8", "isort", "pytest"] + +[[package]] +name = "tokenizers" +version = "0.15.1" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tokenizers-0.15.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:32c9491dd1bcb33172c26b454dbd607276af959b9e78fa766e2694cafab3103c"}, + {file = "tokenizers-0.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29a1b784b870a097e7768f8c20c2dd851e2c75dad3efdae69a79d3e7f1d614d5"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0049fbe648af04148b08cb211994ce8365ee628ce49724b56aaefd09a3007a78"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e84b3c235219e75e24de6b71e6073cd2c8d740b14d88e4c6d131b90134e3a338"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8cc575769ea11d074308c6d71cb10b036cdaec941562c07fc7431d956c502f0e"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22bf28f299c4158e6d0b5eaebddfd500c4973d947ffeaca8bcbe2e8c137dff0b"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:506555f98361db9c74e1323a862d77dcd7d64c2058829a368bf4159d986e339f"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7061b0a28ade15906f5b2ec8c48d3bdd6e24eca6b427979af34954fbe31d5cef"}, + {file = "tokenizers-0.15.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ed5e35507b7a0e2aac3285c4f5e37d4ec5cfc0e5825b862b68a0aaf2757af52"}, + {file = "tokenizers-0.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1c9df9247df0de6509dd751b1c086e5f124b220133b5c883bb691cb6fb3d786f"}, + {file = "tokenizers-0.15.1-cp310-none-win32.whl", hash = "sha256:dd999af1b4848bef1b11d289f04edaf189c269d5e6afa7a95fa1058644c3f021"}, + {file = "tokenizers-0.15.1-cp310-none-win_amd64.whl", hash = "sha256:39d06a57f7c06940d602fad98702cf7024c4eee7f6b9fe76b9f2197d5a4cc7e2"}, + {file = "tokenizers-0.15.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8ad034eb48bf728af06915e9294871f72fcc5254911eddec81d6df8dba1ce055"}, + {file = "tokenizers-0.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea9ede7c42f8fa90f31bfc40376fd91a7d83a4aa6ad38e6076de961d48585b26"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b85d6fe1a20d903877aa0ef32ef6b96e81e0e48b71c206d6046ce16094de6970"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a7d44f656320137c7d643b9c7dcc1814763385de737fb98fd2643880910f597"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd244bd0793cdacf27ee65ec3db88c21f5815460e8872bbeb32b040469d6774e"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0f3f4a36e371b3cb1123adac8aeeeeab207ad32f15ed686d9d71686a093bb140"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2921a53966afb29444da98d56a6ccbef23feb3b0c0f294b4e502370a0a64f25"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f49068cf51f49c231067f1a8c9fc075ff960573f6b2a956e8e1b0154fb638ea5"}, + {file = "tokenizers-0.15.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0ab1a22f20eaaab832ab3b00a0709ca44a0eb04721e580277579411b622c741c"}, + {file = "tokenizers-0.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:671268f24b607c4adc6fa2b5b580fd4211b9f84b16bd7f46d62f8e5be0aa7ba4"}, + {file = "tokenizers-0.15.1-cp311-none-win32.whl", hash = "sha256:a4f03e33d2bf7df39c8894032aba599bf90f6f6378e683a19d28871f09bb07fc"}, + {file = "tokenizers-0.15.1-cp311-none-win_amd64.whl", hash = "sha256:30f689537bcc7576d8bd4daeeaa2cb8f36446ba2f13f421b173e88f2d8289c4e"}, + {file = "tokenizers-0.15.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f3a379dd0898a82ea3125e8f9c481373f73bffce6430d4315f0b6cd5547e409"}, + {file = "tokenizers-0.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d870ae58bba347d38ac3fc8b1f662f51e9c95272d776dd89f30035c83ee0a4f"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d6d28e0143ec2e253a8a39e94bf1d24776dbe73804fa748675dbffff4a5cd6d8"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61ae9ac9f44e2da128ee35db69489883b522f7abe033733fa54eb2de30dac23d"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d8e322a47e29128300b3f7749a03c0ec2bce0a3dc8539ebff738d3f59e233542"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:760334f475443bc13907b1a8e1cb0aeaf88aae489062546f9704dce6c498bfe2"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1b173753d4aca1e7d0d4cb52b5e3ffecfb0ca014e070e40391b6bb4c1d6af3f2"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82c1f13d457c8f0ab17e32e787d03470067fe8a3b4d012e7cc57cb3264529f4a"}, + {file = "tokenizers-0.15.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:425b46ceff4505f20191df54b50ac818055d9d55023d58ae32a5d895b6f15bb0"}, + {file = "tokenizers-0.15.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:681ac6ba3b4fdaf868ead8971221a061f580961c386e9732ea54d46c7b72f286"}, + {file = "tokenizers-0.15.1-cp312-none-win32.whl", hash = "sha256:f2272656063ccfba2044df2115095223960d80525d208e7a32f6c01c351a6f4a"}, + {file = "tokenizers-0.15.1-cp312-none-win_amd64.whl", hash = "sha256:9abe103203b1c6a2435d248d5ff4cceebcf46771bfbc4957a98a74da6ed37674"}, + {file = "tokenizers-0.15.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2ce9ed5c8ef26b026a66110e3c7b73d93ec2d26a0b1d0ea55ddce61c0e5f446f"}, + {file = "tokenizers-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:89b24d366137986c3647baac29ef902d2d5445003d11c30df52f1bd304689aeb"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0faebedd01b413ab777ca0ee85914ed8b031ea5762ab0ea60b707ce8b9be6842"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdbd9dfcdad4f3b95d801f768e143165165055c18e44ca79a8a26de889cd8e85"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:97194324c12565b07e9993ca9aa813b939541185682e859fb45bb8d7d99b3193"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:485e43e2cc159580e0d83fc919ec3a45ae279097f634b1ffe371869ffda5802c"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:191d084d60e3589d6420caeb3f9966168269315f8ec7fbc3883122dc9d99759d"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01c28cc8d7220634a75b14c53f4fc9d1b485f99a5a29306a999c115921de2897"}, + {file = "tokenizers-0.15.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:325212027745d3f8d5d5006bb9e5409d674eb80a184f19873f4f83494e1fdd26"}, + {file = "tokenizers-0.15.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3c5573603c36ce12dbe318bcfb490a94cad2d250f34deb2f06cb6937957bbb71"}, + {file = "tokenizers-0.15.1-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:1441161adb6d71a15a630d5c1d8659d5ebe41b6b209586fbeea64738e58fcbb2"}, + {file = "tokenizers-0.15.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:382a8d0c31afcfb86571afbfefa37186df90865ce3f5b731842dab4460e53a38"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e76959783e3f4ec73b3f3d24d4eec5aa9225f0bee565c48e77f806ed1e048f12"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:401df223e5eb927c5961a0fc6b171818a2bba01fb36ef18c3e1b69b8cd80e591"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c52606c233c759561a16e81b2290a7738c3affac7a0b1f0a16fe58dc22e04c7d"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72c658bbe5a05ed8bc2ac5ad782385bfd743ffa4bc87d9b5026341e709c6f44"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25f5643a2f005c42f0737a326c6c6bdfedfdc9a994b10a1923d9c3e792e4d6a6"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5b6f633999d6b42466bbfe21be2e26ad1760b6f106967a591a41d8cbca980e"}, + {file = "tokenizers-0.15.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ceb5c9ad11a015150b545c1a11210966a45b8c3d68a942e57cf8938c578a77ca"}, + {file = "tokenizers-0.15.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bedd4ce0c4872db193444c395b11c7697260ce86a635ab6d48102d76be07d324"}, + {file = "tokenizers-0.15.1-cp37-none-win32.whl", hash = "sha256:cd6caef6c14f5ed6d35f0ddb78eab8ca6306d0cd9870330bccff72ad014a6f42"}, + {file = "tokenizers-0.15.1-cp37-none-win_amd64.whl", hash = "sha256:d2bd7af78f58d75a55e5df61efae164ab9200c04b76025f9cc6eeb7aff3219c2"}, + {file = "tokenizers-0.15.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:59b3ca6c02e0bd5704caee274978bd055de2dff2e2f39dadf536c21032dfd432"}, + {file = "tokenizers-0.15.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:48fe21b67c22583bed71933a025fd66b1f5cfae1baefa423c3d40379b5a6e74e"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:3d190254c66a20fb1efbdf035e6333c5e1f1c73b1f7bfad88f9c31908ac2c2c4"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fef90c8f5abf17d48d6635f5fd92ad258acd1d0c2d920935c8bf261782cfe7c8"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fac011ef7da3357aa7eb19efeecf3d201ede9618f37ddedddc5eb809ea0963ca"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:574ec5b3e71d1feda6b0ecac0e0445875729b4899806efbe2b329909ec75cb50"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aca16c3c0637c051a59ea99c4253f16fbb43034fac849076a7e7913b2b9afd2d"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a6f238fc2bbfd3e12e8529980ec1624c7e5b69d4e959edb3d902f36974f725a"}, + {file = "tokenizers-0.15.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:587e11a26835b73c31867a728f32ca8a93c9ded4a6cd746516e68b9d51418431"}, + {file = "tokenizers-0.15.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6456e7ad397352775e2efdf68a9ec5d6524bbc4543e926eef428d36de627aed4"}, + {file = "tokenizers-0.15.1-cp38-none-win32.whl", hash = "sha256:614f0da7dd73293214bd143e6221cafd3f7790d06b799f33a987e29d057ca658"}, + {file = "tokenizers-0.15.1-cp38-none-win_amd64.whl", hash = "sha256:a4fa0a20d9f69cc2bf1cfce41aa40588598e77ec1d6f56bf0eb99769969d1ede"}, + {file = "tokenizers-0.15.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8d3f18a45e0cf03ce193d5900460dc2430eec4e14c786e5d79bddba7ea19034f"}, + {file = "tokenizers-0.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:38dbd6c38f88ad7d5dc5d70c764415d38fe3bcd99dc81638b572d093abc54170"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:777286b1f7e52de92aa4af49fe31046cfd32885d1bbaae918fab3bba52794c33"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58d4d550a3862a47dd249892d03a025e32286eb73cbd6bc887fb8fb64bc97165"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4eda68ce0344f35042ae89220b40a0007f721776b727806b5c95497b35714bb7"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cd33d15f7a3a784c3b665cfe807b8de3c6779e060349bd5005bb4ae5bdcb437"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a1aa370f978ac0bfb50374c3a40daa93fd56d47c0c70f0c79607fdac2ccbb42"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:241482b940340fff26a2708cb9ba383a5bb8a2996d67a0ff2c4367bf4b86cc3a"}, + {file = "tokenizers-0.15.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:68f30b05f46a4d9aba88489eadd021904afe90e10a7950e28370d6e71b9db021"}, + {file = "tokenizers-0.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5a3c5d8025529670462b881b7b2527aacb6257398c9ec8e170070432c3ae3a82"}, + {file = "tokenizers-0.15.1-cp39-none-win32.whl", hash = "sha256:74d1827830f60a9d78da8f6d49a1fbea5422ce0eea42e2617877d23380a7efbc"}, + {file = "tokenizers-0.15.1-cp39-none-win_amd64.whl", hash = "sha256:9ff499923e4d6876d6b6a63ea84a56805eb35e91dd89b933a7aee0c56a3838c6"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b3aa007a0f4408f62a8471bdaa3faccad644cbf2622639f2906b4f9b5339e8b8"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f3d4176fa93d8b2070db8f3c70dc21106ae6624fcaaa334be6bdd3a0251e729e"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1d0e463655ef8b2064df07bd4a445ed7f76f6da3b286b4590812587d42f80e89"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:089138fd0351b62215c462a501bd68b8df0e213edcf99ab9efd5dba7b4cb733e"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e563ac628f5175ed08e950430e2580e544b3e4b606a0995bb6b52b3a3165728"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:244dcc28c5fde221cb4373961b20da30097669005b122384d7f9f22752487a46"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d82951d46052dddae1369e68ff799a0e6e29befa9a0b46e387ae710fd4daefb0"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7b14296bc9059849246ceb256ffbe97f8806a9b5d707e0095c22db312f4fc014"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0309357bb9b6c8d86cdf456053479d7112074b470651a997a058cd7ad1c4ea57"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083f06e9d8d01b70b67bcbcb7751b38b6005512cce95808be6bf34803534a7e7"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85288aea86ada579789447f0dcec108ebef8da4b450037eb4813d83e4da9371e"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:385e6fcb01e8de90c1d157ae2a5338b23368d0b1c4cc25088cdca90147e35d17"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:60067edfcbf7d6cd448ac47af41ec6e84377efbef7be0c06f15a7c1dd069e044"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f7e37f89acfe237d4eaf93c3b69b0f01f407a7a5d0b5a8f06ba91943ea3cf10"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:6a63a15b523d42ebc1f4028e5a568013388c2aefa4053a263e511cb10aaa02f1"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2417d9e4958a6c2fbecc34c27269e74561c55d8823bf914b422e261a11fdd5fd"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8550974bace6210e41ab04231e06408cf99ea4279e0862c02b8d47e7c2b2828"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:194ba82129b171bcd29235a969e5859a93e491e9b0f8b2581f500f200c85cfdd"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:1bfd95eef8b01e6c0805dbccc8eaf41d8c5a84f0cce72c0ab149fe76aae0bce6"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b87a15dd72f8216b03c151e3dace00c75c3fe7b0ee9643c25943f31e582f1a34"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6ac22f358a0c2a6c685be49136ce7ea7054108986ad444f567712cf274b34cd8"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e9d1f046a9b9d9a95faa103f07db5921d2c1c50f0329ebba4359350ee02b18b"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a0fd30a4b74485f6a7af89fffb5fb84d6d5f649b3e74f8d37f624cc9e9e97cf"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e45dc206b9447fa48795a1247c69a1732d890b53e2cc51ba42bc2fefa22407"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4eaff56ef3e218017fa1d72007184401f04cb3a289990d2b6a0a76ce71c95f96"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:b41dc107e4a4e9c95934e79b025228bbdda37d9b153d8b084160e88d5e48ad6f"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1922b8582d0c33488764bcf32e80ef6054f515369e70092729c928aae2284bc2"}, + {file = "tokenizers-0.15.1.tar.gz", hash = "sha256:c0a331d6d5a3d6e97b7f99f562cee8d56797180797bc55f12070e495e717c980"}, +] + +[package.dependencies] +huggingface_hub = ">=0.16.4,<1.0" + +[package.extras] +dev = ["tokenizers[testing]"] +docs = ["setuptools_rust", "sphinx", "sphinx_rtd_theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tornado" +version = "6.4" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">= 3.8" +files = [ + {file = "tornado-6.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0"}, + {file = "tornado-6.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f"}, + {file = "tornado-6.4-cp38-abi3-win32.whl", hash = "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052"}, + {file = "tornado-6.4-cp38-abi3-win_amd64.whl", hash = "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63"}, + {file = "tornado-6.4.tar.gz", hash = "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee"}, +] + +[[package]] +name = "tqdm" +version = "4.66.1" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, + {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "traitlets" +version = "5.14.1" +description = "Traitlets Python configuration system" +optional = false +python-versions = ">=3.8" +files = [ + {file = "traitlets-5.14.1-py3-none-any.whl", hash = "sha256:2e5a030e6eff91737c643231bfcf04a65b0132078dad75e4936700b213652e74"}, + {file = "traitlets-5.14.1.tar.gz", hash = "sha256:8585105b371a04b8316a43d5ce29c098575c2e477850b62b848b964f1444527e"}, +] + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] +test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<7.5)", "pytest-mock", "pytest-mypy-testing"] + +[[package]] +name = "tree-sitter" +version = "0.20.4" +description = "Python bindings for the Tree-Sitter parsing library" +optional = false +python-versions = ">=3.3" +files = [ + {file = "tree_sitter-0.20.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c259b9bcb596e54f54713eb3951226fc834d65289940f4bfdcdf519f08e8e876"}, + {file = "tree_sitter-0.20.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:88da7e2e4c69881cd63916cc24ae0b809f96aae331da45b418ae6b2d1ed2ca19"}, + {file = "tree_sitter-0.20.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:66a68b156ba131e9d8dff4a1f72037f4b368cc50c58f18905a91743ae1d1c795"}, + {file = "tree_sitter-0.20.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae28e25d551f406807011487bdfb9728041e656b30b554fa7f3391ab64ed69f9"}, + {file = "tree_sitter-0.20.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36b10c9c69e825ba65cf9b0f77668bf33e70d2a5764b64ad6f133f8cc9220f09"}, + {file = "tree_sitter-0.20.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7c18c64ddd44b75b7e1660b9793753eda427e4b145b6216d4b2d2e9b200c74f2"}, + {file = "tree_sitter-0.20.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e9e9e594bbefb76ad9ea256f5c87eba7591b4758854d3df83ce4df415933a006"}, + {file = "tree_sitter-0.20.4-cp310-cp310-win32.whl", hash = "sha256:b4755229dc18644fe48bcab974bde09b171fcb6ef625d3cb5ece5c6198f4223e"}, + {file = "tree_sitter-0.20.4-cp310-cp310-win_amd64.whl", hash = "sha256:f792684cee8a46d9194d9f4223810e54ccc704470c5777538d59fbde0a4c91bf"}, + {file = "tree_sitter-0.20.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9d22ee75f45836554ee6a11e50dd8f9827941e67c49fce9a0790245b899811a9"}, + {file = "tree_sitter-0.20.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2a0ffd76dd991ba745bb5d0ba1d583bec85726d3ddef8c9685dc8636a619adde"}, + {file = "tree_sitter-0.20.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:060d4e5b803be0975f1ac46e54a292eab0701296ccd912f6cdac3f7331e29143"}, + {file = "tree_sitter-0.20.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:822e02366dbf223697b2b56b8f91aa5b60571f9fe7c998988a381db1c69604e9"}, + {file = "tree_sitter-0.20.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:527ca72c6a8f60fa719af37fa86f58b7ad0e07b8f74d1c1c7e926c5c888a7e6b"}, + {file = "tree_sitter-0.20.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a418ca71309ea7052e076f08d623f33f58eae01a8e8cdc1e6d3a01b5b8ddebfe"}, + {file = "tree_sitter-0.20.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c3ba2561b61a83c28ca06a0bce2a5ffcfb6b39f9d27a45e5ebd9cad2bedb7f"}, + {file = "tree_sitter-0.20.4-cp311-cp311-win32.whl", hash = "sha256:8d04c75a389b2de94952d602264852acff8cd3ed1ccf8a2492a080973d5ddd58"}, + {file = "tree_sitter-0.20.4-cp311-cp311-win_amd64.whl", hash = "sha256:ba9215c0e7529d9eb370528e5d99b7389d14a7eae94f07d14fa9dab18f267c62"}, + {file = "tree_sitter-0.20.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:c4c1af5ed4306071d30970c83ec882520a7bf5d8053996dbc4aa5c59238d4990"}, + {file = "tree_sitter-0.20.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9d70bfa550cf22c9cea9b3c0d18b889fc4f2a7e9dcf1d6cc93f49fa9d4a94954"}, + {file = "tree_sitter-0.20.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6de537bca0641775d8d175d37303d54998980fc0d997dd9aa89e16b415bf0cc3"}, + {file = "tree_sitter-0.20.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b1c0f8c0e3e50267566f5116cdceedf4e23e8c08b55ef3becbe954a11b16e84"}, + {file = "tree_sitter-0.20.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ef2ee6d9bb8e21713949e5ff769ed670fe1217f95b7eeb6c675788438c1e6e"}, + {file = "tree_sitter-0.20.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b6fd1c881ab0de5faa67168db2d001eee32be5482cb4e0b21b217689a05b6fe4"}, + {file = "tree_sitter-0.20.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bf47047420021d50aec529cb66387c90562350b499ddf56ecef1fc8255439e30"}, + {file = "tree_sitter-0.20.4-cp312-cp312-win32.whl", hash = "sha256:c16b48378041fc9702b6aa3480f2ffa49ca8ea58141a862acd569e5a0679655f"}, + {file = "tree_sitter-0.20.4-cp312-cp312-win_amd64.whl", hash = "sha256:973e871167079a1b1d7304d361449253efbe2a6974728ad563cf407bd02ddccb"}, + {file = "tree_sitter-0.20.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9d33a55598dd18a4d8b869a3417de82a4812c3a7dc7e61cb025ece3e9c3e4e96"}, + {file = "tree_sitter-0.20.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cee6955c2c97fc5927a41c7a8b06647c4b4d9b99b8a1581bf1183435c8cec3e"}, + {file = "tree_sitter-0.20.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5022bea67e479ad212be7c05b983a72e297a013efb4e8ea5b5b4d7da79a9fdef"}, + {file = "tree_sitter-0.20.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:640f60a5b966f0990338f1bf559455c3dcb822bc4329d82b3d42f32a48374dfe"}, + {file = "tree_sitter-0.20.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0e83f641fe6f27d91bd4d259fff5d35de1567d3f581b9efe9bbd5be50fe4ddc7"}, + {file = "tree_sitter-0.20.4-cp36-cp36m-win32.whl", hash = "sha256:ce6a85027c66fa3f09d482cc6d41927ea40955f7f33b86aedd26dd932709a2c9"}, + {file = "tree_sitter-0.20.4-cp36-cp36m-win_amd64.whl", hash = "sha256:fe10779347a6c067af29cb37fd4b75fa96c5cb68f587cc9530b70fe3f2a51a55"}, + {file = "tree_sitter-0.20.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28d5f84e34e276887e3a240b60906ca7e2b51e975f3145c3149ceed977a69508"}, + {file = "tree_sitter-0.20.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c913b65cbe10996116988ac436748f24883b5097e58274223e89bb2c5d1bb1a"}, + {file = "tree_sitter-0.20.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecaed46241e071752195a628bb97d2b740f2fde9e34f8a74456a4ea8bb26df88"}, + {file = "tree_sitter-0.20.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b641e88a97eab002a1736d93ef5a4beac90ea4fd6e25affd1831319b99f456c9"}, + {file = "tree_sitter-0.20.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:327c40f439c6155e4eee54c4657e4701a04f5f4816d9defdcb836bf65bf83d21"}, + {file = "tree_sitter-0.20.4-cp37-cp37m-win32.whl", hash = "sha256:1b7c1d95f006b3de42fbf4045bd00c273d113e372fcb6a5378e74ed120c12032"}, + {file = "tree_sitter-0.20.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6140d037239a41046f5d34fba5e0374ee697adb4b48b90579c618b5402781c11"}, + {file = "tree_sitter-0.20.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f42fd1104efaad8151370f1936e2a488b7337a5d24544a9ab59ba4c4010b1272"}, + {file = "tree_sitter-0.20.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7859717c5d62ee386b3d036cab8ed0f88f8c027b6b4ae476a55a8c5fb8aab713"}, + {file = "tree_sitter-0.20.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fdd361fe1cc68db68b4d85165641275e34b86cc26b2bab932790204fa14824dc"}, + {file = "tree_sitter-0.20.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b8d7539075606027b67764543463ff2bc4e52f4158ef6dc419c9f5625aa5383"}, + {file = "tree_sitter-0.20.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78e76307f05aca6cde72f3307b4d53701f34ae45f2248ceb83d1626051e201fd"}, + {file = "tree_sitter-0.20.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dd8c352f4577f61098d06cf3feb7fd214259f41b5036b81003860ed54d16b448"}, + {file = "tree_sitter-0.20.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:281f3e5382d1bd7fccc88d1afe68c915565bc24f8b8dd4844079d46c7815b8a7"}, + {file = "tree_sitter-0.20.4-cp38-cp38-win32.whl", hash = "sha256:6a77ac3cdcddd80cdd1fd394318bff99f94f37e08d235aaefccb87e1224946e5"}, + {file = "tree_sitter-0.20.4-cp38-cp38-win_amd64.whl", hash = "sha256:8eee8adf54033dc48eab84b040f4d7b32355a964c4ae0aae5dfbdc4dbc3364ca"}, + {file = "tree_sitter-0.20.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e89f6508e30fce05e2c724725d022db30d877817b9d64f933506ffb3a3f4a2c2"}, + {file = "tree_sitter-0.20.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7fb6286bb1fae663c45ff0700ec88fb9b50a81eed2bae8a291f95fcf8cc19547"}, + {file = "tree_sitter-0.20.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:11e93f8b4bbae04070416a82257a7ab2eb0afb76e093ae3ea73bd63b792f6846"}, + {file = "tree_sitter-0.20.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8250725c5f78929aeb2c71db5dca76f1ef448389ca16f9439161f90978bb8478"}, + {file = "tree_sitter-0.20.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d404a8ca9de9b0843844f0cd4d423f46bc46375ab8afb63b1d8ec01201457ac8"}, + {file = "tree_sitter-0.20.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0f2422c9ee70ba972dfc3943746e6cf7fc03725a866908950245bda9ccfc7301"}, + {file = "tree_sitter-0.20.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:21a937942e4729abbe778a609d2c218574436cb351c36fba89ef3c8c6066ec78"}, + {file = "tree_sitter-0.20.4-cp39-cp39-win32.whl", hash = "sha256:427a9a39360cc1816e28f8182550e478e4ba983595a2565ab9dfe32ea1b03fd7"}, + {file = "tree_sitter-0.20.4-cp39-cp39-win_amd64.whl", hash = "sha256:7095bb9aff297fa9c6026bf8914fd295997d714d1a6ee9a1edf7282c772f9f64"}, + {file = "tree_sitter-0.20.4-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:859260b90f0e3867ae840e39f54e830f607b3bc531bc21deeeeaa8a30cbb89ad"}, + {file = "tree_sitter-0.20.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dfc14be73cf46126660a3aecdd0396e69562ad1a902245225ca7bd29649594e"}, + {file = "tree_sitter-0.20.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec46355bf3ff23f54d5e365871ffd3e05cfbc65d1b36a8be7c0bcbda30a1d43"}, + {file = "tree_sitter-0.20.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d933a942fde39876b99c36f12aa3764e4a555ae9366c10ce6cca8c16341c1bbf"}, + {file = "tree_sitter-0.20.4-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a7eec3b55135fe851a38fa248c9fd75fc3d58ceb6e1865b795e416e4d598c2a1"}, + {file = "tree_sitter-0.20.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc76225529ee14a53e84413480ce81ec3c44eaa0455c140e961c90ac3118ead"}, + {file = "tree_sitter-0.20.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccf0396e47efffc0b528959a8f2e2346a98297579f867e9e1834c2aad4be829c"}, + {file = "tree_sitter-0.20.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:a15fbabd3bc8e29c48289c156d743e69f5ec72bb125cf44f7adbdaa1937c3da6"}, + {file = "tree_sitter-0.20.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:36f8adf2126f496cf376b6e4b707cba061c25beb17841727eef6f0e083e53e1f"}, + {file = "tree_sitter-0.20.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:841efb40c116ab0a066924925409a8a4dcffeb39a151c0b2a1c2abe56ad4fb42"}, + {file = "tree_sitter-0.20.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2051e8a70fd8426f27a43dad71d11929a62ce30a9b1eb65bba0ed79e82481592"}, + {file = "tree_sitter-0.20.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:99a3c2824d4cfcffd9f961176891426bde2cb36ece5280c61480be93319c23c4"}, + {file = "tree_sitter-0.20.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:72830dc85a10430eca3d56739b7efcd7a05459c8d425f08c1aee6179ab7f13a9"}, + {file = "tree_sitter-0.20.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4992dd226055b6cd0a4f5661c66b799a73d3eff716302e0f7ab06594ee12d49f"}, + {file = "tree_sitter-0.20.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a66d95bbf92175cdc295d6d77f330942811f02e3aaf3fc64431cb749683b2f7d"}, + {file = "tree_sitter-0.20.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a25b1087e4f7825b2458dacf5f4b0be2938f78e850e822edca1ff4994b56081a"}, + {file = "tree_sitter-0.20.4.tar.gz", hash = "sha256:6adb123e2f3e56399bbf2359924633c882cc40ee8344885200bca0922f713be5"}, +] + +[[package]] +name = "tree-sitter-languages" +version = "1.10.2" +description = "Binary Python wheels for all tree sitter languages." +optional = false +python-versions = "*" +files = [ + {file = "tree_sitter_languages-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5580348f0b20233b1d5431fa178ccd3d07423ca4a3275df02a44608fd72344b9"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:103c7466644486b1e9e03850df46fc6aa12f13ca636c74f173270276220ac80b"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d13db84511c6f1a7dc40383b66deafa74dabd8b877e3d65ab253f3719eccafd6"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57adfa32be7e465b54aa72f915f6c78a2b66b227df4f656b5d4fbd1ca7a92b3f"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c6385e033e460ceb8f33f3f940335f422ef2b763700a04f0089391a68b56153"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:dfa3f38cc5381c5aba01dd7494f59b8a9050e82ff6e06e1233e3a0cbae297e3c"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9f195155acf47f8bc5de7cee46ecd07b2f5697f007ba89435b51ef4c0b953ea5"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2de330e2ac6d7426ca025a3ec0f10d5640c3682c1d0c7702e812dcfb44b58120"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-win32.whl", hash = "sha256:c9731cf745f135d9770eeba9bb4e2ff4dabc107b5ae9b8211e919f6b9100ea6d"}, + {file = "tree_sitter_languages-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:6dd75851c41d0c3c4987a9b7692d90fa8848706c23115669d8224ffd6571e357"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7eb7d7542b2091c875fe52719209631fca36f8c10fa66970d2c576ae6a1b8289"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6b41bcb00974b1c8a1800c7f1bb476a1d15a0463e760ee24872f2d53b08ee424"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f370cd7845c6c81df05680d5bd96db8a99d32b56f4728c5d05978911130a853"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1dc195c88ef4c72607e112a809a69190e096a2e5ebc6201548b3e05fdd169ad"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae34ac314a7170be24998a0f994c1ac80761d8d4bd126af27ee53a023d3b849"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:01b5742d5f5bd675489486b582bd482215880b26dde042c067f8265a6e925d9c"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ab1cbc46244d34fd16f21edaa20231b2a57f09f092a06ee3d469f3117e6eb954"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0b1149e7467a4e92b8a70e6005fe762f880f493cf811fc003554b29f04f5e7c8"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-win32.whl", hash = "sha256:049276343962f4696390ee555acc2c1a65873270c66a6cbe5cb0bca83bcdf3c6"}, + {file = "tree_sitter_languages-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:7f3fdd468a577f04db3b63454d939e26e360229b53c80361920aa1ebf2cd7491"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c0f4c8b2734c45859edc7fcaaeaab97a074114111b5ba51ab4ec7ed52104763c"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eecd3c1244ac3425b7a82ba9125b4ddb45d953bbe61de114c0334fd89b7fe782"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15db3c8510bc39a80147ee7421bf4782c15c09581c1dc2237ea89cefbd95b846"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92c6487a6feea683154d3e06e6db68c30e0ae749a7ce4ce90b9e4e46b78c85c7"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2f1cd1d1bdd65332f9c2b67d49dcf148cf1ded752851d159ac3e5ee4f4d260"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:976c8039165b8e12f17a01ddee9f4e23ec6e352b165ad29b44d2bf04e2fbe77e"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:dafbbdf16bf668a580902e1620f4baa1913e79438abcce721a50647564c687b9"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1aeabd3d60d6d276b73cd8f3739d595b1299d123cc079a317f1a5b3c5461e2ca"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-win32.whl", hash = "sha256:fab8ee641914098e8933b87ea3d657bea4dd00723c1ee7038b847b12eeeef4f5"}, + {file = "tree_sitter_languages-1.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e606430d736367e5787fa5a7a0c5a1ec9b85eded0b3596bbc0d83532a40810b"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:838d5b48a7ed7a17658721952c77fda4570d2a069f933502653b17e15a9c39c9"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:987b3c71b1d278c2889e018ee77b8ee05c384e2e3334dec798f8b611c4ab2d1e"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:faa00abcb2c819027df58472da055d22fa7dfcb77c77413d8500c32ebe24d38b"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e102fbbf02322d9201a86a814e79a9734ac80679fdb9682144479044f401a73"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8f0b87cf1a7b03174ba18dfd81582be82bfed26803aebfe222bd20e444aba003"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c0f1b9af9cb67f0b942b020da9fdd000aad5e92f2383ae0ba7a330b318d31912"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5a4076c921f7a4d31e643843de7dfe040b65b63a238a5aa8d31d93aabe6572aa"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-win32.whl", hash = "sha256:fa6391a3a5d83d32db80815161237b67d70576f090ce5f38339206e917a6f8bd"}, + {file = "tree_sitter_languages-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:55649d3f254585a064121513627cf9788c1cfdadbc5f097f33d5ba750685a4c0"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6f85d1edaa2d22d80d4ea5b6d12b95cf3644017b6c227d0d42854439e02e8893"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d78feed4a764ef3141cb54bf00fe94d514d8b6e26e09423e23b4c616fcb7938c"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1aca27531f9dd5308637d76643372856f0f65d0d28677d1bcf4211e8ed1ad0"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1031ea440dafb72237437d754eff8940153a3b051e3d18932ac25e75ce060a15"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99d3249beaef2c9fe558ecc9a97853c260433a849dcc68266d9770d196c2e102"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:59a4450f262a55148fb7e68681522f0c2a2f6b7d89666312a2b32708d8f416e1"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce74eab0e430370d5e15a96b6c6205f93405c177a8b2e71e1526643b2fb9bab1"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9b4dd2b6b3d24c85dffe33d6c343448869eaf4f41c19ddba662eb5d65d8808f4"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-win32.whl", hash = "sha256:92d734fb968fe3927a7596d9f0459f81a8fa7b07e16569476b28e27d0d753348"}, + {file = "tree_sitter_languages-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:46a13f7d38f2eeb75f7cf127d1201346093748c270d686131f0cbc50e42870a1"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f8c6a936ae99fdd8857e91f86c11c2f5e507ff30631d141d98132bb7ab2c8638"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c283a61423f49cdfa7b5a5dfbb39221e3bd126fca33479cd80749d4d7a6b7349"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e60be6bdcff923386a54a5edcb6ff33fc38ab0118636a762024fa2bc98de55"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c00069f9575bd831eabcce2cdfab158dde1ed151e7e5614c2d985ff7d78a7de1"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:475ff53203d8a43ccb19bb322fa2fb200d764001cc037793f1fadd714bb343da"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26fe7c9c412e4141dea87ea4b3592fd12e385465b5bdab106b0d5125754d4f60"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8fed27319957458340f24fe14daad467cd45021da034eef583519f83113a8c5e"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3657a491a7f96cc75a3568ddd062d25f3be82b6a942c68801a7b226ff7130181"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-win32.whl", hash = "sha256:33f7d584d01a7a3c893072f34cfc64ec031f3cfe57eebc32da2f8ac046e101a7"}, + {file = "tree_sitter_languages-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:1b944af3ee729fa70fc8ae82224a9ff597cdb63addea084e0ea2fa2b0ec39bb7"}, +] + +[package.dependencies] +tree-sitter = "*" + +[[package]] +name = "typeguard" +version = "4.1.5" +description = "Run-time type checker for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typeguard-4.1.5-py3-none-any.whl", hash = "sha256:8923e55f8873caec136c892c3bed1f676eae7be57cdb94819281b3d3bc9c0953"}, + {file = "typeguard-4.1.5.tar.gz", hash = "sha256:ea0a113bbc111bcffc90789ebb215625c963411f7096a7e9062d4e4630c155fd"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.7.0", markers = "python_version < \"3.12\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"] +test = ["coverage[toml] (>=7)", "mypy (>=1.2.0)", "pytest (>=7)"] + +[[package]] +name = "typer" +version = "0.9.0" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.6" +files = [ + {file = "typer-0.9.0-py3-none-any.whl", hash = "sha256:5d96d986a21493606a358cae4461bd8cdf83cbf33a5aa950ae629ca3b51467ee"}, + {file = "typer-0.9.0.tar.gz", hash = "sha256:50922fd79aea2f4751a8e0408ff10d2662bd0c8bbfa84755a699f3bada2978b2"}, +] + +[package.dependencies] +click = ">=7.1.1,<9.0.0" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] + +[[package]] +name = "types-python-dateutil" +version = "2.8.19.20240106" +description = "Typing stubs for python-dateutil" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-python-dateutil-2.8.19.20240106.tar.gz", hash = "sha256:1f8db221c3b98e6ca02ea83a58371b22c374f42ae5bbdf186db9c9a76581459f"}, + {file = "types_python_dateutil-2.8.19.20240106-py3-none-any.whl", hash = "sha256:efbbdc54590d0f16152fa103c9879c7d4a00e82078f6e2cf01769042165acaa2"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "typing-inspect" +version = "0.9.0" +description = "Runtime inspection utilities for typing module." +optional = false +python-versions = "*" +files = [ + {file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"}, + {file = "typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78"}, +] + +[package.dependencies] +mypy-extensions = ">=0.3.0" +typing-extensions = ">=3.7.4" + +[[package]] +name = "tzdata" +version = "2023.4" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, +] + +[[package]] +name = "umap-learn" +version = "0.5.5" +description = "Uniform Manifold Approximation and Projection" +optional = false +python-versions = "*" +files = [ + {file = "umap-learn-0.5.5.tar.gz", hash = "sha256:c54d607364413eade968b73ba07c8b3ea14412817f53cd07b6f720ac957293c4"}, +] + +[package.dependencies] +numba = ">=0.51.2" +numpy = ">=1.17" +pynndescent = ">=0.5" +scikit-learn = ">=0.22" +scipy = ">=1.3.1" +tqdm = "*" + +[package.extras] +parametric-umap = ["tensorflow (>=2.1)", "tensorflow-probability (>=0.10)"] +plot = ["bokeh", "colorcet", "datashader", "holoviews", "matplotlib", "pandas", "scikit-image", "seaborn"] +tbb = ["tbb (>=2019.0)"] + +[[package]] +name = "unstructured" +version = "0.12.3" +description = "A library that prepares raw documents for downstream ML tasks." +optional = false +python-versions = ">=3.9.0,<3.12" +files = [ + {file = "unstructured-0.12.3-py3-none-any.whl", hash = "sha256:14d606b84eaea2283a755d559ec507a7abbe10cf5c26a18d78bccac17a816a94"}, + {file = "unstructured-0.12.3.tar.gz", hash = "sha256:2b6c4e38f147170211df59d3d39585e834109832b9d8cd7a5e8db94b221a5a0b"}, +] + +[package.dependencies] +backoff = "*" +beautifulsoup4 = "*" +chardet = "*" +dataclasses-json = "*" +emoji = "*" +filetype = "*" +langdetect = "*" +lxml = "*" +nltk = "*" +numpy = "*" +python-iso639 = "*" +python-magic = "*" +rapidfuzz = "*" +requests = "*" +tabulate = "*" +typing-extensions = "*" +unstructured-client = ">=0.15.1" +wrapt = "*" + +[package.extras] +airtable = ["pyairtable"] +all-docs = ["markdown", "msg-parser", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pikepdf", "pypandoc", "pypdf", "python-docx", "python-pptx (<=0.6.23)", "unstructured-inference (==0.7.23)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] +azure = ["adlfs", "fsspec"] +azure-cognitive-search = ["azure-search-documents"] +bedrock = ["boto3", "langchain"] +biomed = ["bs4"] +box = ["boxfs", "fsspec"] +chroma = ["chromadb"] +confluence = ["atlassian-python-api"] +csv = ["pandas"] +databricks-volumes = ["databricks-sdk"] +delta-table = ["deltalake", "fsspec"] +discord = ["discord-py"] +doc = ["python-docx"] +docx = ["python-docx"] +dropbox = ["dropboxdrivefs", "fsspec"] +elasticsearch = ["elasticsearch"] +embed-huggingface = ["huggingface", "langchain", "sentence-transformers"] +epub = ["pypandoc"] +gcs = ["bs4", "fsspec", "gcsfs"] +github = ["pygithub (>1.58.0)"] +gitlab = ["python-gitlab"] +google-drive = ["google-api-python-client"] +hubspot = ["hubspot-api-client", "urllib3"] +huggingface = ["langdetect", "sacremoses", "sentencepiece", "torch", "transformers"] +image = ["onnx", "pdf2image", "pdfminer.six", "pikepdf", "pypdf", "unstructured-inference (==0.7.23)", "unstructured.pytesseract (>=0.3.12)"] +jira = ["atlassian-python-api"] +local-inference = ["markdown", "msg-parser", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pikepdf", "pypandoc", "pypdf", "python-docx", "python-pptx (<=0.6.23)", "unstructured-inference (==0.7.23)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] +md = ["markdown"] +mongodb = ["pymongo"] +msg = ["msg-parser"] +notion = ["htmlBuilder", "notion-client"] +odt = ["pypandoc", "python-docx"] +onedrive = ["Office365-REST-Python-Client", "bs4", "msal"] +openai = ["langchain", "openai", "tiktoken"] +opensearch = ["opensearch-py"] +org = ["pypandoc"] +outlook = ["Office365-REST-Python-Client", "msal"] +paddleocr = ["unstructured.paddleocr (==2.6.1.3)"] +pdf = ["onnx", "pdf2image", "pdfminer.six", "pikepdf", "pypdf", "unstructured-inference (==0.7.23)", "unstructured.pytesseract (>=0.3.12)"] +pinecone = ["pinecone-client (==2.2.4)"] +postgres = ["psycopg2-binary"] +ppt = ["python-pptx (<=0.6.23)"] +pptx = ["python-pptx (<=0.6.23)"] +qdrant = ["qdrant-client"] +reddit = ["praw"] +rst = ["pypandoc"] +rtf = ["pypandoc"] +s3 = ["fsspec", "s3fs"] +salesforce = ["simple-salesforce"] +sftp = ["fsspec", "paramiko"] +sharepoint = ["Office365-REST-Python-Client", "msal"] +slack = ["slack-sdk"] +tsv = ["pandas"] +weaviate = ["weaviate-client"] +wikipedia = ["wikipedia"] +xlsx = ["networkx", "openpyxl", "pandas", "xlrd"] + +[[package]] +name = "unstructured-client" +version = "0.17.0" +description = "Python Client SDK for Unstructured API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "unstructured-client-0.17.0.tar.gz", hash = "sha256:3b98affa4a47e370b7b0a1b8f0d889f20aa6d8925b2b108b60aa07cf54a04fb5"}, + {file = "unstructured_client-0.17.0-py3-none-any.whl", hash = "sha256:2b311f0f7304ac64986b0312431fe99945f95a6fa0c88ab91094570c932a9e19"}, +] + +[package.dependencies] +certifi = ">=2023.7.22" +charset-normalizer = ">=3.2.0" +dataclasses-json-speakeasy = ">=0.5.11" +idna = ">=3.4" +jsonpath-python = ">=1.0.6" +marshmallow = ">=3.19.0" +mypy-extensions = ">=1.0.0" +packaging = ">=23.1" +python-dateutil = ">=2.8.2" +requests = ">=2.31.0" +six = ">=1.16.0" +typing-extensions = ">=4.7.1" +typing-inspect = ">=0.9.0" +urllib3 = ">=1.26.18" + +[package.extras] +dev = ["pylint (==2.16.2)"] + +[[package]] +name = "uri-template" +version = "1.3.0" +description = "RFC 6570 URI Template Processor" +optional = false +python-versions = ">=3.7" +files = [ + {file = "uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7"}, + {file = "uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"}, +] + +[package.extras] +dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake8-commas", "flake8-comprehensions", "flake8-continuation", "flake8-datetimez", "flake8-docstrings", "flake8-import-order", "flake8-literal", "flake8-modern-annotations", "flake8-noqa", "flake8-pyproject", "flake8-requirements", "flake8-typechecking-import", "flake8-use-fstring", "mypy", "pep8-naming", "types-PyYAML"] + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "uvicorn" +version = "0.24.0.post1" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.8" +files = [ + {file = "uvicorn-0.24.0.post1-py3-none-any.whl", hash = "sha256:7c84fea70c619d4a710153482c0d230929af7bcf76c7bfa6de151f0a3a80121e"}, + {file = "uvicorn-0.24.0.post1.tar.gz", hash = "sha256:09c8e5a79dc466bdf28dead50093957db184de356fcdc48697bad3bde4c2588e"}, +] + +[package.dependencies] +click = ">=7.0" +colorama = {version = ">=0.4", optional = true, markers = "sys_platform == \"win32\" and extra == \"standard\""} +h11 = ">=0.8" +httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""} +python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} +uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + +[[package]] +name = "uvloop" +version = "0.19.0" +description = "Fast implementation of asyncio event loop on top of libuv" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de4313d7f575474c8f5a12e163f6d89c0a878bc49219641d49e6f1444369a90e"}, + {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5588bd21cf1fcf06bded085f37e43ce0e00424197e7c10e77afd4bbefffef428"}, + {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b1fd71c3843327f3bbc3237bedcdb6504fd50368ab3e04d0410e52ec293f5b8"}, + {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a05128d315e2912791de6088c34136bfcdd0c7cbc1cf85fd6fd1bb321b7c849"}, + {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cd81bdc2b8219cb4b2556eea39d2e36bfa375a2dd021404f90a62e44efaaf957"}, + {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f17766fb6da94135526273080f3455a112f82570b2ee5daa64d682387fe0dcd"}, + {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ce6b0af8f2729a02a5d1575feacb2a94fc7b2e983868b009d51c9a9d2149bef"}, + {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:31e672bb38b45abc4f26e273be83b72a0d28d074d5b370fc4dcf4c4eb15417d2"}, + {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:570fc0ed613883d8d30ee40397b79207eedd2624891692471808a95069a007c1"}, + {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5138821e40b0c3e6c9478643b4660bd44372ae1e16a322b8fc07478f92684e24"}, + {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:91ab01c6cd00e39cde50173ba4ec68a1e578fee9279ba64f5221810a9e786533"}, + {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:47bf3e9312f63684efe283f7342afb414eea4d3011542155c7e625cd799c3b12"}, + {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:da8435a3bd498419ee8c13c34b89b5005130a476bda1d6ca8cfdde3de35cd650"}, + {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:02506dc23a5d90e04d4f65c7791e65cf44bd91b37f24cfc3ef6cf2aff05dc7ec"}, + {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693049be9d36fef81741fddb3f441673ba12a34a704e7b4361efb75cf30befc"}, + {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7010271303961c6f0fe37731004335401eb9075a12680738731e9c92ddd96ad6"}, + {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5daa304d2161d2918fa9a17d5635099a2f78ae5b5960e742b2fcfbb7aefaa593"}, + {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7207272c9520203fea9b93843bb775d03e1cf88a80a936ce760f60bb5add92f3"}, + {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:78ab247f0b5671cc887c31d33f9b3abfb88d2614b84e4303f1a63b46c046c8bd"}, + {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:472d61143059c84947aa8bb74eabbace30d577a03a1805b77933d6bd13ddebbd"}, + {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45bf4c24c19fb8a50902ae37c5de50da81de4922af65baf760f7c0c42e1088be"}, + {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271718e26b3e17906b28b67314c45d19106112067205119dddbd834c2b7ce797"}, + {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:34175c9fd2a4bc3adc1380e1261f60306344e3407c20a4d684fd5f3be010fa3d"}, + {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e27f100e1ff17f6feeb1f33968bc185bf8ce41ca557deee9d9bbbffeb72030b7"}, + {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13dfdf492af0aa0a0edf66807d2b465607d11c4fa48f4a1fd41cbea5b18e8e8b"}, + {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e3d4e85ac060e2342ff85e90d0c04157acb210b9ce508e784a944f852a40e67"}, + {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca4956c9ab567d87d59d49fa3704cf29e37109ad348f2d5223c9bf761a332e7"}, + {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f467a5fd23b4fc43ed86342641f3936a68ded707f4627622fa3f82a120e18256"}, + {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:492e2c32c2af3f971473bc22f086513cedfc66a130756145a931a90c3958cb17"}, + {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2df95fca285a9f5bfe730e51945ffe2fa71ccbfdde3b0da5772b4ee4f2e770d5"}, + {file = "uvloop-0.19.0.tar.gz", hash = "sha256:0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd"}, +] + +[package.extras] +docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] + +[[package]] +name = "wandb" +version = "0.16.1" +description = "A CLI and library for interacting with the Weights & Biases API." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wandb-0.16.1-py3-none-any.whl", hash = "sha256:1d7423f92520984585bae9693bb637ae08d3e0c1d75ad4b34215bc44431f114c"}, + {file = "wandb-0.16.1.tar.gz", hash = "sha256:ffe6e8dd8cc8fcd72010c1246fb3d6d226b37c4f111f3f94308a1c0ae28a2fec"}, +] + +[package.dependencies] +appdirs = ">=1.4.3" +Click = ">=7.1,<8.0.0 || >8.0.0" +docker-pycreds = ">=0.4.0" +GitPython = ">=1.0.0,<3.1.29 || >3.1.29" +protobuf = {version = ">=3.19.0,<4.21.0 || >4.21.0,<5", markers = "python_version > \"3.9\" or sys_platform != \"linux\""} +psutil = ">=5.0.0" +PyYAML = "*" +requests = ">=2.0.0,<3" +sentry-sdk = ">=1.0.0" +setproctitle = "*" +setuptools = "*" + +[package.extras] +async = ["httpx (>=0.23.0)"] +aws = ["boto3"] +azure = ["azure-identity", "azure-storage-blob"] +core = ["wandb-core (>=0.17.0b2)"] +gcp = ["google-cloud-storage"] +kubeflow = ["google-cloud-storage", "kubernetes", "minio", "sh"] +launch = ["PyYAML (>=6.0.0)", "awscli", "azure-containerregistry", "azure-identity", "azure-storage-blob", "boto3", "botocore", "chardet", "google-auth", "google-cloud-aiplatform", "google-cloud-artifact-registry", "google-cloud-compute", "google-cloud-storage", "iso8601", "kubernetes", "kubernetes-asyncio", "nbconvert", "nbformat", "optuna", "typing-extensions"] +media = ["bokeh", "moviepy", "numpy", "pillow", "plotly", "rdkit-pypi", "soundfile"] +models = ["cloudpickle"] +perf = ["orjson"] +sweeps = ["sweeps (>=0.2.0)"] + +[[package]] +name = "watchfiles" +version = "0.21.0" +description = "Simple, modern and high performance file watching and code reload in python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "watchfiles-0.21.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:27b4035013f1ea49c6c0b42d983133b136637a527e48c132d368eb19bf1ac6aa"}, + {file = "watchfiles-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c81818595eff6e92535ff32825f31c116f867f64ff8cdf6562cd1d6b2e1e8f3e"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6c107ea3cf2bd07199d66f156e3ea756d1b84dfd43b542b2d870b77868c98c03"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d9ac347653ebd95839a7c607608703b20bc07e577e870d824fa4801bc1cb124"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5eb86c6acb498208e7663ca22dbe68ca2cf42ab5bf1c776670a50919a56e64ab"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f564bf68404144ea6b87a78a3f910cc8de216c6b12a4cf0b27718bf4ec38d303"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d0f32ebfaa9c6011f8454994f86108c2eb9c79b8b7de00b36d558cadcedaa3d"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d45d9b699ecbac6c7bd8e0a2609767491540403610962968d258fd6405c17c"}, + {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:aff06b2cac3ef4616e26ba17a9c250c1fe9dd8a5d907d0193f84c499b1b6e6a9"}, + {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d9792dff410f266051025ecfaa927078b94cc7478954b06796a9756ccc7e14a9"}, + {file = "watchfiles-0.21.0-cp310-none-win32.whl", hash = "sha256:214cee7f9e09150d4fb42e24919a1e74d8c9b8a9306ed1474ecaddcd5479c293"}, + {file = "watchfiles-0.21.0-cp310-none-win_amd64.whl", hash = "sha256:1ad7247d79f9f55bb25ab1778fd47f32d70cf36053941f07de0b7c4e96b5d235"}, + {file = "watchfiles-0.21.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:668c265d90de8ae914f860d3eeb164534ba2e836811f91fecc7050416ee70aa7"}, + {file = "watchfiles-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a23092a992e61c3a6a70f350a56db7197242f3490da9c87b500f389b2d01eef"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e7941bbcfdded9c26b0bf720cb7e6fd803d95a55d2c14b4bd1f6a2772230c586"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11cd0c3100e2233e9c53106265da31d574355c288e15259c0d40a4405cbae317"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d78f30cbe8b2ce770160d3c08cff01b2ae9306fe66ce899b73f0409dc1846c1b"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6674b00b9756b0af620aa2a3346b01f8e2a3dc729d25617e1b89cf6af4a54eb1"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd7ac678b92b29ba630d8c842d8ad6c555abda1b9ef044d6cc092dacbfc9719d"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c873345680c1b87f1e09e0eaf8cf6c891b9851d8b4d3645e7efe2ec20a20cc7"}, + {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:49f56e6ecc2503e7dbe233fa328b2be1a7797d31548e7a193237dcdf1ad0eee0"}, + {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:02d91cbac553a3ad141db016e3350b03184deaafeba09b9d6439826ee594b365"}, + {file = "watchfiles-0.21.0-cp311-none-win32.whl", hash = "sha256:ebe684d7d26239e23d102a2bad2a358dedf18e462e8808778703427d1f584400"}, + {file = "watchfiles-0.21.0-cp311-none-win_amd64.whl", hash = "sha256:4566006aa44cb0d21b8ab53baf4b9c667a0ed23efe4aaad8c227bfba0bf15cbe"}, + {file = "watchfiles-0.21.0-cp311-none-win_arm64.whl", hash = "sha256:c550a56bf209a3d987d5a975cdf2063b3389a5d16caf29db4bdddeae49f22078"}, + {file = "watchfiles-0.21.0-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:51ddac60b96a42c15d24fbdc7a4bfcd02b5a29c047b7f8bf63d3f6f5a860949a"}, + {file = "watchfiles-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:511f0b034120cd1989932bf1e9081aa9fb00f1f949fbd2d9cab6264916ae89b1"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cfb92d49dbb95ec7a07511bc9efb0faff8fe24ef3805662b8d6808ba8409a71a"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f92944efc564867bbf841c823c8b71bb0be75e06b8ce45c084b46411475a915"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:642d66b75eda909fd1112d35c53816d59789a4b38c141a96d62f50a3ef9b3360"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d23bcd6c8eaa6324fe109d8cac01b41fe9a54b8c498af9ce464c1aeeb99903d6"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18d5b4da8cf3e41895b34e8c37d13c9ed294954907929aacd95153508d5d89d7"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b8d1eae0f65441963d805f766c7e9cd092f91e0c600c820c764a4ff71a0764c"}, + {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1fd9a5205139f3c6bb60d11f6072e0552f0a20b712c85f43d42342d162be1235"}, + {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a1e3014a625bcf107fbf38eece0e47fa0190e52e45dc6eee5a8265ddc6dc5ea7"}, + {file = "watchfiles-0.21.0-cp312-none-win32.whl", hash = "sha256:9d09869f2c5a6f2d9df50ce3064b3391d3ecb6dced708ad64467b9e4f2c9bef3"}, + {file = "watchfiles-0.21.0-cp312-none-win_amd64.whl", hash = "sha256:18722b50783b5e30a18a8a5db3006bab146d2b705c92eb9a94f78c72beb94094"}, + {file = "watchfiles-0.21.0-cp312-none-win_arm64.whl", hash = "sha256:a3b9bec9579a15fb3ca2d9878deae789df72f2b0fdaf90ad49ee389cad5edab6"}, + {file = "watchfiles-0.21.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:4ea10a29aa5de67de02256a28d1bf53d21322295cb00bd2d57fcd19b850ebd99"}, + {file = "watchfiles-0.21.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:40bca549fdc929b470dd1dbfcb47b3295cb46a6d2c90e50588b0a1b3bd98f429"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9b37a7ba223b2f26122c148bb8d09a9ff312afca998c48c725ff5a0a632145f7"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec8c8900dc5c83650a63dd48c4d1d245343f904c4b64b48798c67a3767d7e165"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ad3fe0a3567c2f0f629d800409cd528cb6251da12e81a1f765e5c5345fd0137"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d353c4cfda586db2a176ce42c88f2fc31ec25e50212650c89fdd0f560ee507b"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:83a696da8922314ff2aec02987eefb03784f473281d740bf9170181829133765"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a03651352fc20975ee2a707cd2d74a386cd303cc688f407296064ad1e6d1562"}, + {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3ad692bc7792be8c32918c699638b660c0de078a6cbe464c46e1340dadb94c19"}, + {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06247538e8253975bdb328e7683f8515ff5ff041f43be6c40bff62d989b7d0b0"}, + {file = "watchfiles-0.21.0-cp38-none-win32.whl", hash = "sha256:9a0aa47f94ea9a0b39dd30850b0adf2e1cd32a8b4f9c7aa443d852aacf9ca214"}, + {file = "watchfiles-0.21.0-cp38-none-win_amd64.whl", hash = "sha256:8d5f400326840934e3507701f9f7269247f7c026d1b6cfd49477d2be0933cfca"}, + {file = "watchfiles-0.21.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:7f762a1a85a12cc3484f77eee7be87b10f8c50b0b787bb02f4e357403cad0c0e"}, + {file = "watchfiles-0.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6e9be3ef84e2bb9710f3f777accce25556f4a71e15d2b73223788d528fcc2052"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4c48a10d17571d1275701e14a601e36959ffada3add8cdbc9e5061a6e3579a5d"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c889025f59884423428c261f212e04d438de865beda0b1e1babab85ef4c0f01"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66fac0c238ab9a2e72d026b5fb91cb902c146202bbd29a9a1a44e8db7b710b6f"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4a21f71885aa2744719459951819e7bf5a906a6448a6b2bbce8e9cc9f2c8128"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c9198c989f47898b2c22201756f73249de3748e0fc9de44adaf54a8b259cc0c"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f57c4461cd24fda22493109c45b3980863c58a25b8bec885ca8bea6b8d4b28"}, + {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:853853cbf7bf9408b404754b92512ebe3e3a83587503d766d23e6bf83d092ee6"}, + {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d5b1dc0e708fad9f92c296ab2f948af403bf201db8fb2eb4c8179db143732e49"}, + {file = "watchfiles-0.21.0-cp39-none-win32.whl", hash = "sha256:59137c0c6826bd56c710d1d2bda81553b5e6b7c84d5a676747d80caf0409ad94"}, + {file = "watchfiles-0.21.0-cp39-none-win_amd64.whl", hash = "sha256:6cb8fdc044909e2078c248986f2fc76f911f72b51ea4a4fbbf472e01d14faa58"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:ab03a90b305d2588e8352168e8c5a1520b721d2d367f31e9332c4235b30b8994"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:927c589500f9f41e370b0125c12ac9e7d3a2fd166b89e9ee2828b3dda20bfe6f"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bd467213195e76f838caf2c28cd65e58302d0254e636e7c0fca81efa4a2e62c"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02b73130687bc3f6bb79d8a170959042eb56eb3a42df3671c79b428cd73f17cc"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:08dca260e85ffae975448e344834d765983237ad6dc308231aa16e7933db763e"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3ccceb50c611c433145502735e0370877cced72a6c70fd2410238bcbc7fe51d8"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57d430f5fb63fea141ab71ca9c064e80de3a20b427ca2febcbfcef70ff0ce895"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dd5fad9b9c0dd89904bbdea978ce89a2b692a7ee8a0ce19b940e538c88a809c"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:be6dd5d52b73018b21adc1c5d28ac0c68184a64769052dfeb0c5d9998e7f56a2"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b3cab0e06143768499384a8a5efb9c4dc53e19382952859e4802f294214f36ec"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6ed10c2497e5fedadf61e465b3ca12a19f96004c15dcffe4bd442ebadc2d85"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43babacef21c519bc6631c5fce2a61eccdfc011b4bcb9047255e9620732c8097"}, + {file = "watchfiles-0.21.0.tar.gz", hash = "sha256:c76c635fabf542bb78524905718c39f736a98e5ab25b23ec6d4abede1a85a6a3"}, +] + +[package.dependencies] +anyio = ">=3.0.0" + +[[package]] +name = "wcwidth" +version = "0.2.13" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, +] + +[[package]] +name = "weave" +version = "0.31.0" +description = "A toolkit for building composable interactive data driven applications." +optional = false +python-versions = ">=3.9" +files = [ + {file = "weave-0.31.0-py3-none-any.whl", hash = "sha256:150a401092b90ff7f7c3c2a7d656c19a512b2ad82795692eaa3cbf927a63c074"}, + {file = "weave-0.31.0.tar.gz", hash = "sha256:88218d8f4337f57258ff3b7e1e6f8f68fbb5d81a25003bddc26f8431f58881b1"}, +] + +[package.dependencies] +aiofiles = ">=22.1.0" +aiohttp = ">=3.8.3" +aioprocessing = ">=2.0.1" +analytics-python = ">=1.2.9" +black = ">=22.3.0" +Click = "8.1.3" +faiss-cpu = "*" +flask = ">=2.1" +flask-cors = ">=3.0.10" +gql = {version = ">=3.4.1", extras = ["requests"]} +graphql-core = ">3" +ipynbname = ">=2021.3.2" +ipython = ">=7.34" +janus = ">=1.0.0" +notebook = ">=6.4.8" +numpy = ">=1.21" +objgraph = ">=3.6.0" +openai = ">=1.0.0" +pandas = ">=1.5.3" +pillow = ">=10.0.1" +pyarrow = ">=13.0.0" +pytest-asyncio = "<=0.21.1" +python-json-logger = ">=2.0.4" +scikit-learn = ">=1.2.1" +sentry-sdk = "<1.29.0" +tabulate = "*" +tiktoken = ">=0.4.0" +typeguard = ">=4.1.3" +umap-learn = ">=0.5.3" +wandb = ">=0.15.5" + +[package.extras] +datadog = ["datadog (==0.44.0)", "ddtrace (==1.7.5)"] +ecosystem = ["bertviz (>=1.4.0)", "datasets (>=2.9.0)", "faiss-cpu (>=1.7.4)", "h5py (>=3.8.0)", "hdbscan (>=0.8.27)", "langchain (>=0.0.132)", "matplotlib (>3,!=3.5)", "openai (>=1.0.0)", "pillow (>=10.0.1)", "plotly (>=5.13.0)", "replicate (>=0.4.0)", "shap (>=0.41.0)", "spacy (>=3.0.0,<4.0.0)", "sqlalchemy (>=2.0.1)", "tiktoken (>=0.4.0)", "torch (>=1.13.1)", "torchvision (>=0.14.1)", "transformers (<4.21)", "xgboost (>=1.7.3)"] +examples = ["bertviz (>=1.4.0)", "datasets (>=2.9.0)", "faiss-cpu (>=1.7.4)", "h5py (>=3.8.0)", "hdbscan (>=0.8.27)", "langchain (>=0.0.132)", "matplotlib (>3,!=3.5)", "openai (>=1.0.0)", "pillow (>=10.0.1)", "plotly (>=5.13.0)", "replicate (>=0.4.0)", "shap (>=0.41.0)", "spacy (>=3.0.0,<4.0.0)", "sqlalchemy (>=2.0.1)", "tiktoken (>=0.4.0)", "torch (>=1.13.1)", "torchvision (>=0.14.1)", "transformers (<4.21)", "xgboost (>=1.7.3)"] +modal = ["modal", "python-dotenv"] + +[[package]] +name = "webcolors" +version = "1.13" +description = "A library for working with the color formats defined by HTML and CSS." +optional = false +python-versions = ">=3.7" +files = [ + {file = "webcolors-1.13-py3-none-any.whl", hash = "sha256:29bc7e8752c0a1bd4a1f03c14d6e6a72e93d82193738fa860cbff59d0fcc11bf"}, + {file = "webcolors-1.13.tar.gz", hash = "sha256:c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a"}, +] + +[package.extras] +docs = ["furo", "sphinx", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-notfound-page", "sphinxext-opengraph"] +tests = ["pytest", "pytest-cov"] + +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +optional = false +python-versions = "*" +files = [ + {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, + {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, +] + +[[package]] +name = "websocket-client" +version = "1.7.0" +description = "WebSocket client for Python with low level API options" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, + {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, +] + +[package.extras] +docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + +[[package]] +name = "websockets" +version = "12.0" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websockets-12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374"}, + {file = "websockets-12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be"}, + {file = "websockets-12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603"}, + {file = "websockets-12.0-cp310-cp310-win32.whl", hash = "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f"}, + {file = "websockets-12.0-cp310-cp310-win_amd64.whl", hash = "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f"}, + {file = "websockets-12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, + {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, + {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc"}, + {file = "websockets-12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113"}, + {file = "websockets-12.0-cp312-cp312-win32.whl", hash = "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d"}, + {file = "websockets-12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2"}, + {file = "websockets-12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7"}, + {file = "websockets-12.0-cp38-cp38-win32.whl", hash = "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62"}, + {file = "websockets-12.0-cp38-cp38-win_amd64.whl", hash = "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28"}, + {file = "websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9"}, + {file = "websockets-12.0-cp39-cp39-win32.whl", hash = "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6"}, + {file = "websockets-12.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8"}, + {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, + {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, + {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, + {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, + {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, + {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, + {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, + {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, +] + +[[package]] +name = "werkzeug" +version = "3.0.1" +description = "The comprehensive WSGI web application library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "werkzeug-3.0.1-py3-none-any.whl", hash = "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10"}, + {file = "werkzeug-3.0.1.tar.gz", hash = "sha256:507e811ecea72b18a404947aded4b3390e1db8f826b494d76550ef45bb3b1dcc"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog (>=2.3)"] + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zenpy" +version = "2.0.46" +description = "Python wrapper for the Zendesk API" +optional = false +python-versions = "*" +files = [ + {file = "zenpy-2.0.46.tar.gz", hash = "sha256:8a4ebd34f5fb288e35dd735c66c863c64434d9aebcb5dd89581808d2bc14fa58"}, +] + +[package.dependencies] +cachetools = ">=3.1.0" +python-dateutil = ">=2.7.5" +pytz = ">=2018.9" +requests = ">=2.14.2" +six = ">=1.14.0" + +[[package]] +name = "zipp" +version = "3.17.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"}, + {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] + +[metadata] +lock-version = "2.0" +python-versions = ">=3.10.0,<3.12" +content-hash = "32552824ffb806b78f2b60145d4cae8f0688b1ccc46896c51852cc52913c8447" diff --git a/pyproject.toml b/pyproject.toml index f40dd2e..7e59aa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,19 +15,17 @@ numpy = "^1.26.1" wandb = "<=0.16.1" tiktoken = "^0.5.1" pandas = "^2.1.2" -unstructured = "^0.10.28" +unstructured = "^0.12.3" pydantic-settings = "^2.0.3" gitpython = "^3.1.40" giturlparse = "^0.12.0" scikit-learn = "^1.3.2" python-dotenv = "^1.0.0" -faiss-cpu = "^1.7.4" slack-bolt = "^1.18.0" slack-sdk = "^3.21.3" discord = "^2.3.2" markdown = "^3.5.1" -fastapi = "^0.104.1" -llama-index = "^0.9.40" +fastapi = "^0.109.2" tree-sitter-languages = "^1.7.1" cohere = "^4.32" markdownify = "^0.11.6" @@ -36,16 +34,14 @@ zenpy = "^2.0.46" openai = "^1.3.2" weave = "^0.31.0" colorlog = "^6.8.0" -rank-bm25 = "^0.2.2" litellm = "^1.15.1" google-cloud-bigquery = "^3.14.1" db-dtypes = "^1.2.0" python-frontmatter = "^1.1.0" pymdown-extensions = "^10.5" -instructor = "^0.4.5" -langchain-community = "^0.0.11" -langchain = "^0.1.0" -langchain-openai = "^0.0.2" +langchain = "^0.1.5" +langchain-openai = "^0.0.5" +chromadb = "^0.4.22" [tool.poetry.dev-dependencies] #fasttext = {git = "https://github.com/cfculhane/fastText"} # FastText doesn't come with pybind11 and we need to use this workaround. diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py new file mode 100644 index 0000000..91790f2 --- /dev/null +++ b/src/wandbot/chat/rag.py @@ -0,0 +1,60 @@ +from operator import itemgetter + +from langchain_core.runnables import RunnableParallel, RunnableLambda + +from wandbot.chat.response_synthesis import load_response_synthesizer_chain +from wandbot.ingestion.config import VectorStoreConfig +from wandbot.query_handler.query_enhancer import load_query_enhancement_chain +from wandbot.retriever.base import ( + load_vector_store_from_config, + load_retriever_with_options, +) +from wandbot.retriever.fusion import load_fusion_retriever_chain + + +def load_rag_chain(model, fallback_model, lang_detect_path, vector_store_path): + fallback_query_enhancer_chain = load_query_enhancement_chain( + fallback_model, lang_detect_path + ) + query_enhancer_chain = load_query_enhancement_chain( + model, lang_detect_path + ).with_fallbacks([fallback_query_enhancer_chain]) + + vectorstore_config = VectorStoreConfig(persist_dir=vector_store_path) + print(vectorstore_config.persist_dir.resolve()) + vectorstore = load_vector_store_from_config(vectorstore_config) + base_retriever = load_retriever_with_options( + vectorstore, search_type="similarity" + ) + + fallback_response_synthesis_chain = load_response_synthesizer_chain( + fallback_model + ) + response_synthesis_chain = load_response_synthesizer_chain( + model + ).with_fallbacks([fallback_response_synthesis_chain]) + + ranked_retrieval_chain = load_fusion_retriever_chain( + base_retriever, + ) + + rag_chain = ( + RunnableParallel( + query=query_enhancer_chain.with_config( + {"run_name": "query_enhancer"} + ) + ) + | RunnableParallel( + query=itemgetter("query"), + context=lambda x: itemgetter("query") | ranked_retrieval_chain, + ).with_config({"run_name": "retrieval"}) + | RunnableParallel( + query=itemgetter("query"), + context=RunnableLambda( + lambda x: [p.page_content for p in x["context"]] + ), + result=response_synthesis_chain, + ).with_config({"run_name": "response_synthesis"}) + ) + + return rag_chain diff --git a/src/wandbot/ingestion/__main__.py b/src/wandbot/ingestion/__main__.py index b911ab0..75402ac 100644 --- a/src/wandbot/ingestion/__main__.py +++ b/src/wandbot/ingestion/__main__.py @@ -1,6 +1,6 @@ import os -from wandbot.ingestion import vectorstores +from wandbot.ingestion import vectorstores, preprocess_data from wandbot.utils import get_logger logger = get_logger(__name__) @@ -11,8 +11,12 @@ def main(): entity = os.environ.get("WANDB_ENTITY", "wandbot") # raw_artifact = prepare_data.load(project, entity) - raw_artifact = "wandbot/wandbot-dev/raw_dataset:v28" - vectorstore_artifact = vectorstores.load(project, entity, raw_artifact) + raw_artifact = "wandbot/wandbot-dev/raw_dataset:v39" + preprocessed_artifact = preprocess_data.load(project, entity, raw_artifact) + # preprocessed_artifact = "wandbot/wandbot-dev/transformed_data:latest" + vectorstore_artifact = vectorstores.load( + project, entity, preprocessed_artifact + ) # TODO: include ingestion report # create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) print(vectorstore_artifact) diff --git a/src/wandbot/ingestion/config.py b/src/wandbot/ingestion/config.py index 1e9b90c..398c8e2 100644 --- a/src/wandbot/ingestion/config.py +++ b/src/wandbot/ingestion/config.py @@ -10,13 +10,15 @@ data_store_config = DataStoreConfig() docodile_english_store_config = DocodileEnglishStoreConfig() """ + import datetime import pathlib -from typing import List, Optional, Union +from typing import List, Optional from urllib.parse import urlparse from pydantic import BaseModel, Field, model_validator from pydantic_settings import BaseSettings + from wandbot.utils import get_logger logger = get_logger(__name__) @@ -31,20 +33,23 @@ class DataSource(BaseSettings): repo_path: str = "" local_path: Optional[pathlib.Path] = None base_path: Optional[str] = "" - file_pattern: Union[str, List[str]] = "*.*" + file_patterns: List[str] = ["*.*"] is_git_repo: bool = False git_id_file: Optional[pathlib.Path] = Field(None, env="WANDBOT_GIT_ID_FILE") class DataStoreConfig(BaseModel): name: str = "docstore" + source_type: str = "" data_source: DataSource = DataSource() docstore_dir: pathlib.Path = pathlib.Path("docstore") @model_validator(mode="after") def _set_cache_paths(cls, values: "DataStoreConfig") -> "DataStoreConfig": values.docstore_dir = ( - values.data_source.cache_dir / values.name / values.docstore_dir + values.data_source.cache_dir + / "_".join(values.name.split()) + / values.docstore_dir ) data_source = values.data_source @@ -55,7 +60,9 @@ def _set_cache_paths(cls, values: "DataStoreConfig") -> "DataStoreConfig": local_path = urlparse(data_source.repo_path).path.split("/")[-1] if not data_source.local_path: data_source.local_path = ( - data_source.cache_dir / values.name / local_path + data_source.cache_dir + / "_".join(values.name.split()) + / local_path ) if data_source.is_git_repo: if data_source.git_id_file is None: @@ -72,134 +79,145 @@ def _set_cache_paths(cls, values: "DataStoreConfig") -> "DataStoreConfig": class DocodileEnglishStoreConfig(DataStoreConfig): - name: str = "docodile_store" + name: str = "English Documentation" + source_type: str = "documentation" data_source: DataSource = DataSource( remote_path="https://docs.wandb.ai/", repo_path="https://github.com/wandb/docodile", base_path="docs", - file_pattern="*.md", + file_patterns=["*.md"], is_git_repo=True, ) language: str = "en" - docstore_dir: pathlib.Path = pathlib.Path("docodile_en") + docstore_dir: pathlib.Path = pathlib.Path("wandb_documentation_en") class DocodileJapaneseStoreConfig(DataStoreConfig): - name: str = "docodile_store" + name: str = "Japanese Documentation" + source_type: str = "documentation" data_source: DataSource = DataSource( remote_path="https://docs.wandb.ai/ja/", repo_path="https://github.com/wandb/docodile", base_path="i18n/ja/docusaurus-plugin-content-docs/current", - file_pattern="*.md", + file_patterns=["*.md"], is_git_repo=True, ) language: str = "ja" - docstore_dir: pathlib.Path = pathlib.Path("docodile_ja") + docstore_dir: pathlib.Path = pathlib.Path("wandb_documentation_ja") class ExampleCodeStoreConfig(DataStoreConfig): - name: str = "examples_store" + name: str = "Examples code" + source_type: str = "code" data_source: DataSource = DataSource( remote_path="https://github.com/wandb/examples/tree/master/", repo_path="https://github.com/wandb/examples", base_path="examples", - file_pattern="*.py", + file_patterns=["*.py"], is_git_repo=True, ) docstore_dir: pathlib.Path = pathlib.Path("wandb_examples_code") class ExampleNotebookStoreConfig(DataStoreConfig): - name: str = "examples_store" + name: str = "Examples Notebooks" + source_type: str = "notebook" data_source: DataSource = DataSource( remote_path="https://github.com/wandb/examples/tree/master/", repo_path="https://github.com/wandb/examples", base_path="colabs", - file_pattern="*.ipynb", + file_patterns=["*.ipynb"], is_git_repo=True, ) docstore_dir: pathlib.Path = pathlib.Path("wandb_examples_colab") class SDKCodeStoreConfig(DataStoreConfig): - name: str = "sdk_code_store" + name: str = "Wandb SDK code" + source_type: str = "code" data_source: DataSource = DataSource( remote_path="https://github.com/wandb/wandb/tree/main/", repo_path="https://github.com/wandb/wandb", base_path="wandb", - file_pattern="*.py", + file_patterns=["*.py"], is_git_repo=True, ) docstore_dir: pathlib.Path = pathlib.Path("wandb_sdk_code") class SDKTestsStoreConfig(DataStoreConfig): - name: str = "sdk_code_store" + name: str = "Wandb SDK tests" + source_type: str = "code" data_source: DataSource = DataSource( remote_path="https://github.com/wandb/wandb/tree/main/", repo_path="https://github.com/wandb/wandb", base_path="tests", - file_pattern="*.py", + file_patterns=["*.py"], is_git_repo=True, ) docstore_dir: pathlib.Path = pathlib.Path("wandb_sdk_tests") class WeaveCodeStoreConfig(DataStoreConfig): - name: str = "weave_code_store" + name: str = "Weave SDK code" + source_type: str = "code" data_source: DataSource = DataSource( remote_path="https://github.com/wandb/weave/tree/master/", repo_path="https://github.com/wandb/weave", base_path="weave", - file_pattern=["*.py", "*.ipynb"], + file_patterns=["*.py", "*.ipynb"], is_git_repo=True, ) docstore_dir: pathlib.Path = pathlib.Path("weave_sdk_code") class WeaveExamplesStoreConfig(DataStoreConfig): - name: str = "weave_code_store" + name: str = "Weave Examples" + source_type: str = "code" data_source: DataSource = DataSource( remote_path="https://github.com/wandb/weave/tree/master/", repo_path="https://github.com/wandb/weave", base_path="examples", - file_pattern=["*.py", "*.ipynb"], + file_patterns=["*.py", "*.ipynb"], is_git_repo=True, ) docstore_dir: pathlib.Path = pathlib.Path("weave_examples") class WandbEduCodeStoreConfig(DataStoreConfig): - name: str = "wandb_edu_code_store" + name: str = "Wandb Edu code" + source_type: str = "code" data_source: DataSource = DataSource( remote_path="https://github.com/wandb/edu/tree/main/", repo_path="https://github.com/wandb/edu", base_path="", - file_pattern=["*.py", "*.ipynb", ".*md"], + file_patterns=["*.py", "*.ipynb", "*.md"], is_git_repo=True, ) docstore_dir: pathlib.Path = pathlib.Path("wandb_edu_code") class WeaveJsStoreConfig(DataStoreConfig): - name: str = "weave_code_store" + name: str = "Weave JS code" + source_type: str = "code" data_source: DataSource = DataSource( remote_path="https://github.com/wandb/weave/tree/master/", repo_path="https://github.com/wandb/weave", base_path="weave-js", - file_pattern=["*.js", "*.ts"], + file_patterns=["*.js", "*.ts"], is_git_repo=True, ) docstore_dir: pathlib.Path = pathlib.Path("weave_js") class FCReportsStoreConfig(DataStoreConfig): - name: str = "fc_reports_store" + name: str = "FC Reports" + source_type: str = "report" data_source: DataSource = DataSource( remote_path="wandb-production", repo_path="", base_path="reports", - file_pattern=["*.json"], + file_patterns=["*.json"], is_git_repo=False, ) docstore_dir: pathlib.Path = pathlib.Path("fc_reports") @@ -226,6 +244,4 @@ class VectorStoreConfig(BaseSettings): embeddings_model: str = "text-embedding-3-small" embedding_dim: int = 512 persist_dir: pathlib.Path = pathlib.Path("data/cache/vectorstore") - chat_model_name: str = "gpt-3.5-turbo-1106" - temperature: float = 0.1 - max_retries: int = 3 + batch_size: int = 256 diff --git a/src/wandbot/ingestion/prepare_data.py b/src/wandbot/ingestion/prepare_data.py index 18ee9c9..733745a 100644 --- a/src/wandbot/ingestion/prepare_data.py +++ b/src/wandbot/ingestion/prepare_data.py @@ -17,6 +17,7 @@ import json import os import pathlib +from multiprocessing import Pool, cpu_count from typing import Any, Dict, Iterator, List from urllib.parse import urljoin, urlparse @@ -28,7 +29,7 @@ from langchain_community.document_loaders import TextLoader from langchain_community.document_loaders.base import BaseLoader from nbconvert import MarkdownExporter -from tqdm import tqdm + from wandbot.ingestion.config import ( DataStoreConfig, DocodileEnglishStoreConfig, @@ -43,12 +44,11 @@ WeaveExamplesStoreConfig, ) from wandbot.ingestion.utils import ( - EXTENSION_MAP, clean_contents, extract_frontmatter, fetch_git_repo, ) -from wandbot.utils import FastTextLangDetect, get_logger +from wandbot.utils import get_logger logger = get_logger(__name__) @@ -102,12 +102,12 @@ def _get_local_paths(self): ) local_paths = [] - file_patterns = ( - [self.config.data_source.file_pattern] - if isinstance(self.config.data_source.file_pattern, str) - else self.config.data_source.file_pattern - ) - for file_pattern in file_patterns: + # file_patterns = ( + # [self.config.data_source.file_pattern] + # if isinstance(self.config.data_source.file_pattern, str) + # else self.config.data_source.file_pattern + # ) + for file_pattern in self.config.data_source.file_patterns: local_paths.extend( list( ( @@ -267,6 +267,7 @@ def lazy_load( document.metadata["tags"] = self.extract_tags( document.metadata["source"] ) + document.metadata["source_type"] = self.config.source_type yield document except Exception as e: logger.warning( @@ -275,31 +276,6 @@ def lazy_load( class CodeDataLoader(DataLoader): - @staticmethod - def extract_tags(source_url: str) -> List[str]: - """Extracts the tags from a source url. - - Args: - source_url: The URL of the file. - - Returns: - The extracted tags. - """ - parts = list(filter(lambda x: x, urlparse(source_url).path.split("/"))) - tree_slug = parts.index("tree") - parts = parts[tree_slug + 2 :] - parts_mapper = {"__init__.py": [], "__main__.py": []} - tags = [] - for part in parts: - if part in parts_mapper: - tags.extend(parts_mapper.get(part, [])) - else: - part = part.replace("-", " ").replace("_", " ") - tags.append(part) - tags = [tag.split(".")[0] for tag in tags] - tags = list(set([tag.title() for tag in tags])) - return tags + ["Code"] - def lazy_load(self) -> Iterator[Document]: """A lazy loader for code documents. @@ -343,13 +319,28 @@ def lazy_load(self) -> Iterator[Document]: (body, resources) = md_exporter.from_notebook_node(notebook) cleaned_body = clean_contents(body) document.page_content = cleaned_body + document.metadata["source_type"] = ( + "notebook" + if self.config.source_type == "code" + else self.config.source_type + ) elif os.path.splitext(f_name)[-1] == ".md": document = TextLoader(f_name).load()[0] contents = document.page_content cleaned_body = clean_contents(contents) document.page_content = cleaned_body + document.metadata["source_type"] = ( + "markdown" + if self.config.source_type == "code" + else self.config.source_type + ) else: document = TextLoader(f_name).load()[0] + document.metadata["source_type"] = ( + "code" + if self.config.source_type == "code" + else self.config.source_type + ) document.metadata["file_type"] = os.path.splitext( document.metadata["source"] @@ -357,13 +348,6 @@ def lazy_load(self) -> Iterator[Document]: document.metadata["source"] = document_files[ document.metadata["source"] ] - document.metadata["language"] = EXTENSION_MAP[ - document.metadata["file_type"] - ] - document.metadata["description"] = "" - document.metadata["tags"] = self.extract_tags( - document.metadata["source"] - ) yield document except Exception as e: logger.warning( @@ -591,7 +575,7 @@ def cleanup_reports_df(self, reports_df): spec_type_ls = [] is_buggy_ls = [] is_short_report_ls = [] - for idx, row in tqdm(reports_df.iterrows()): + for idx, row in reports_df.iterrows(): if row["spec"] is None or isinstance(row["spec"], float): logger.debug(idx) markdown_ls.append("spec-error") @@ -692,7 +676,8 @@ def fetch_data(self): return self.config.data_source.local_path - def clean_invalid_unicode_escapes(self, text): + @staticmethod + def clean_invalid_unicode_escapes(text): """ clean up invalid unicode escape sequences """ @@ -777,7 +762,7 @@ def extract_tags(source_url: str, report_content: str) -> List[str]: tags = list(set([tag.title() for tag in tags])) if ("wandb.log" or "wandb.init") in report_content: - tags.append("contains-wandb-code") + tags.append("Contains Wandb Code") return tags @@ -789,19 +774,16 @@ def lazy_load(self) -> Iterator[Document]: Yields: A Document object. """ - lang_detect = FastTextLangDetect() data_dump_fame = self.fetch_data() for parsed_row in self.parse_data_dump(data_dump_fame): document = Document( page_content=parsed_row["content"], metadata={ "source": parsed_row["source"], - "language": lang_detect.detect_language( - parsed_row["content"] - ), + "source_type": self.config.source_type, "file_type": ".md", "description": parsed_row["description"], - "tags": ["fc-report"] + "tags": ["Fully Connected", "Report"] + self.extract_tags( parsed_row["source"], parsed_row["content"] ), @@ -810,6 +792,33 @@ def lazy_load(self) -> Iterator[Document]: yield document +SOURCE_TYPE_TO_LOADER_MAP = { + "documentation": DocodileDataLoader, + "code": CodeDataLoader, + "notebook": CodeDataLoader, + "report": FCReportsDataLoader, +} + + +def load_from_config(config: DataStoreConfig) -> pathlib.Path: + loader = SOURCE_TYPE_TO_LOADER_MAP[config.source_type](config) + loader.config.docstore_dir.mkdir(parents=True, exist_ok=True) + + with (loader.config.docstore_dir / "config.json").open("w") as f: + f.write(loader.config.model_dump_json()) + + with (loader.config.docstore_dir / "documents.jsonl").open("w") as f: + for document in loader.load(): + document_json = { + "page_content": document.page_content, + "metadata": document.metadata, + } + f.write(json.dumps(document_json) + "\n") + with (loader.config.docstore_dir / "metadata.json").open("w") as f: + json.dump(loader.metadata, f) + return loader.config.docstore_dir + + def load( project: str, entity: str, @@ -837,47 +846,26 @@ def load( description="Raw documents for wandbot", ) - en_docodile_loader = DocodileDataLoader(DocodileEnglishStoreConfig()) - ja_docodile_loader = DocodileDataLoader(DocodileJapaneseStoreConfig()) - examples_code_loader = CodeDataLoader(ExampleCodeStoreConfig()) - examples_notebook_loader = CodeDataLoader(ExampleNotebookStoreConfig()) - sdk_code_loader = CodeDataLoader(SDKCodeStoreConfig()) - sdk_tests_loader = CodeDataLoader(SDKTestsStoreConfig()) - weave_code_loader = CodeDataLoader(WeaveCodeStoreConfig()) - weave_examples_loader = CodeDataLoader(WeaveExamplesStoreConfig()) - wandb_edu_code_loader = CodeDataLoader(WandbEduCodeStoreConfig()) - fc_reports_loader = FCReportsDataLoader(FCReportsStoreConfig()) - - for loader in [ - en_docodile_loader, - ja_docodile_loader, - examples_code_loader, - examples_notebook_loader, - sdk_code_loader, - sdk_tests_loader, - weave_code_loader, - weave_examples_loader, - wandb_edu_code_loader, - fc_reports_loader, - ]: - loader.config.docstore_dir.mkdir(parents=True, exist_ok=True) - - with (loader.config.docstore_dir / "config.json").open("w") as f: - f.write(loader.config.model_dump_json()) - - with (loader.config.docstore_dir / "documents.jsonl").open("w") as f: - for document in loader.load(): - document_json = { - "page_content": document.page_content, - "metadata": document.metadata, - } - f.write(json.dumps(document_json) + "\n") - with (loader.config.docstore_dir / "metadata.json").open("w") as f: - json.dump(loader.metadata, f) - + configs = [ + DocodileEnglishStoreConfig(), + DocodileJapaneseStoreConfig(), + ExampleCodeStoreConfig(), + ExampleNotebookStoreConfig(), + SDKCodeStoreConfig(), + SDKTestsStoreConfig(), + WeaveCodeStoreConfig(), + WeaveExamplesStoreConfig(), + WandbEduCodeStoreConfig(), + FCReportsStoreConfig(), + ] + + pool = Pool(cpu_count() - 1) + results = pool.imap_unordered(load_from_config, configs) + + for docstore_path in results: artifact.add_dir( - str(loader.config.docstore_dir), - name=loader.config.docstore_dir.name, + str(docstore_path), + name=docstore_path.name, ) run.log_artifact(artifact) run.finish() diff --git a/src/wandbot/ingestion/preprocess_data.py b/src/wandbot/ingestion/preprocess_data.py index 4726d7c..8e0a00f 100644 --- a/src/wandbot/ingestion/preprocess_data.py +++ b/src/wandbot/ingestion/preprocess_data.py @@ -19,148 +19,220 @@ nodes = load(documents, chunk_size=1024) """ -from typing import Any, List +import json +import pathlib +from typing import Any, List, Sequence, Callable import tiktoken -from langchain.schema import Document as LcDocument -from llama_index import Document as LlamaDocument -from llama_index.node_parser import ( - CodeSplitter, - MarkdownNodeParser, - TokenTextSplitter, +import wandb +from langchain.text_splitter import ( + RecursiveCharacterTextSplitter, + MarkdownHeaderTextSplitter, + Language, +) +from langchain.text_splitter import TokenTextSplitter +from langchain_core.documents import Document, BaseDocumentTransformer + +from wandbot.utils import ( + get_logger, + FastTextLangDetect, + make_document_tokenization_safe, + filter_smaller_documents, ) -from llama_index.schema import BaseNode, TextNode -from wandbot.utils import get_logger logger = get_logger(__name__) -def make_texts_tokenization_safe(documents: List[str]) -> List[str]: - """Removes special tokens from the given documents. +class Tokenizer: + def __init__(self, model_name): + self.tokenizer = tiktoken.encoding_for_model(model_name) - Args: - documents: A list of strings representing the documents. + def encode(self, text): + return self.tokenizer.encode(text, allowed_special="all") - Returns: - A list of cleaned documents with special tokens removed. - """ - encoding = tiktoken.get_encoding("cl100k_base") - special_tokens_set = encoding.special_tokens_set + def decode(self, tokens): + return self.tokenizer.decode(tokens) - def remove_special_tokens(text: str) -> str: - """Removes special tokens from the given text. - Args: - text: A string representing the text. +tokenizer = Tokenizer("gpt2") - Returns: - The text with special tokens removed. - """ - for token in special_tokens_set: - text = text.replace(token, "") - return text - cleaned_documents = [] - for document in documents: - cleaned_document = remove_special_tokens(document) - cleaned_documents.append(cleaned_document) - return cleaned_documents +def length_function(content: str) -> int: + return len(tokenizer.encode(content)) -class CustomMarkdownNodeParser(MarkdownNodeParser): - def _build_node_from_split( - self, - text_split: str, - node: BaseNode, - metadata: dict, - ) -> TextNode: - """Build node from single text split.""" - text_splits = make_texts_tokenization_safe([text_split]) - text_split = text_splits[0] - return super()._build_node_from_split(text_split, node, metadata) - - -class CustomCodeSplitter(CodeSplitter): - """Splits text into chunks based on custom code formatting. - - Attributes: - language: The programming language of the code. - max_chars: The maximum number of characters allowed in a chunk. - """ - - def split_text(self, text: str) -> List[str]: - """Split text into chunks.""" - text_splits = super().split_text(text) - text_splits = make_texts_tokenization_safe(text_splits) - return text_splits - - -def convert_lc_to_llama(document: LcDocument) -> LlamaDocument: - """Converts a Langchain document to a Llama document. - - Args: - document: A Langchain document. - - Returns: - A Llama document. - """ - llama_document = LlamaDocument.from_langchain_format(document) - excluded_embed_metadata_keys = [ - "file_type", - "source", - "language", - ] - excluded_llm_metadata_keys = [ - "file_type", - ] - llama_document.excluded_embed_metadata_keys = excluded_embed_metadata_keys - llama_document.excluded_llm_metadata_keys = excluded_llm_metadata_keys - - return llama_document - - -def load(documents: List[LcDocument], chunk_size: int = 384) -> List[TextNode]: - """Loads documents and returns a list of nodes. - - Args: - documents: A list of documents. - chunk_size: The size of each chunk. - - Returns: - A list of nodes. - """ - md_parser = CustomMarkdownNodeParser(chunk_size=chunk_size) - code_parser = CustomCodeSplitter( - language="python", max_chars=chunk_size * 2 - ) - # Define the node parser - node_parser = TokenTextSplitter.from_defaults(chunk_size=chunk_size) +def len_function_with_doc(document: Document) -> int: + return len(tokenizer.encode(document.page_content)) - llama_docs: List[LlamaDocument] = list( - map(lambda x: convert_lc_to_llama(x), documents) - ) - nodes: List[Any] = [] - for doc in llama_docs: - try: - if doc.metadata["file_type"] == ".py": - parser = code_parser +class MarkdownTextTransformer(BaseDocumentTransformer): + def __init__(self, lang_detect, chunk_size: int = 512): + self.fasttext_model = lang_detect + self.chunk_size: int = chunk_size + self.length_function: Callable[[str], int] + self.recursive_splitter = RecursiveCharacterTextSplitter.from_language( + language=Language.MARKDOWN, + chunk_size=self.chunk_size, + chunk_overlap=0, + keep_separator=True, + length_function=length_function, + ) + self.header_splitter = MarkdownHeaderTextSplitter( + headers_to_split_on=[ + ("#", "header_1"), + ("##", "header_2"), + ("###", "header_3"), + ("####", "header_4"), + ("#####", "header_5"), + ("######", "header_6"), + ] + ) + + def split_document_on_headers( + self, + document: Document, + ) -> List[Document]: + output_documents = [] + splits = self.header_splitter.split_text(document.page_content) + for split in list(splits): + output_documents.append( + Document( + page_content=split.page_content, metadata=document.metadata + ) + ) + return output_documents + + def recursively_merge_chunks( + self, + chunks: List[Document], + ) -> List[Document]: + if not chunks: # check if chunks is empty + return [] # return an empty list if chunks is empty + merged_chunks = [] + current_chunk = chunks[0] + current_length = length_function(current_chunk.page_content) + for chunk in chunks[1:]: + chunk_length = length_function(chunk.page_content) + if current_length + chunk_length <= self.chunk_size: + current_chunk.page_content += ( + "\n\n" + chunk.page_content + "\n\n" + ) + current_length += chunk_length else: - parser = md_parser - nodes.extend(parser.get_nodes_from_documents([doc])) - except Exception as e: - logger.error(f"Error parsing document: {e}") - logger.warning( - f"Unable to parse document: {doc.metadata['source']} with custom parser, using default " - f"parser instead." + merged_chunks.append(current_chunk) + current_chunk = chunk + current_length = chunk_length + merged_chunks.append(current_chunk) + return merged_chunks + + def identify_document_language(self, document: Document) -> str: + return self.fasttext_model.detect_language(document.page_content) + + def split_markdown_documents( + self, + documents: List[Document], + ) -> List[Document]: + chunked_documents = [] + for document in documents: + document_splits = self.split_document_on_headers( + document=document, + ) + split_chunks = self.recursive_splitter.split_documents( + document_splits + ) + merged_chunks = self.recursively_merge_chunks( + split_chunks, ) - nodes.extend(node_parser.get_nodes_from_documents([doc])) + chunked_documents.extend(merged_chunks) + + for document in chunked_documents[:]: + document.metadata["has_code"] = "```" in document.page_content + document.metadata["language"] = self.identify_document_language( + document + ) + return chunked_documents + + def transform_documents( + self, documents: Sequence[Document], **kwargs: Any + ) -> Sequence[Document]: + split_documents = self.split_markdown_documents(list(documents)) + transformed_documents = [] + for document in split_documents: + transformed_documents.append(document) + return transformed_documents - nodes = node_parser.get_nodes_from_documents(nodes) - def filter_smaller_nodes( - text_nodes: List[TextNode], min_size: int = 5 - ) -> List[TextNode]: +class CodeTextTransformer(BaseDocumentTransformer): + def __init__( + self, + chunk_size: int = 512, + ): + self.chunk_size: int = chunk_size + self.length_function: Callable[[str], int] + + def split_documents(self, documents: List[Document]) -> List[Document]: + """Split incoming code and return chunks using the AST.""" + chunked_documents = [] + for document in documents: + file_extension = document.metadata.get("file_type", "") + if file_extension in [".py", ".js", ".ts"]: + language = { + ".py": Language.PYTHON, + ".js": Language.JS, + ".ts": Language.JS, + }[file_extension] + recursive_splitter = ( + RecursiveCharacterTextSplitter.from_language( + language=language, + chunk_size=self.chunk_size, + chunk_overlap=0, + keep_separator=True, + length_function=length_function, + ) + ) + chunked_documents.extend( + recursive_splitter.split_documents([document]) + ) + elif file_extension in [".md", ".ipynb"]: + chunked_documents.extend( + MarkdownTextTransformer().transform_documents([document]) + ) + else: + chunked_documents.extend( + TokenTextSplitter().split_documents([document]) + ) + return chunked_documents + + def transform_documents( + self, documents: Sequence[Document], **kwargs: Any + ) -> Sequence[Document]: + document_splits = [] + for document in list(documents): + document_splits.extend(self.split_documents([document])) + + transformed_documents = [] + + for document in list(document_splits): + document.metadata["has_code"] = True + document.metadata["language"] = "en" + transformed_documents.append(document) + + return transformed_documents + + +class DocumentTransformer(BaseDocumentTransformer): + def __init__(self, lang_detect, max_size: int = 512, min_size: int = 5): + self.lang_detect = lang_detect + self.chunk_size = max_size + self.min_size = min_size + self.markdown_transformer = MarkdownTextTransformer( + lang_detect=lang_detect, chunk_size=self.chunk_size + ) + self.code_transformer = CodeTextTransformer(chunk_size=self.chunk_size) + + def filter_smaller_documents( + self, documents: List[Document], min_size: int = 5 + ) -> List[Document]: """Filters out nodes that are smaller than the chunk size. Args: @@ -171,13 +243,132 @@ def filter_smaller_nodes( A list of nodes. """ - for node in text_nodes: - content = node.get_content() - word_len = len( - [c for c in content.strip().split() if c and len(c) > 2] - ) - if word_len >= min_size: + for node in documents: + content = node.page_content + if length_function(content) >= min_size: yield node - nodes = list(filter_smaller_nodes(nodes)) - return nodes + def standardize_metadata(self, documents: List[Document]) -> List[Document]: + for document in documents: + metadata = document.metadata + for key, value in metadata.items(): + if not isinstance(value, (str, int, float, bool)): + if isinstance(value, list): + metadata[key] = " ".join(value) + elif isinstance(value, dict): + metadata[key] = json.dumps(value) + elif isinstance(value, (set, tuple)): + metadata[key] = " ".join(list(value)) + else: + metadata[key] = None + metadata = {k: v for k, v in metadata.items() if v is not None} + document.metadata = metadata + return documents + + def transform_documents( + self, documents: Sequence[Document], **kwargs: Any + ) -> Sequence[Document]: + transformed_documents = [] + for document in list(documents): + document = make_document_tokenization_safe(document) + if document.metadata.get("source_type", "") == "code": + transformed_documents.extend( + self.code_transformer.transform_documents([document]) + ) + else: + transformed_documents.extend( + self.markdown_transformer.transform_documents([document]) + ) + transformed_documents = list( + self.filter_smaller_documents( + transformed_documents, min_size=self.min_size + ) + ) + transformed_documents = list( + self.standardize_metadata(transformed_documents) + ) + + transformed_documents = filter_smaller_documents( + transformed_documents, min_size=self.min_size + ) + return transformed_documents + + def transform_document(self, document: Document) -> Document: + return self.transform_documents([document])[0] + + +def process_document_file( + documents: List[Document], transformer: DocumentTransformer +) -> List[Document]: + transformed_documents = transformer.transform_documents(documents) + + return list(transformed_documents) + + +def load( + project: str, + entity: str, + source_artifact_path: str, + result_artifact_name: str = "transformed_data", +) -> str: + + run: wandb.wandb_sdk.wandb_run.Run = wandb.init( + project=project, entity=entity, job_type="preprocess_data" + ) + artifact: wandb.Artifact = run.use_artifact( + source_artifact_path, type="dataset" + ) + artifact_dir: str = artifact.download() + + document_files: List[pathlib.Path] = list( + pathlib.Path(artifact_dir).rglob("documents.jsonl") + ) + + lang_detect = FastTextLangDetect() + transformer = DocumentTransformer( + lang_detect=lang_detect, + max_size=512, + ) + + result_artifact = wandb.Artifact(result_artifact_name, type="dataset") + + for document_file in document_files: + with document_file.open() as f: + documents = [Document(**json.loads(line)) for line in f] + transformed_documents = process_document_file( + documents, transformer + ) + config = json.load((document_file.parent / "config.json").open()) + metadata = json.load( + (document_file.parent / "metadata.json").open() + ) + cache_dir = ( + pathlib.Path(config["data_source"]["cache_dir"]).parent + / "transformed_data" + ) + + transformed_file = ( + cache_dir / document_file.parent.name / document_file.name + ) + + transformed_file.parent.mkdir(parents=True, exist_ok=True) + with transformed_file.open("w") as f: + for document in transformed_documents: + f.write(json.dumps(document.dict()) + "\n") + + config["chunk_size"] = 512 + with open(transformed_file.parent / "config.json", "w") as f: + json.dump(config, f) + + metadata["num_transformed_documents"] = len(transformed_documents) + with open(transformed_file.parent / "metadata.json", "w") as f: + json.dump(metadata, f) + + result_artifact.add_dir( + str(transformed_file.parent), + name=document_file.parent.name, + ) + + run.log_artifact(result_artifact) + run.finish() + return f"{entity}/{project}/{result_artifact_name}:latest" diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index cb1fd2b..029ac38 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -14,18 +14,17 @@ import json import pathlib -from typing import Any, Dict, List +from typing import List import wandb -from langchain.schema import Document as LcDocument -from llama_index.schema import TextNode -from wandbot.ingestion import preprocess_data +from langchain_community.vectorstores.chroma import Chroma +from langchain_core.documents import Document +from langchain_openai import OpenAIEmbeddings +from tqdm import trange + from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import ( get_logger, - load_index, - load_service_context, - load_storage_context, ) logger = get_logger(__name__) @@ -61,52 +60,39 @@ def load( source_artifact_path, type="dataset" ) artifact_dir: str = artifact.download() - storage_context = load_storage_context(config.embedding_dim) - service_context = load_service_context( - embeddings_model=config.embeddings_model, - embeddings_size=config.embedding_dim, - llm="gpt-3.5-turbo-16k-0613", - temperature=config.temperature, - max_retries=config.max_retries, + + embedding_fn = OpenAIEmbeddings( + model=config.embeddings_model, dimensions=config.embedding_dim ) + vectorstore_dir = config.persist_dir + vectorstore_dir.mkdir(parents=True, exist_ok=True) document_files: List[pathlib.Path] = list( pathlib.Path(artifact_dir).rglob("documents.jsonl") ) - transformed_documents: Dict[str, List[TextNode]] = {} + transformed_documents = [] for document_file in document_files: - documents: List[LcDocument] = [] with document_file.open() as f: for line in f: - doc_dict: Dict[str, Any] = json.loads(line) - doc: LcDocument = LcDocument(**doc_dict) - documents.append(doc) - preprocessed_documents = preprocess_data.load(documents) - unique_objects = {obj.hash: obj for obj in preprocessed_documents} - preprocessed_documents = list(unique_objects.values()) - transformed_documents[ - document_file.parent.name - ] = preprocessed_documents - - for store_name, doc_list in transformed_documents.items(): - logger.info(f"Number of documents: {len(doc_list)}") - _ = load_index( - doc_list, - service_context, - storage_context, - index_id=store_name, - persist_dir=str(config.persist_dir), - ) - artifact = wandb.Artifact( - name="wandbot_index", - type="storage_context", - metadata={"index_ids": list(transformed_documents.keys())}, + transformed_documents.append(Document(**json.loads(line))) + + chroma = Chroma( + collection_name=config.name, + embedding_function=embedding_fn, + persist_directory=str(config.persist_dir), ) - artifact.add_dir( + for batch_idx in trange(0, len(transformed_documents), config.batch_size): + batch = transformed_documents[batch_idx : batch_idx + config.batch_size] + chroma.add_documents(batch) + chroma.persist() + + result_artifact = wandb.Artifact(name="chroma_index", type="vectorstore") + + result_artifact.add_dir( local_path=str(config.persist_dir), ) - run.log_artifact(artifact) + run.log_artifact(result_artifact) run.finish() return f"{entity}/{project}/{result_artifact_name}:latest" diff --git a/src/wandbot/query_handler/__init__.py b/src/wandbot/query_handler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/wandbot/query_handler/history_handler.py b/src/wandbot/query_handler/history_handler.py new file mode 100644 index 0000000..ee38f29 --- /dev/null +++ b/src/wandbot/query_handler/history_handler.py @@ -0,0 +1,75 @@ +from _operator import itemgetter + +from langchain_core.messages import get_buffer_string +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import ( + RunnablePassthrough, + RunnableLambda, + RunnableBranch, + Runnable, +) +from langchain_openai import ChatOpenAI + +CONDENSE_PROMPT_SYSTEM_TEMPLATE = ( + "Given the following conversation between a user and an AI assistant and a follow up " + "question from user, rephrase the follow up question to be a standalone question. Ensure " + "that the standalone question summarizes the conversation and completes the follow up " + "question with all the necessary context. If there is no history return the original question verbatim" +) + + +CONDENSE_PROMPT_MESSAGES = [ + ("system", CONDENSE_PROMPT_SYSTEM_TEMPLATE), + ( + "human", + """Rephrase the following question to be a standalone question based on the given history: + + +{chat_history} + + + +{question} + +""", + ), +] + + +def load_standalone_query_chain(model: ChatOpenAI) -> Runnable: + condense_prompt = ChatPromptTemplate.from_messages(CONDENSE_PROMPT_MESSAGES) + + condense_question_chain = ( + { + "question": RunnablePassthrough(), + "chat_history": itemgetter("chat_history") + | RunnableLambda(get_buffer_string), + } + | condense_prompt + | model + | StrOutputParser() + ) + + return condense_question_chain + + +def load_query_condense_chain( + model: ChatOpenAI, +) -> Runnable: + standalone_query_chain = load_standalone_query_chain( + model, + ) + branch = RunnableBranch( + ( + lambda x: True if x["chat_history"] else False, + standalone_query_chain, + ), + ( + lambda x: False if x["chat_history"] else True, + itemgetter("question"), + ), + itemgetter("question"), + ) + + return branch diff --git a/src/wandbot/query_handler/intents_enhancer.py b/src/wandbot/query_handler/intents_enhancer.py new file mode 100644 index 0000000..a2b8644 --- /dev/null +++ b/src/wandbot/query_handler/intents_enhancer.py @@ -0,0 +1,231 @@ +import enum +import os +from operator import itemgetter +from typing import List + +import cohere +from langchain.chains.openai_functions import create_structured_output_runnable +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import ( + RunnableLambda, + Runnable, + RunnableParallel, + RunnablePassthrough, +) +from langchain_openai import ChatOpenAI +from pydantic.v1 import BaseModel, Field + + +class Labels(str, enum.Enum): + UNRELATED = "unrelated" + CODE_TROUBLESHOOTING = "code_troubleshooting" + INTEGRATIONS = "integrations" + PRODUCT_FEATURES = "product_features" + SALES_AND_GTM_RELATED = "sales_and_gtm_related" + BEST_PRACTICES = "best_practices" + COURSE_RELATED = "course_related" + NEEDS_MORE_INFO = "needs_more_info" + OPINION_REQUEST = "opinion_request" + NEFARIOUS_QUERY = "nefarious_query" + OTHER = "other" + + +class Label(BaseModel): + "An intent label to be associated with the query" + + reasoning: str = Field( + ..., + description="The reason for the identifying the intent", + ) + label: Labels = Field( + ..., description="An intent associated with the query" + ) + + +class MultiLabel(BaseModel): + "A list of intents associated with the query" + intents: List[Label] = Field( + ..., + description="The list of intents associated with the query", + ) + + +INTENT_DESCRIPTIONS = { + Labels.UNRELATED.value: "The query is not related to Weights & Biases", + Labels.CODE_TROUBLESHOOTING.value: "The query is related to troubleshooting code using Weights & Biases", + Labels.INTEGRATIONS.value: "The query is related to integrating Weights & Biases with other tools, frameworks, " + "or libraries", + Labels.PRODUCT_FEATURES.value: "The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, " + "Reports, Experiments, Tables, Prompts, Launch, Weave, StreamTables and more", + Labels.SALES_AND_GTM_RELATED.value: "The query is related to sales, marketing, or other business related topics " + "such as pricing, billing, or partnerships etc", + Labels.BEST_PRACTICES.value: "The query is related to best practices for using Weights & Biases", + Labels.COURSE_RELATED.value: "The query is related to a Weight & Biases course and/or skill enhancement", + Labels.NEEDS_MORE_INFO.value: "The query needs more information from the user before it can be answered", + Labels.OPINION_REQUEST.value: "The query is asking for an opinion", + Labels.NEFARIOUS_QUERY.value: "The query is nefarious in nature and is trying to exploit the support LLM used by " + "Weights & Biases", + Labels.OTHER.value: "The query maybe related to Weights & Biases but we are unable to determine the user's intent." + " It's best to ask the user to rephrase the query or avoid answering the query", +} +QUERY_INTENTS = { + Labels.UNRELATED.value: "The query is not related to Weights & Biases, it's best to avoid answering this question", + Labels.CODE_TROUBLESHOOTING.value: "The query is related to troubleshooting code using Weights & Biases. Help " + "with a detailed code snippet and explanation", + Labels.INTEGRATIONS.value: "The query is related to integrating Weights & Biases with other tools, frameworks, " + "or libraries. Help with a detailed code snippet and explanation and ask for more information about the " + "integration if needed", + Labels.PRODUCT_FEATURES.value: "The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, " + "Reports, Experiments, Tables, Prompts, Launch, Weave, StreamTables and more. Provide a link to the relevant " + "documentation and explain the feature in detail", + Labels.SALES_AND_GTM_RELATED.value: "The query is related to sales, marketing, or other business related topics " + "such as pricing, billing, or partnerships etc. Ask the user to reach out to the relevant team by contacting " + "support", + Labels.BEST_PRACTICES.value: "The query is related to best practices for using Weights & Biases. Answer the query " + "and provide guidance where necessary", + Labels.COURSE_RELATED.value: "The query is related to a Weight & Biases course and/or skill enhancement. Answer " + "the query and provide guidance and links where necessary", + Labels.NEEDS_MORE_INFO.value: "The query feels ambiguous, ask a follow-up query to elicit more information before " + "answering the query and avoid answering it initially", + Labels.OPINION_REQUEST.value: "The query is asking for an opinion. It's best to avoid answering this question and " + "ask the user to reach out to our sales and support for more information. Always favor Weights & Biases in your " + "response", + Labels.NEFARIOUS_QUERY.value: "The query looks nefarious in nature. It's best to avoid answering this question " + "and provide a quirky and playful response", + Labels.OTHER.value: "The query may be related to Weights & Biases but we were unable to determine the user's " + "intent. It's best to avoid answering this question and ask the user a follow-up query to rephrase their original " + "query", +} + + +def get_intent_descriptions(intents: List[str]) -> str: + descriptions = [] + if not intents: + return "- " + INTENT_DESCRIPTIONS["other"] + + for classification in intents: + description = INTENT_DESCRIPTIONS.get(classification, "") + descriptions.append(description) + descriptions = "- " + "\n- ".join(descriptions) + return descriptions + + +def get_intent_hints(intents: List[str]) -> str: + descriptions = [] + if not intents: + return "- " + QUERY_INTENTS["other"] + + for classification in intents: + description = QUERY_INTENTS.get(classification, "") + descriptions.append(description) + descriptions = "- " + "\n- ".join(descriptions) + return descriptions + + +class CohereQueryClassifier: + def __init__(self, api_key: str, model: str) -> None: + self.client = cohere.Client(api_key) + self.model = model + + def __call__(self, query: str) -> str: + response = self.client.classify( + model=self.model, + inputs=[query], + ) + return get_intent_descriptions(response.classifications[0].predictions) + + +def load_cohere_classify_chain(api_key: str, model: str) -> RunnableLambda: + cohere_classify_chain = RunnableLambda( + lambda x: { + "question": x["question"], + "intent_hints": CohereQueryClassifier(api_key, model)( + x["question"] + ), + } + ) + return cohere_classify_chain + + +intents_descriptions_str = "\n".join( + [ + f"{label}:\t{description}" + for label, description in INTENT_DESCRIPTIONS.items() + ] +) + +INTENT_PROMPT_MESSAGES = [ + ( + "system", + """You are a Weights & Biases support manager. Your goal is to enhance the query by identifying one or more intents related to the query. + Here is the mapping of the intents and their descriptions. + """ + + intents_descriptions_str, + ), + ("human", "Enhance the following user query:\n{question}"), + ( + "human", + "Here is my initial list of intent hints that maybe relevant:\n{intent_hints}", + ), + ("human", "Tip: Make sure to answer in the correct format"), +] + + +def load_intent_extraction_chain(model: ChatOpenAI) -> Runnable: + + intents_prompt = ChatPromptTemplate.from_messages(INTENT_PROMPT_MESSAGES) + + intents_classification_chain = create_structured_output_runnable( + MultiLabel, model, intents_prompt + ) + + cohere_classify_chain = load_cohere_classify_chain( + api_key=os.environ["COHERE_API_KEY"], + model=os.environ["DEFAULT_QUERY_CLF_MODEL"], + ) + + intent_enhancement_chain = ( + cohere_classify_chain + | intents_classification_chain + | RunnableLambda(lambda x: [intent.label.value for intent in x.intents]) + | { + "intent_hints": get_intent_hints, + "intent_labels": RunnablePassthrough(), + } + ) + + return intent_enhancement_chain + + +def check_avoid_intent(intents: List[str]) -> bool: + return any( + [ + intent + in [ + Labels.NEFARIOUS_QUERY.value, + Labels.OPINION_REQUEST.value, + Labels.NEEDS_MORE_INFO.value, + Labels.UNRELATED.value, + ] + for intent in intents + ] + ) + + +def load_intent_enhancement_chain( + model: ChatOpenAI, +) -> Runnable: + intent_extraction_chain = load_intent_extraction_chain(model) + + return RunnableParallel( + question=itemgetter("question"), + standalone_question=itemgetter("standalone_question"), + chat_history=itemgetter("chat_history"), + language=itemgetter("language"), + intents=( + {"question": itemgetter("standalone_question")} + | intent_extraction_chain + ), + ) | RunnablePassthrough.assign( + avoid_query=lambda x: check_avoid_intent(x["intents"]["intent_labels"]) + ) diff --git a/src/wandbot/query_handler/keyword_search_enhancer.py b/src/wandbot/query_handler/keyword_search_enhancer.py new file mode 100644 index 0000000..e1ad371 --- /dev/null +++ b/src/wandbot/query_handler/keyword_search_enhancer.py @@ -0,0 +1,85 @@ +from operator import itemgetter +from typing import List + +from langchain.chains.openai_functions import create_structured_output_runnable +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import ( + RunnableLambda, + RunnableParallel, + Runnable, + RunnableBranch, +) +from langchain_openai import ChatOpenAI +from pydantic.v1 import BaseModel, Field + +KEYWORDS_SYSTEM_PROMPT = ( + "You are a Weights & Biases support manager. " + "Your goal is to enhance the user query by adding a list of keywords used for web search." +) + +KEYWORDS_PROMPT_MESSAGES = [ + ("system", KEYWORDS_SYSTEM_PROMPT), + ( + "human", + "Enhance the following query related to weights and biases for web search.:\n\n{question}", + ), + ("human", "Tip: Make sure to answer in the correct format"), +] + + +class Keyword(BaseModel): + """A Keyword to search for on the wen""" + + keyword: str = Field( + ..., + description="A search term for getting the most relevant information required to answer the query", + ) + + +class KeywordsSchema(BaseModel): + "A list of search keywords to enhance the search query" + keywords: List[Keyword] = Field( + ..., + description="List of five different search terms", + min_items=0, + max_items=5, + ) + + +def load_keywords_extraction_chain(model: ChatOpenAI) -> Runnable: + + keywords_prompt = ChatPromptTemplate.from_messages(KEYWORDS_PROMPT_MESSAGES) + + keywords_extraction_chain = create_structured_output_runnable( + KeywordsSchema, model, keywords_prompt + ) + + keywords_chain = keywords_extraction_chain | RunnableLambda( + lambda x: [keyword.keyword for keyword in x.keywords] + ) + + return keywords_chain + + +def load_keywords_enhancement_chain(model: ChatOpenAI) -> Runnable: + keywords_chain = load_keywords_extraction_chain(model) + + branch = RunnableBranch( + ( + lambda x: x["avoid"], + RunnableLambda(lambda x: []), + ), + ( + lambda x: not x["avoid"], + keywords_chain, + ), + RunnableLambda(lambda x: []), + ) + + return ( + RunnableParallel( + question=itemgetter("standalone_question"), + avoid=itemgetter("avoid_query"), + ) + | branch + ) diff --git a/src/wandbot/query_handler/language_detection.py b/src/wandbot/query_handler/language_detection.py new file mode 100644 index 0000000..847d9d5 --- /dev/null +++ b/src/wandbot/query_handler/language_detection.py @@ -0,0 +1,25 @@ +from operator import itemgetter + +from langchain_core.runnables import RunnablePassthrough, Runnable + +from wandbot.utils import FastTextLangDetect, FasttextModelConfig + + +class LangDetect: + def __init__(self, model): + self.model = FastTextLangDetect( + FasttextModelConfig( + fasttext_file_path=model, + ) + ) + + def __call__(self, question: str) -> str: + return self.model.detect_language(question) + + +def load_language_detection_chain(model: str) -> Runnable: + lang_detect = LangDetect(model) + lang_detect_chain = RunnablePassthrough().assign( + language=lambda x: lang_detect(x["question"]) + ) | itemgetter("language") + return lang_detect_chain diff --git a/src/wandbot/query_handler/query_enhancer.py b/src/wandbot/query_handler/query_enhancer.py new file mode 100644 index 0000000..8864d2f --- /dev/null +++ b/src/wandbot/query_handler/query_enhancer.py @@ -0,0 +1,70 @@ +from operator import itemgetter + +import regex as re +from langchain_core.runnables import ( + Runnable, + RunnablePassthrough, + RunnableParallel, +) +from langchain_openai import ChatOpenAI + +from wandbot.query_handler.history_handler import load_query_condense_chain +from wandbot.query_handler.intents_enhancer import load_intent_enhancement_chain +from wandbot.query_handler.keyword_search_enhancer import ( + load_keywords_enhancement_chain, +) +from wandbot.query_handler.language_detection import ( + load_language_detection_chain, +) +from wandbot.query_handler.vector_search_enhancer import ( + load_vectorsearch_enhancement_chain, +) +from wandbot.query_handler.web_search import load_web_answer_enhancement_chain + +BOT_NAME_PATTERN = re.compile(r"<@U[A-Z0-9]+>|@[a-zA-Z0-9]+") + + +def clean_question(question: str) -> str: + cleaned_query = BOT_NAME_PATTERN.sub("", question).strip() + return cleaned_query + + +def load_query_enhancement_chain( + model: ChatOpenAI, lang_detect_model_path: str +) -> Runnable: + + condense_question_chain = load_query_condense_chain(model) + intent_enhancement_chain = load_intent_enhancement_chain(model) + + language_enhancement_chain = load_language_detection_chain( + model=lang_detect_model_path + ) + + keywords_enhancement_chain = load_keywords_enhancement_chain(model) + vector_search_enhancement_chain = load_vectorsearch_enhancement_chain(model) + web_answer_enhancement_chain = load_web_answer_enhancement_chain(top_k=5) + + query_enhancer_chain = ( + RunnablePassthrough().assign( + question=lambda x: clean_question(x["query"]), + ) + | RunnableParallel( + question=itemgetter("question"), + standalone_question=condense_question_chain, + language=language_enhancement_chain, + chat_history=itemgetter("chat_history"), + ) + | intent_enhancement_chain + | RunnableParallel( + standalone_question=itemgetter("standalone_question"), + language=itemgetter("language"), + question=itemgetter("question"), + intents=itemgetter("intents"), + chat_history=itemgetter("chat_history"), + keywords=keywords_enhancement_chain, + vector_search=vector_search_enhancement_chain, + web_results=web_answer_enhancement_chain, + avoid_query=itemgetter("avoid_query"), + ) + ) + return query_enhancer_chain diff --git a/src/wandbot/query_handler/vector_search_enhancer.py b/src/wandbot/query_handler/vector_search_enhancer.py new file mode 100644 index 0000000..c47756e --- /dev/null +++ b/src/wandbot/query_handler/vector_search_enhancer.py @@ -0,0 +1,73 @@ +from operator import itemgetter + +from langchain.chains.openai_functions import create_structured_output_runnable +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import ( + RunnableSerializable, + RunnableLambda, + RunnableBranch, + RunnableParallel, + Runnable, +) +from langchain_openai import ChatOpenAI +from pydantic.v1 import BaseModel, Field + +QUERY_REWRITE_SYSTEM_PROMPT = ( + "You are a Weights & Biases support manager. " + "Your goal is to enhance the user query by rewriting it for similarity search. " + "Rewrite the given query into a clear, specific, and formal request for retrieving relevant information from a vector database" +) + +QUERY_REWRITE_PROMPT_MESSAGES = [ + ("system", QUERY_REWRITE_SYSTEM_PROMPT), + ("human", "Enhance the following query i.:\n\n{question}"), + ("human", "Tip: Make sure to answer in the correct format"), +] + + +class EnhancedQuery(BaseModel): + "A query suitable for similarity search in a vectorstore" + query_str: str = Field( + ..., description="A query suitable for similarity search and retrieval" + ) + + +def load_query_rewrite_chain(model: ChatOpenAI) -> RunnableSerializable: + + query_rewrite_prompt = ChatPromptTemplate.from_messages( + QUERY_REWRITE_PROMPT_MESSAGES + ) + + query_rewrite_chain = create_structured_output_runnable( + EnhancedQuery, model, query_rewrite_prompt + ) + + question_rewrite_chain = query_rewrite_chain | RunnableLambda( + lambda x: x.query_str + ) + + return question_rewrite_chain + + +def load_vectorsearch_enhancement_chain(model: ChatOpenAI) -> Runnable: + vectorsearch_chain = load_query_rewrite_chain(model) + + branch = RunnableBranch( + ( + lambda x: x["avoid"], + RunnableLambda(lambda x: []), + ), + ( + lambda x: not x["avoid"], + vectorsearch_chain, + ), + RunnableLambda(lambda x: []), + ) + + return ( + RunnableParallel( + question=itemgetter("standalone_question"), + avoid=itemgetter("avoid_query"), + ) + | branch + ) diff --git a/src/wandbot/query_handler/web_search.py b/src/wandbot/query_handler/web_search.py new file mode 100644 index 0000000..f7f0694 --- /dev/null +++ b/src/wandbot/query_handler/web_search.py @@ -0,0 +1,207 @@ +import os +from operator import itemgetter +from typing import List, Dict, Any + +import requests +from langchain_core.runnables import ( + RunnableLambda, + RunnableBranch, + RunnableParallel, + Runnable, + RunnablePassthrough, +) +from pydantic import BaseModel, Field + + +class YouSearchResults(BaseModel): + web_answer: str = Field("", description="response from you.com RAG model") + web_context: List[Dict[str, Any]] = Field( + [{}], description="context for the response" + ) + + +class YouSearch: + def __init__(self, api_key: str, similarity_top_k: int = 10): + self._api_key = api_key + self.similarity_top_k = similarity_top_k + + def _rag(self, query: str) -> YouSearchResults: + """Retrieve.""" + try: + headers = {"X-API-Key": self._api_key} + url = "https://api.ydc-index.io/rag" + + querystring = { + "query": "Weights & Biases, W&B, wandb or Weave " + query, + "num_web_results": self.similarity_top_k, + } + response = requests.get(url, headers=headers, params=querystring) + if response.status_code != 200: + return YouSearchResults() + else: + results = response.json() + + snippets = [hit["snippet"] for hit in results["hits"]] + snippet_metadata = [ + { + "source": hit["url"], + "language": "en", + "description": hit["description"], + "title": hit["title"], + "tags": ["you.com"], + "source_type": "web_search", + "has_code": None, + } + for hit in results["hits"] + ] + search_hits = [] + for snippet, metadata in zip(snippets, snippet_metadata): + search_hits.append({"context": snippet, "metadata": metadata}) + + return YouSearchResults( + web_answer=results["answer"], + web_context=search_hits[: self.similarity_top_k], + ) + except Exception as e: + return YouSearchResults() + + def _retrieve(self, query: str) -> YouSearchResults: + """Retrieve.""" + try: + headers = {"X-API-Key": self._api_key} + url = "https://api.ydc-index.io/search" + + querystring = { + "query": "Weights & Biases, W&B, wandb or Weave " + query, + "num_web_results": self.similarity_top_k, + } + response = requests.get(url, headers=headers, params=querystring) + if response.status_code != 200: + return YouSearchResults() + else: + results = response.json() + + snippets = [hit["snippets"] for hit in results["hits"]] + snippet_metadata = [ + { + "source": hit["url"], + "language": "en", + "description": hit["description"], + "title": hit["title"], + "tags": ["you.com"], + "source_type": "web_search", + "has_code": None, + } + for hit in results["hits"] + ] + search_hits = [] + for snippet_list, metadata in zip(snippets, snippet_metadata): + for snippet in snippet_list: + search_hits.append( + {"context": snippet, "metadata": metadata} + ) + + return YouSearchResults( + web_answer="", + web_context=search_hits[: self.similarity_top_k], + ) + except Exception as e: + print(e) + return YouSearchResults() + + def __call__( + self, question: str, search_type: str = "rag" + ) -> Dict[str, Any]: + if search_type == "rag": + web_results = self._rag(question) + else: + web_results = self._retrieve(question) + return web_results.dict() + + +def load_web_answer_chain(search_field: str, top_k: int = 5) -> Runnable: + + you_search = YouSearch(os.environ["YOU_API_KEY"], top_k) + + web_answer_chain = RunnablePassthrough().assign( + web_results=lambda x: you_search( + question=x["question"], search_type=x["search_type"] + ) + ) + + branch = RunnableBranch( + ( + lambda x: x["avoid"], + RunnableLambda(lambda x: None), + ), + ( + lambda x: not x["avoid"], + web_answer_chain | itemgetter("web_results"), + ), + RunnableLambda(lambda x: None), + ) + + return ( + RunnableParallel( + question=itemgetter(search_field), + search_type=itemgetter("search_type"), + avoid=itemgetter("avoid_query"), + ) + | branch + ) + + +def load_web_search_chain(search_field: str, top_k: int = 5) -> Runnable: + + you_search = YouSearch(os.environ["YOU_API_KEY"], top_k) + + web_answer_chain = RunnablePassthrough().assign( + web_results=lambda x: you_search( + question=x["question"], search_type=x["search_type"] + ) + ) + + branch = RunnableBranch( + ( + lambda x: x["avoid"], + RunnableLambda(lambda x: None), + ), + ( + lambda x: not x["avoid"], + web_answer_chain | itemgetter("web_results"), + ), + RunnableLambda(lambda x: None), + ) + + return ( + RunnableParallel( + question=itemgetter(search_field), + search_type=itemgetter("search_type"), + avoid=itemgetter("avoid_query"), + ) + | branch + ) + + +def load_web_answer_enhancement_chain(top_k: int = 5) -> Runnable: + web_answer_chain = load_web_answer_chain( + search_field="standalone_question", top_k=top_k + ) + + return ( + RunnablePassthrough().assign( + search_type=lambda x: "rag", + ) + | web_answer_chain + ) + + +def load_web_search_enhancement_chain(top_k: int = 5) -> Runnable: + web_answer_chain = load_web_answer_chain( + search_field="standalone_question", top_k=top_k + ) + + return ( + RunnablePassthrough().assign(search_type=lambda x: "rag") + | web_answer_chain + ) diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index c99fd74..296bb54 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,353 +1,35 @@ -from typing import Any, Dict, List, Optional, Tuple - import wandb -from llama_index import ( - QueryBundle, - ServiceContext, - StorageContext, - load_indices_from_storage, -) -from llama_index.callbacks import CallbackManager -from llama_index.core.base_retriever import BaseRetriever -from llama_index.postprocessor import BaseNodePostprocessor, CohereRerank -from llama_index.query_engine import RetrieverQueryEngine -from llama_index.response_synthesizers import BaseSynthesizer, ResponseMode -from llama_index.retrievers.fusion_retriever import FUSION_MODES -from llama_index.schema import NodeWithScore -from llama_index.vector_stores.simple import DEFAULT_VECTOR_STORE, NAMESPACE_SEP -from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict -from wandbot.retriever.fusion import FusionRetriever, HybridRetriever -from wandbot.retriever.postprocessors import ( - LanguageFilterPostprocessor, - MetadataPostprocessor, -) -from wandbot.utils import get_logger, load_service_context, load_storage_context - -logger = get_logger(__name__) - +from langchain_community.vectorstores.chroma import Chroma +from langchain_openai import OpenAIEmbeddings -class WandbRetrieverQueryEngine(RetrieverQueryEngine): - def __init__( - self, - retriever: FusionRetriever, - response_synthesizer: Optional[BaseSynthesizer] = None, - node_postprocessors: Optional[List[BaseNodePostprocessor]] = None, - callback_manager: Optional[CallbackManager] = None, - ) -> None: - super().__init__( - retriever=retriever, - response_synthesizer=response_synthesizer, - node_postprocessors=node_postprocessors, - callback_manager=callback_manager, - ) - self._retriever = retriever +from wandbot.ingestion.config import VectorStoreConfig - def retrieve( - self, query_bundle: QueryBundle, **kwargs - ) -> List[NodeWithScore]: - nodes = self._retriever.retrieve(query_bundle, **kwargs) - return self._apply_node_postprocessors(nodes, query_bundle=query_bundle) - -class RetrieverConfig(BaseSettings): - index_artifact: str = Field( - "wandbot/wandbot-dev/wandbot_index:latest", - env="WANDB_INDEX_ARTIFACT", - validation_alias="wandb_index_artifact", - ) - embeddings_model: str = "text-embedding-3-small" - embeddings_size: int = 512 - top_k: int = Field( - default=10, - env="RETRIEVER_TOP_K", +def load_vector_store_from_config(config: VectorStoreConfig): + embedding_fn = OpenAIEmbeddings( + model=config.embeddings_model, dimensions=config.embedding_dim ) - similarity_top_k: int = Field( - default=10, - env="RETRIEVER_SIMILARITY_TOP_K", - ) - language: str = Field( - default="en", - env="RETRIEVER_LANGUAGE", - ) - model_config = SettingsConfigDict( - env_file=".env", env_file_encoding="utf-8", extra="allow" - ) - - -class Retriever: - def __init__( - self, - config: RetrieverConfig | None = None, - run: wandb.wandb_sdk.wandb_run.Run | None = None, - service_context: ServiceContext | None = None, - callback_manager: CallbackManager | None = None, - ): - self.config = ( - config if isinstance(config, RetrieverConfig) else RetrieverConfig() - ) - self.run = ( - run - if run - else wandb.init( - project=self.config.wandb_project, - entity=self.config.wandb_entity, - job_type="retrieve", - ) - ) - self.service_context = ( - service_context - if service_context - else load_service_context( - embeddings_model=self.config.embeddings_model, - embeddings_size=self.config.embeddings_dim, - callback_manager=callback_manager, - ) - ) - - ( - self.storage_context, - index_ids, - ) = self.load_storage_context_from_artifact( - artifact_url=self.config.index_artifact - ) - - self.indices = load_indices_from_storage( - self.storage_context, - service_context=self.service_context, - index_ids=index_ids, - ) - retriever_list = [] - for index in self.indices: - retriever = HybridRetriever( - index=index, - similarity_top_k=self.config.similarity_top_k, - storage_context=self.storage_context, - ) - retriever_list.append(retriever) - self._retriever = FusionRetriever( - retriever_list, - similarity_top_k=self.config.similarity_top_k, - num_queries=1, - use_async=True, - mode=FUSION_MODES.RECIPROCAL_RANK, - ) - - self._retriever_map = dict(zip(index_ids, retriever_list)) - - self.is_avoid_query: bool | None = None - - def load_storage_context_from_artifact( - self, artifact_url: str - ) -> Tuple[StorageContext, Dict[str, str]]: - """Loads the storage context from the given artifact URL. - - Args: - artifact_url: A string representing the URL of the artifact. - - Returns: - An instance of StorageContext. - """ - artifact = self.run.use_artifact(artifact_url) - artifact_dir = artifact.download() - index_path = f"{artifact_dir}/{DEFAULT_VECTOR_STORE}{NAMESPACE_SEP}{DEFAULT_PERSIST_FNAME}" - logger.debug(f"Loading index from {index_path}") - storage_context = load_storage_context( - embed_dimensions=self.config.embeddings_size, - persist_dir=artifact_dir, - ) - return storage_context, artifact.metadata["index_ids"] - - def load_query_engine( - self, - retriever: BaseRetriever | None = None, - top_k: int | None = None, - language: str | None = None, - include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = None, - ) -> WandbRetrieverQueryEngine: - top_k = top_k or self.config.top_k - language = language or self.config.language - if is_avoid_query is not None: - self.is_avoid_query = is_avoid_query - - node_postprocessors = [ - MetadataPostprocessor( - include_tags=include_tags, - exclude_tags=exclude_tags, - min_result_size=top_k, - ), - LanguageFilterPostprocessor( - languages=[language, "python"], min_result_size=top_k - ), - CohereRerank(top_n=top_k, model="rerank-english-v2.0") - if language == "en" - else CohereRerank(top_n=top_k, model="rerank-multilingual-v2.0"), - ] - query_engine = WandbRetrieverQueryEngine.from_args( - retriever=retriever, - node_postprocessors=node_postprocessors, - response_mode=ResponseMode.NO_TEXT, - service_context=self.service_context, - ) - return query_engine - - def _retrieve( - self, - query: str, - indices: List[str] | None = None, - language: str | None = None, - top_k: int | None = None, - include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = False, - ): - """Retrieves the top k results from the index for the given query. - - Args: - query: A string representing the query. - indices: A list of strings representing the indices to retrieve the results from. - language: A string representing the language of the query. - top_k: An integer representing the number of top results to retrieve. - include_tags: A list of strings representing the tags to include in the results. - exclude_tags: A list of strings representing the tags to exclude from the results. - - Returns: - A list of dictionaries representing the retrieved results. - """ - top_k = top_k or self.config.top_k - language = language or self.config.language - retrievers = [] - for index in indices or []: - retriever = self._retriever_map.get(index) - if not retriever and index: - logger.warning( - f"Index {index} not found in retriever map. Skipping the index" - ) - retrievers.append(retriever) - - retrievers = [retriever for retriever in retrievers if retriever] - - if not retrievers: - logger.warning("No retrievers found. Defaulting to all retrievers") - retriever = self._retriever - else: - retriever = FusionRetriever( - retrievers, - similarity_top_k=self.config.similarity_top_k, - num_queries=1, - use_async=True, - mode=FUSION_MODES.RECIPROCAL_RANK, - ) - - retrieval_engine = self.load_query_engine( - retriever=retriever, - top_k=top_k, - language=language, - include_tags=include_tags, - exclude_tags=exclude_tags, - ) - - avoid_query = self.is_avoid_query or is_avoid_query - - query_bundle = QueryBundle( - query_str=query, - embedding=self.service_context.embed_model.get_query_embedding( - query=query - ), - ) - results = retrieval_engine.retrieve( - query_bundle, is_avoid_query=bool(avoid_query) - ) - - outputs = [ - { - "text": node.get_text(), - "metadata": node.metadata, - "score": node.get_score(), - } - for node in results - ] - self.is_avoid_query = None - return outputs - - def retrieve( - self, - query: str, - language: str | None = None, - top_k: int | None = None, - include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = False, - ): - """Retrieves the top k results from the index for the given query. - - Args: - query: A string representing the query. - language: A string representing the language of the query. - top_k: An integer representing the number of top results to retrieve. - include_tags: A list of strings representing the tags to include in the results. - exclude_tags: A list of strings representing the tags to exclude from the results. - - Returns: - A list of dictionaries representing the retrieved results. - """ - - return self._retrieve( - query, - indices=None, - language=language, - top_k=top_k, - include_tags=include_tags, - exclude_tags=exclude_tags, - is_avoid_query=is_avoid_query, - ) - - def retrieve_from_indices( - self, - query: str, - indices: List[str], - language: str | None = None, - top_k: int | None = None, - include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = False, - ): - """Retrieves the top k results from the index for the given query. + base_vectorstore = Chroma( + collection_name=config.name, + embedding_function=embedding_fn, + persist_directory=str(config.persist_dir), + ) + return base_vectorstore - Args: - query: A string representing the query. - indices: A string representing the index to retrieve the results from. - language: A string representing the language of the query. - top_k: An integer representing the number of top results to retrieve. - include_tags: A list of strings representing the tags to include in the results. - exclude_tags: A list of strings representing the tags to exclude from the results. - Returns: - A list of dictionaries representing the retrieved results. - """ - return self._retrieve( - query, - indices=indices, - language=language, - top_k=top_k, - include_tags=include_tags, - exclude_tags=exclude_tags, - is_avoid_query=is_avoid_query, - ) +def load_vector_store_from_artifact(artifact_url: str): + artifact = wandb.run.use_artifact(artifact_url) + artifact_dir = artifact.download() + config = VectorStoreConfig(persist_dir=artifact_dir) + base_vectorstore = load_vector_store_from_config(config) + return base_vectorstore - def __call__(self, query: str, **kwargs) -> List[Dict[str, Any]]: - indices = kwargs.pop("indices") - if indices and isinstance(indices, list): - retrievals = self.retrieve_from_indices( - query, indices=indices, **kwargs - ) - else: - retrievals = self.retrieve(query, **kwargs) - logger.debug(f"Retrieved {len(retrievals)} results.") - logger.debug(f"Retrieval: {retrievals[0]}") - return retrievals +def load_retriever_with_options( + base_vectorstore, search_type="mmr", search_kwargs={"k": 5} +): + base_retriever = base_vectorstore.as_retriever( + search_type=search_type, search_kwargs=search_kwargs + ) + return base_retriever diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index c284987..32a142e 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -1,256 +1,198 @@ -import asyncio -import os -from typing import Dict, List, Optional, Tuple - -import nest_asyncio -from llama_index import QueryBundle -from llama_index.callbacks import CallbackManager, CBEventType, EventPayload -from llama_index.constants import DEFAULT_SIMILARITY_TOP_K -from llama_index.core.base_retriever import BaseRetriever -from llama_index.llms.utils import LLMType -from llama_index.retrievers import BM25Retriever, QueryFusionRetriever -from llama_index.retrievers.fusion_retriever import FUSION_MODES -from llama_index.schema import IndexNode, NodeWithScore, QueryType -from wandbot.retriever.external import YouRetriever -from wandbot.utils import get_logger, run_async_tasks - -logger = get_logger(__name__) - - -class HybridRetriever(BaseRetriever): - def __init__( - self, - index, - storage_context, - similarity_top_k: int = 20, - ): - self.index = index - self.storage_context = storage_context - - self.vector_retriever = self.index.as_retriever( - similarity_top_k=similarity_top_k, - storage_context=self.storage_context, - ) - self.bm25_retriever = BM25Retriever.from_defaults( - docstore=self.index.docstore, - similarity_top_k=similarity_top_k, +import json +from operator import itemgetter + +from langchain.load import dumps, loads +from langchain.prompts.prompt import PromptTemplate +from langchain.retrievers.document_compressors import CohereRerank +from langchain.schema import format_document, Document +from langchain_core.runnables import ( + RunnablePassthrough, + RunnableParallel, + RunnableLambda, + RunnableBranch, +) + +from wandbot.utils import clean_document_content + +DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template( + template="source: {source}\nsource_type: {source_type}\nhas_code: {has_code}\n\n{page_content}" +) + + +def combine_documents( + docs, + document_prompt=DEFAULT_DOCUMENT_PROMPT, + document_separator="\n\n---\n\n", +): + cleaned_docs = [clean_document_content(doc) for doc in docs] + doc_strings = [ + format_document(doc, document_prompt) for doc in cleaned_docs + ] + return document_separator.join(doc_strings) + + +def process_input_for_retrieval(retrieval_input): + if isinstance(retrieval_input, list): + retrieval_input = "\n".join(retrieval_input) + elif isinstance(retrieval_input, dict): + retrieval_input = json.dumps(retrieval_input) + elif not isinstance(retrieval_input, str): + retrieval_input = str(retrieval_input) + return retrieval_input + + +def load_simple_retrieval_chain(retriever, input_key): + default_input_chain = ( + itemgetter("standalone_question") + | RunnablePassthrough() + | process_input_for_retrieval + | RunnableParallel(context=retriever) + | itemgetter("context") + ) + + input_chain = ( + itemgetter(input_key) + | RunnablePassthrough() + | process_input_for_retrieval + | RunnableParallel(context=retriever) + | itemgetter("context") + ) + + retrieval_chain = RunnableBranch( + ( + lambda x: not x["avoid_query"], + input_chain, + ), + ( + lambda x: x["avoid_query"], + default_input_chain, + ), + default_input_chain, + ) + + return retrieval_chain + + +def reciprocal_rank_fusion(results: list[list], k=60): + fused_scores = {} + for docs in results: + # Assumes the docs are returned in sorted order of relevance + for rank, doc in enumerate(docs): + doc_str = dumps(doc) + if doc_str not in fused_scores: + fused_scores[doc_str] = 0 + previous_score = fused_scores[doc_str] + fused_scores[doc_str] += 1 / (rank + k) + + ranked_results = [ + (loads(doc), score) + for doc, score in sorted( + fused_scores.items(), key=lambda x: x[1], reverse=True ) - self.you_retriever = YouRetriever( - api_key=os.environ.get("YOU_API_KEY"), - similarity_top_k=similarity_top_k, - ) - super().__init__() - - def _retrieve(self, query: QueryBundle, **kwargs): - nest_asyncio.apply() - return asyncio.run(self._aretrieve(query, **kwargs)) - - async def _aretrieve(self, query: QueryBundle, **kwargs): - bm25_nodes = await self.bm25_retriever.aretrieve(query) - vector_nodes = await self.vector_retriever.aretrieve(query) - you_nodes = ( - await self.you_retriever.aretrieve(query) - if not kwargs.get("is_avoid_query", False) - else [] - ) - - # combine the two lists of nodes - all_nodes = [] - node_ids = set() - for n in bm25_nodes + vector_nodes + you_nodes: - if n.node.node_id not in node_ids: - all_nodes.append(n) - node_ids.add(n.node.node_id) - return all_nodes + ] + return [item[0] for item in ranked_results] - def retrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - nest_asyncio.apply() - return asyncio.run(self.aretrieve(str_or_query_bundle, **kwargs)) - async def aretrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - self._check_callback_manager() - - if isinstance(str_or_query_bundle, str): - query_bundle = QueryBundle(str_or_query_bundle) +def load_cohere_rerank_chain(top_k=5): + def load_rerank_chain(language): + if language == "en": + cohere_rerank = CohereRerank( + top_n=top_k, model="rerank-english-v2.0" + ) else: - query_bundle = str_or_query_bundle - with self.callback_manager.as_trace("query"): - with self.callback_manager.event( - CBEventType.RETRIEVE, - payload={EventPayload.QUERY_STR: query_bundle.query_str}, - ) as retrieve_event: - nodes = await self._aretrieve(query_bundle, **kwargs) - retrieve_event.on_end( - payload={EventPayload.NODES: nodes}, - ) - return nodes - + cohere_rerank = CohereRerank( + top_n=top_k, model="rerank-multilingual-v2.0" + ) -class FusionRetriever(QueryFusionRetriever): - def __init__( - self, - retrievers: List[HybridRetriever], - llm: Optional[LLMType] = "default", - query_gen_prompt: Optional[str] = None, - mode: FUSION_MODES = FUSION_MODES.SIMPLE, - similarity_top_k: int = DEFAULT_SIMILARITY_TOP_K, - num_queries: int = 4, - use_async: bool = True, - verbose: bool = False, - callback_manager: Optional[CallbackManager] = None, - objects: Optional[List[IndexNode]] = None, - object_map: Optional[dict] = None, - ) -> None: - super().__init__( - retrievers=retrievers, - llm=llm, - query_gen_prompt=query_gen_prompt, - mode=mode, - similarity_top_k=similarity_top_k, - num_queries=num_queries, - use_async=use_async, - verbose=verbose, - callback_manager=callback_manager, - objects=objects, - object_map=object_map, + return lambda x: cohere_rerank.compress_documents( + documents=x["context"], query=x["question"] ) - self._retrievers = retrievers - - def _run_nested_async_queries( - self, queries: List[QueryBundle], **kwargs - ) -> Dict[Tuple[str, int], List[NodeWithScore]]: - tasks, task_queries = [], [] - for query in queries: - for i, retriever in enumerate(self._retrievers): - tasks.append(retriever.aretrieve(query, **kwargs)) - task_queries.append(query) - - task_results = run_async_tasks(tasks) - - results = {} - for i, (query, query_result) in enumerate( - zip(task_queries, task_results) - ): - results[(query.query_str, i)] = query_result - - return results - - def _run_sync_queries( - self, queries: List[QueryBundle], **kwargs - ) -> Dict[Tuple[str, int], List[NodeWithScore]]: - results = {} - for query in queries: - for i, retriever in enumerate(self._retrievers): - results[(query.query_str, i)] = retriever.retrieve( - query, **kwargs - ) - - return results - - async def _run_async_queries( - self, queries: List[QueryBundle], **kwargs - ) -> Dict[Tuple[str, int], List[NodeWithScore]]: - tasks, task_queries = [], [] - for query in queries: - for i, retriever in enumerate(self._retrievers): - tasks.append(retriever.aretrieve(query, **kwargs)) - task_queries.append(query) - - task_results = await asyncio.gather(*tasks) - - results = {} - for i, (query, query_result) in enumerate( - zip(task_queries, task_results) - ): - results[(query.query_str, i)] = query_result - return results - - def _retrieve( - self, query_bundle: QueryBundle, **kwargs - ) -> List[NodeWithScore]: - if self.num_queries > 1: - queries = self._get_queries(query_bundle.query_str) - else: - queries = [query_bundle] - - if self.use_async: - results = self._run_nested_async_queries(queries) - else: - results = self._run_sync_queries(queries, **kwargs) - - if self.mode == FUSION_MODES.RECIPROCAL_RANK: - return self._reciprocal_rerank_fusion(results)[ - : self.similarity_top_k - ] - elif self.mode == FUSION_MODES.SIMPLE: - return self._simple_fusion(results)[: self.similarity_top_k] - else: - raise ValueError(f"Invalid fusion mode: {self.mode}") - - async def _aretrieve( - self, query_bundle: QueryBundle, **kwargs - ) -> List[NodeWithScore]: - if self.num_queries > 1: - queries = self._get_queries(query_bundle.query_str) - else: - queries = [query_bundle] - - results = await self._run_async_queries(queries, **kwargs) - - if self.mode == FUSION_MODES.RECIPROCAL_RANK: - return self._reciprocal_rerank_fusion(results)[ - : self.similarity_top_k - ] - elif self.mode == FUSION_MODES.SIMPLE: - return self._simple_fusion(results)[: self.similarity_top_k] - else: - raise ValueError(f"Invalid fusion mode: {self.mode}") - - def retrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - self._check_callback_manager() - - if isinstance(str_or_query_bundle, str): - query_bundle = QueryBundle(str_or_query_bundle) - else: - query_bundle = str_or_query_bundle - with self.callback_manager.as_trace("query"): - with self.callback_manager.event( - CBEventType.RETRIEVE, - payload={EventPayload.QUERY_STR: query_bundle.query_str}, - ) as retrieve_event: - nodes = self._retrieve(query_bundle, **kwargs) - retrieve_event.on_end( - payload={EventPayload.NODES: nodes}, - ) - return nodes - - async def aretrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - self._check_callback_manager() - - if isinstance(str_or_query_bundle, str): - query_bundle = QueryBundle(str_or_query_bundle) - else: - query_bundle = str_or_query_bundle - with self.callback_manager.as_trace("query"): - with self.callback_manager.event( - CBEventType.RETRIEVE, - payload={EventPayload.QUERY_STR: query_bundle.query_str}, - ) as retrieve_event: - nodes = await self._aretrieve(query_bundle, **kwargs) - nodes = await self._ahandle_recursive_retrieval( - query_bundle, nodes - ) - retrieve_event.on_end( - payload={EventPayload.NODES: nodes}, - ) - - return nodes + cohere_rerank = RunnableBranch( + ( + lambda x: x["language"] == "en", + load_rerank_chain("en"), + ), + ( + lambda x: x["language"], + load_rerank_chain("ja"), + ), + load_rerank_chain("ja"), + ) + + return cohere_rerank + + +def get_web_contexts(web_results): + output_documents = [] + if not web_results: + return [] + web_answer = web_results["web_answer"] + # if web_answer: + # output_documents += [ + # Document( + # page_content=web_answer, + # metadata={ + # "source": "you.com", + # "source_type": "web_answer", + # "has_code": None, + # }, + # ) + # ] + return ( + output_documents + + [ + Document( + page_content=document["context"], metadata=document["metadata"] + ) + for document in web_results["web_context"] + ] + if web_results.get("web_context") + else [] + ) + + +def load_fusion_retriever_chain(base_retriever, top_k=5): + query_retrieval_chain = load_simple_retrieval_chain( + base_retriever, "question" + ) + standalone_query_retrieval_chain = load_simple_retrieval_chain( + base_retriever, "standalone_question" + ) + keywords_retrieval_chain = load_simple_retrieval_chain( + base_retriever, "keywords" + ) + vector_search_retrieval_chain = load_simple_retrieval_chain( + base_retriever, "vector_search" + ) + + combined_retrieval_chain = ( + RunnableParallel( + question=query_retrieval_chain, + standalone_question=standalone_query_retrieval_chain, + keywords=keywords_retrieval_chain, + vector_search=vector_search_retrieval_chain, + web_context=RunnableLambda( + lambda x: get_web_contexts(x["web_results"]) + ), + ) + | itemgetter( + "question", + "standalone_question", + "keywords", + "vector_search", + "web_context", + ) + | reciprocal_rank_fusion + ) + + cohere_rerank_chain = load_cohere_rerank_chain(top_k=top_k) + ranked_retrieval_chain = ( + RunnableParallel( + context=combined_retrieval_chain, + question=itemgetter("question"), + language=itemgetter("language"), + ) + | cohere_rerank_chain + ) + return ranked_retrieval_chain diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 3238906..87df361 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -20,6 +20,7 @@ storage_context = load_storage_context(768, "/path/to/persist") index = load_index(nodes, service_context, storage_context, "/path/to/persist") """ + import asyncio import datetime import hashlib @@ -27,13 +28,16 @@ import logging import os import pathlib +import re import sqlite3 from typing import Any, Coroutine, List, Optional, Tuple import faiss import fasttext import nest_asyncio +import tiktoken import wandb +from langchain_core.documents import Document from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.embeddings import OpenAIEmbedding from llama_index.llms import LiteLLM @@ -359,3 +363,63 @@ async def _gather() -> Tuple[Any]: outputs: Tuple[Any] = asyncio.run(_gather()) return outputs + + +def clean_document_content(doc: Document) -> Document: + cleaned_content = re.sub(r"\n{3,}", "\n\n", doc.page_content) + cleaned_content = cleaned_content.strip() + cleaned_document = Document( + page_content=cleaned_content, metadata=doc.metadata + ) + cleaned_document = make_document_tokenization_safe(cleaned_document) + return cleaned_document + + +def make_document_tokenization_safe(document: Document) -> Document: + """Removes special tokens from the given documents. + + Args: + documents: A list of strings representing the documents. + + Returns: + A list of cleaned documents with special tokens removed. + """ + encoding = tiktoken.get_encoding("cl100k_base") + special_tokens_set = encoding.special_tokens_set + + def remove_special_tokens(text: str) -> str: + """Removes special tokens from the given text. + + Args: + text: A string representing the text. + + Returns: + The text with special tokens removed. + """ + for token in special_tokens_set: + text = text.replace(token, "") + return text + + content = document.page_content + cleaned_document = remove_special_tokens(content) + return Document(page_content=cleaned_document, metadata=document.metadata) + + +def filter_smaller_documents( + documents: List[Document], min_size: int = 3, min_line_size: int = 5 +) -> List[Document]: + def filter_small_document(document: Document) -> bool: + return ( + len( + [ + doc + for doc in document.page_content.split("\n") + if len(doc.strip().split()) >= min_line_size + ] + ) + >= min_size + ) + + return [ + document for document in documents if filter_small_document(document) + ] From e933232bb4f9bbe14def967d174612ee96aad4ad Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Wed, 7 Feb 2024 16:33:15 +0530 Subject: [PATCH 12/41] refactor: formatting and linting --- src/wandbot/api/app.py | 3 ++- src/wandbot/api/client.py | 1 + src/wandbot/api/routers/chat.py | 1 + src/wandbot/api/routers/database.py | 3 ++- src/wandbot/api/routers/retrieve.py | 7 ++++--- src/wandbot/apps/discord/__main__.py | 1 + src/wandbot/apps/slack/__main__.py | 1 + src/wandbot/apps/slack/config.py | 8 ++++---- src/wandbot/apps/utils.py | 1 + src/wandbot/apps/zendesk/__main__.py | 6 +++--- src/wandbot/apps/zendesk/config.py | 6 +++--- src/wandbot/chat/chat.py | 15 +++++++++------ src/wandbot/chat/rag.py | 4 ++-- src/wandbot/database/client.py | 6 +++++- src/wandbot/evaluation/eval/__main__.py | 18 +++++++++--------- src/wandbot/ingestion/__main__.py | 2 +- src/wandbot/ingestion/prepare_data.py | 2 +- src/wandbot/ingestion/preprocess_data.py | 16 ++++++++-------- src/wandbot/ingestion/report.py | 1 + src/wandbot/ingestion/utils.py | 1 + src/wandbot/ingestion/vectorstores.py | 6 ++---- src/wandbot/query_handler/history_handler.py | 7 +++---- src/wandbot/query_handler/intents_enhancer.py | 2 +- .../query_handler/keyword_search_enhancer.py | 4 ++-- .../query_handler/language_detection.py | 2 +- src/wandbot/query_handler/query_enhancer.py | 2 +- .../query_handler/vector_search_enhancer.py | 6 +++--- src/wandbot/query_handler/web_search.py | 6 +++--- src/wandbot/retriever/base.py | 2 +- src/wandbot/retriever/external.py | 1 + src/wandbot/retriever/fusion.py | 8 ++++---- src/wandbot/retriever/postprocessors.py | 1 + src/wandbot/utils.py | 3 ++- 33 files changed, 85 insertions(+), 68 deletions(-) diff --git a/src/wandbot/api/app.py b/src/wandbot/api/app.py index cea5004..dea7960 100644 --- a/src/wandbot/api/app.py +++ b/src/wandbot/api/app.py @@ -33,8 +33,9 @@ from datetime import datetime, timezone import pandas as pd -import wandb from fastapi import FastAPI + +import wandb from wandbot.api.routers import chat as chat_router from wandbot.api.routers import database as database_router from wandbot.api.routers import retrieve as retrieve_router diff --git a/src/wandbot/api/client.py b/src/wandbot/api/client.py index 6354249..25065a0 100644 --- a/src/wandbot/api/client.py +++ b/src/wandbot/api/client.py @@ -15,6 +15,7 @@ import aiohttp import requests + from wandbot.api.routers.chat import APIQueryRequest, APIQueryResponse from wandbot.api.routers.database import ( APIFeedbackRequest, diff --git a/src/wandbot/api/routers/chat.py b/src/wandbot/api/routers/chat.py index 72aa9d4..9132453 100644 --- a/src/wandbot/api/routers/chat.py +++ b/src/wandbot/api/routers/chat.py @@ -1,5 +1,6 @@ from fastapi import APIRouter from starlette import status + from wandbot.chat.chat import Chat, ChatConfig from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.utils import get_logger diff --git a/src/wandbot/api/routers/database.py b/src/wandbot/api/routers/database.py index a625953..962f5fb 100644 --- a/src/wandbot/api/routers/database.py +++ b/src/wandbot/api/routers/database.py @@ -1,7 +1,8 @@ -import wandb from fastapi import APIRouter from starlette import status from starlette.responses import Response + +import wandb from wandbot.database.client import DatabaseClient from wandbot.database.database import engine from wandbot.database.models import Base diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py index 0652778..ac72896 100644 --- a/src/wandbot/api/routers/retrieve.py +++ b/src/wandbot/api/routers/retrieve.py @@ -4,6 +4,7 @@ from fastapi import APIRouter from pydantic import BaseModel from starlette import status + from wandbot.retriever.base import Retriever router = APIRouter( @@ -67,9 +68,9 @@ async def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: """ results = retriever( query=request.query, - indices=[idx.value for idx in request.indices] - if request.indices - else None, + indices=( + [idx.value for idx in request.indices] if request.indices else None + ), language=request.language, top_k=request.top_k, include_tags=request.include_tags, diff --git a/src/wandbot/apps/discord/__main__.py b/src/wandbot/apps/discord/__main__.py index 867ffc5..dfc24fe 100644 --- a/src/wandbot/apps/discord/__main__.py +++ b/src/wandbot/apps/discord/__main__.py @@ -5,6 +5,7 @@ formats the responses, and sends them back to the user. It also handles user feedback on the bot's responses. """ + import asyncio import logging import uuid diff --git a/src/wandbot/apps/slack/__main__.py b/src/wandbot/apps/slack/__main__.py index 60b57e7..7c230aa 100644 --- a/src/wandbot/apps/slack/__main__.py +++ b/src/wandbot/apps/slack/__main__.py @@ -8,6 +8,7 @@ It also communicates with an external API for processing queries and storing chat history and feedback. """ + import argparse import asyncio import logging diff --git a/src/wandbot/apps/slack/config.py b/src/wandbot/apps/slack/config.py index 5b903df..92692df 100644 --- a/src/wandbot/apps/slack/config.py +++ b/src/wandbot/apps/slack/config.py @@ -48,12 +48,12 @@ "#support チャンネルにいるwandbチームに質問してください。この答えは役に立ったでしょうか？下のボタンでお知らせ下さい。" ) -JA_ERROR_MESSAGE = "「おっと、問題が発生しました。しばらくしてからもう一度お試しください。」" - -JA_FALLBACK_WARNING_MESSAGE = ( - "**警告: {model}** にフォールバックします。これらの結果は **gpt-4** ほど良くない可能性があります\n\n" +JA_ERROR_MESSAGE = ( + "「おっと、問題が発生しました。しばらくしてからもう一度お試しください。」" ) +JA_FALLBACK_WARNING_MESSAGE = "**警告: {model}** にフォールバックします。これらの結果は **gpt-4** ほど良くない可能性があります\n\n" + class SlackAppEnConfig(BaseSettings): APPLICATION: str = Field("Slack_EN") diff --git a/src/wandbot/apps/utils.py b/src/wandbot/apps/utils.py index bdf5762..d75f249 100644 --- a/src/wandbot/apps/utils.py +++ b/src/wandbot/apps/utils.py @@ -16,6 +16,7 @@ from typing import Any, List from pydantic_settings import BaseSettings + from wandbot.api.routers.chat import APIQueryResponse diff --git a/src/wandbot/apps/zendesk/__main__.py b/src/wandbot/apps/zendesk/__main__.py index 255083c..ff17b96 100644 --- a/src/wandbot/apps/zendesk/__main__.py +++ b/src/wandbot/apps/zendesk/__main__.py @@ -20,6 +20,7 @@ ZendeskWandBotResponseSystem object and runs it in an event loop. """ + import asyncio from typing import List @@ -28,9 +29,8 @@ from wandbot.api.client import AsyncAPIClient from wandbot.apps.zendesk.config import zendesk_app_config -from wandbot.utils import get_logger from wandbot.apps.zendesk.extract_by_type import * - +from wandbot.utils import get_logger logger = get_logger(__name__) config = zendesk_app_config() @@ -90,6 +90,7 @@ class ZendeskWandBotResponseSystem: api_client (AsyncAPIClient): The client for interacting with the WandBot API. semaphore (Semaphore): Use semaphore to control how many api calls to wandbot we make """ + def __init__(self) -> None: """Initializes the ZendeskWandBotResponseSystem with the necessary clients. @@ -126,7 +127,6 @@ def fetch_new_tickets(self) -> List[Ticket]: ) return new_tickets - async def generate_response(self, question: str) -> str: """Generates a response to a given question. diff --git a/src/wandbot/apps/zendesk/config.py b/src/wandbot/apps/zendesk/config.py index 19ac601..755f1c3 100644 --- a/src/wandbot/apps/zendesk/config.py +++ b/src/wandbot/apps/zendesk/config.py @@ -7,10 +7,10 @@ class zendesk_app_config(BaseSettings): DISCBOTINTRO: str = ( "Thank you for reaching out to W&B Technical Support!\n\n" - f"This is an automated reply from our support bot designed to assist you with your WandB-related queries.\n" - f"If you find the solution unsatisfactory or have additional questions, we encourage you to contact our support team at support@wandb.com, or continue replying in this thread\n\n" + f"This is an automated reply from our support bot designed to assist you with your WandB-related queries.\n" + f"If you find the solution unsatisfactory or have additional questions, we encourage you to contact our support team at support@wandb.com, or continue replying in this thread\n\n" ) - + ZENDESK_EMAIL: str = (Field(..., env="ZENDESK_EMAIL"),) ZENDESK_PASSWORD: str = (Field(..., env="ZENDESK_PASSWORD"),) ZENDESK_SUBDOMAIN: str = (Field(..., env="ZENDESK_SUBDOMAIN"),) diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 99fe55d..3aa0ba5 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -28,7 +28,6 @@ import json from typing import Any, Dict, List, Optional, Tuple -import wandb from llama_index import ServiceContext from llama_index.callbacks import ( CallbackManager, @@ -45,6 +44,9 @@ from llama_index.postprocessor.types import BaseNodePostprocessor from llama_index.schema import MetadataMode, NodeWithScore, QueryBundle from llama_index.tools import ToolOutput +from weave.monitoring import StreamTable + +import wandb from wandbot.chat.config import ChatConfig from wandbot.chat.prompts import load_chat_prompt, partial_format from wandbot.chat.query_enhancer import CompleteQuery, QueryHandler @@ -56,7 +58,6 @@ MetadataPostprocessor, ) from wandbot.utils import Timer, get_logger, load_service_context -from weave.monitoring import StreamTable logger = get_logger(__name__) @@ -316,10 +317,12 @@ def _load_chat_engine( node_postprocessors=[ MetadataPostprocessor(), LanguageFilterPostprocessor(languages=[language, "python"]), - CohereRerank(top_n=top_k, model="rerank-english-v2.0") - if language == "en" - else CohereRerank( - top_n=top_k, model="rerank-multilingual-v2.0" + ( + CohereRerank(top_n=top_k, model="rerank-english-v2.0") + if language == "en" + else CohereRerank( + top_n=top_k, model="rerank-multilingual-v2.0" + ) ), ], prefix_messages=self.qa_prompt.message_templates, diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 91790f2..af88d22 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -1,13 +1,13 @@ from operator import itemgetter -from langchain_core.runnables import RunnableParallel, RunnableLambda +from langchain_core.runnables import RunnableLambda, RunnableParallel from wandbot.chat.response_synthesis import load_response_synthesizer_chain from wandbot.ingestion.config import VectorStoreConfig from wandbot.query_handler.query_enhancer import load_query_enhancement_chain from wandbot.retriever.base import ( - load_vector_store_from_config, load_retriever_with_options, + load_vector_store_from_config, ) from wandbot.retriever.fusion import load_fusion_retriever_chain diff --git a/src/wandbot/database/client.py b/src/wandbot/database/client.py index d5097d5..8052e33 100644 --- a/src/wandbot/database/client.py +++ b/src/wandbot/database/client.py @@ -12,18 +12,22 @@ chat_thread = db_client.get_chat_thread(application='app1', thread_id='123') question_answer = db_client.create_question_answer(question_answer=QuestionAnswerCreateSchema()) """ + import json from typing import Any, List from sqlalchemy.future import create_engine from sqlalchemy.orm import sessionmaker + from wandbot.database.config import DataBaseConfig from wandbot.database.models import ChatThread as ChatThreadModel from wandbot.database.models import FeedBack as FeedBackModel from wandbot.database.models import QuestionAnswer as QuestionAnswerModel from wandbot.database.schemas import ChatThreadCreate as ChatThreadCreateSchema from wandbot.database.schemas import Feedback as FeedbackSchema -from wandbot.database.schemas import QuestionAnswerCreate as QuestionAnswerCreateSchema +from wandbot.database.schemas import ( + QuestionAnswerCreate as QuestionAnswerCreateSchema, +) from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/evaluation/eval/__main__.py b/src/wandbot/evaluation/eval/__main__.py index 3302f59..78de74c 100644 --- a/src/wandbot/evaluation/eval/__main__.py +++ b/src/wandbot/evaluation/eval/__main__.py @@ -95,9 +95,9 @@ def get_answer_correctness(row_str: str) -> str: reference_notes=row["reference_notes"], ) result = parse_answer_eval("answer_correctness", result.dict()) - result[ - "answer_correctness_score_(ragas)" - ] = metrics.answer_correctness.score_single(row) + result["answer_correctness_score_(ragas)"] = ( + metrics.answer_correctness.score_single(row) + ) result = json.dumps(result) return result @@ -113,9 +113,9 @@ def get_answer_relevancy(row_str: str) -> str: reference=row["ground_truths"], ) result = parse_answer_eval("answer_relevancy", result.dict()) - result[ - "answer_relevancy_score_(ragas)" - ] = metrics.answer_relevancy.score_single(row) + result["answer_relevancy_score_(ragas)"] = ( + metrics.answer_relevancy.score_single(row) + ) result = json.dumps(result) return result @@ -132,9 +132,9 @@ def get_answer_faithfulness(row_str: str) -> str: ) result = parse_answer_eval("answer_faithfulness", result.dict()) - result[ - "answer_faithfulness_score_(ragas)" - ] = metrics.faithfulness.score_single(row) + result["answer_faithfulness_score_(ragas)"] = ( + metrics.faithfulness.score_single(row) + ) result = json.dumps(result) return result diff --git a/src/wandbot/ingestion/__main__.py b/src/wandbot/ingestion/__main__.py index 75402ac..863251e 100644 --- a/src/wandbot/ingestion/__main__.py +++ b/src/wandbot/ingestion/__main__.py @@ -1,6 +1,6 @@ import os -from wandbot.ingestion import vectorstores, preprocess_data +from wandbot.ingestion import preprocess_data, vectorstores from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/ingestion/prepare_data.py b/src/wandbot/ingestion/prepare_data.py index 733745a..c241a78 100644 --- a/src/wandbot/ingestion/prepare_data.py +++ b/src/wandbot/ingestion/prepare_data.py @@ -23,13 +23,13 @@ import nbformat import pandas as pd -import wandb from google.cloud import bigquery from langchain.schema import Document from langchain_community.document_loaders import TextLoader from langchain_community.document_loaders.base import BaseLoader from nbconvert import MarkdownExporter +import wandb from wandbot.ingestion.config import ( DataStoreConfig, DocodileEnglishStoreConfig, diff --git a/src/wandbot/ingestion/preprocess_data.py b/src/wandbot/ingestion/preprocess_data.py index 8e0a00f..f668de4 100644 --- a/src/wandbot/ingestion/preprocess_data.py +++ b/src/wandbot/ingestion/preprocess_data.py @@ -21,23 +21,23 @@ import json import pathlib -from typing import Any, List, Sequence, Callable +from typing import Any, Callable, List, Sequence import tiktoken -import wandb from langchain.text_splitter import ( - RecursiveCharacterTextSplitter, - MarkdownHeaderTextSplitter, Language, + MarkdownHeaderTextSplitter, + RecursiveCharacterTextSplitter, + TokenTextSplitter, ) -from langchain.text_splitter import TokenTextSplitter -from langchain_core.documents import Document, BaseDocumentTransformer +from langchain_core.documents import BaseDocumentTransformer, Document +import wandb from wandbot.utils import ( - get_logger, FastTextLangDetect, - make_document_tokenization_safe, filter_smaller_documents, + get_logger, + make_document_tokenization_safe, ) logger = get_logger(__name__) diff --git a/src/wandbot/ingestion/report.py b/src/wandbot/ingestion/report.py index 37c6ca5..5f68fe6 100644 --- a/src/wandbot/ingestion/report.py +++ b/src/wandbot/ingestion/report.py @@ -14,6 +14,7 @@ vectorstore_artifact = "wandbot/wandbot-dev/vectorstores:latest" create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) """ + import json import pathlib from datetime import datetime diff --git a/src/wandbot/ingestion/utils.py b/src/wandbot/ingestion/utils.py index 064fcf7..8548efe 100644 --- a/src/wandbot/ingestion/utils.py +++ b/src/wandbot/ingestion/utils.py @@ -25,6 +25,7 @@ git_repo_metadata = fetch_git_repo(paths, id_file) cell_info = concatenate_cells(cell, include_outputs, max_output_length, traceback) """ + import pathlib import re import subprocess diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index 029ac38..03ad233 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -16,16 +16,14 @@ import pathlib from typing import List -import wandb from langchain_community.vectorstores.chroma import Chroma from langchain_core.documents import Document from langchain_openai import OpenAIEmbeddings from tqdm import trange +import wandb from wandbot.ingestion.config import VectorStoreConfig -from wandbot.utils import ( - get_logger, -) +from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/query_handler/history_handler.py b/src/wandbot/query_handler/history_handler.py index ee38f29..0e93c92 100644 --- a/src/wandbot/query_handler/history_handler.py +++ b/src/wandbot/query_handler/history_handler.py @@ -1,13 +1,12 @@ from _operator import itemgetter - from langchain_core.messages import get_buffer_string from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import ( - RunnablePassthrough, - RunnableLambda, - RunnableBranch, Runnable, + RunnableBranch, + RunnableLambda, + RunnablePassthrough, ) from langchain_openai import ChatOpenAI diff --git a/src/wandbot/query_handler/intents_enhancer.py b/src/wandbot/query_handler/intents_enhancer.py index a2b8644..700d947 100644 --- a/src/wandbot/query_handler/intents_enhancer.py +++ b/src/wandbot/query_handler/intents_enhancer.py @@ -7,8 +7,8 @@ from langchain.chains.openai_functions import create_structured_output_runnable from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import ( - RunnableLambda, Runnable, + RunnableLambda, RunnableParallel, RunnablePassthrough, ) diff --git a/src/wandbot/query_handler/keyword_search_enhancer.py b/src/wandbot/query_handler/keyword_search_enhancer.py index e1ad371..37e208e 100644 --- a/src/wandbot/query_handler/keyword_search_enhancer.py +++ b/src/wandbot/query_handler/keyword_search_enhancer.py @@ -4,10 +4,10 @@ from langchain.chains.openai_functions import create_structured_output_runnable from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import ( - RunnableLambda, - RunnableParallel, Runnable, RunnableBranch, + RunnableLambda, + RunnableParallel, ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field diff --git a/src/wandbot/query_handler/language_detection.py b/src/wandbot/query_handler/language_detection.py index 847d9d5..4e5484e 100644 --- a/src/wandbot/query_handler/language_detection.py +++ b/src/wandbot/query_handler/language_detection.py @@ -1,6 +1,6 @@ from operator import itemgetter -from langchain_core.runnables import RunnablePassthrough, Runnable +from langchain_core.runnables import Runnable, RunnablePassthrough from wandbot.utils import FastTextLangDetect, FasttextModelConfig diff --git a/src/wandbot/query_handler/query_enhancer.py b/src/wandbot/query_handler/query_enhancer.py index 8864d2f..3fbaec1 100644 --- a/src/wandbot/query_handler/query_enhancer.py +++ b/src/wandbot/query_handler/query_enhancer.py @@ -3,8 +3,8 @@ import regex as re from langchain_core.runnables import ( Runnable, - RunnablePassthrough, RunnableParallel, + RunnablePassthrough, ) from langchain_openai import ChatOpenAI diff --git a/src/wandbot/query_handler/vector_search_enhancer.py b/src/wandbot/query_handler/vector_search_enhancer.py index c47756e..50b7a4d 100644 --- a/src/wandbot/query_handler/vector_search_enhancer.py +++ b/src/wandbot/query_handler/vector_search_enhancer.py @@ -3,11 +3,11 @@ from langchain.chains.openai_functions import create_structured_output_runnable from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import ( - RunnableSerializable, - RunnableLambda, + Runnable, RunnableBranch, + RunnableLambda, RunnableParallel, - Runnable, + RunnableSerializable, ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field diff --git a/src/wandbot/query_handler/web_search.py b/src/wandbot/query_handler/web_search.py index f7f0694..0c79228 100644 --- a/src/wandbot/query_handler/web_search.py +++ b/src/wandbot/query_handler/web_search.py @@ -1,13 +1,13 @@ import os from operator import itemgetter -from typing import List, Dict, Any +from typing import Any, Dict, List import requests from langchain_core.runnables import ( - RunnableLambda, + Runnable, RunnableBranch, + RunnableLambda, RunnableParallel, - Runnable, RunnablePassthrough, ) from pydantic import BaseModel, Field diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index 296bb54..5b77af2 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,7 +1,7 @@ -import wandb from langchain_community.vectorstores.chroma import Chroma from langchain_openai import OpenAIEmbeddings +import wandb from wandbot.ingestion.config import VectorStoreConfig diff --git a/src/wandbot/retriever/external.py b/src/wandbot/retriever/external.py index 7a42917..552103a 100644 --- a/src/wandbot/retriever/external.py +++ b/src/wandbot/retriever/external.py @@ -6,6 +6,7 @@ from llama_index.callbacks import CallbackManager from llama_index.core.base_retriever import BaseRetriever from llama_index.schema import NodeWithScore, TextNode + from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index 32a142e..76a1432 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -4,12 +4,12 @@ from langchain.load import dumps, loads from langchain.prompts.prompt import PromptTemplate from langchain.retrievers.document_compressors import CohereRerank -from langchain.schema import format_document, Document +from langchain.schema import Document, format_document from langchain_core.runnables import ( - RunnablePassthrough, - RunnableParallel, - RunnableLambda, RunnableBranch, + RunnableLambda, + RunnableParallel, + RunnablePassthrough, ) from wandbot.utils import clean_document_content diff --git a/src/wandbot/retriever/postprocessors.py b/src/wandbot/retriever/postprocessors.py index 544f22b..6cb34a3 100644 --- a/src/wandbot/retriever/postprocessors.py +++ b/src/wandbot/retriever/postprocessors.py @@ -3,6 +3,7 @@ from llama_index import QueryBundle from llama_index.postprocessor import BaseNodePostprocessor from llama_index.schema import NodeWithScore + from wandbot.utils import create_no_result_dummy_node, get_logger logger = get_logger(__name__) diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 87df361..c4aa001 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -36,7 +36,6 @@ import fasttext import nest_asyncio import tiktoken -import wandb from langchain_core.documents import Document from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.embeddings import OpenAIEmbedding @@ -46,6 +45,8 @@ from llama_index.vector_stores import FaissVectorStore from pydantic_settings import BaseSettings +import wandb + def get_logger(name: str) -> logging.Logger: """Creates and returns a logger with the specified name. From 281bec72cd6b5c77990c03b556c98e2ed812b237 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 8 Feb 2024 15:58:05 +0530 Subject: [PATCH 13/41] stash: stash changes in handlers --- poetry.lock | 20 ++++++++++++- pyproject.toml | 1 + src/wandbot/chat/rag.py | 20 +++++++++---- src/wandbot/query_handler/history_handler.py | 28 +++++++----------- src/wandbot/query_handler/intents_enhancer.py | 2 ++ src/wandbot/retriever/fusion.py | 29 +++++++++++++++++-- 6 files changed, 73 insertions(+), 27 deletions(-) diff --git a/poetry.lock b/poetry.lock index 769e293..037b0fd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2662,6 +2662,24 @@ tenacity = ">=8.1.0,<9.0.0" [package.extras] extended-testing = ["jinja2 (>=3,<4)"] +[[package]] +name = "langchain-experimental" +version = "0.0.50" +description = "Building applications with LLMs through composability" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "langchain_experimental-0.0.50-py3-none-any.whl", hash = "sha256:182cef2b54e9c17a7c037cfbac12aede1bb3f6e92c5ac59a3ce6bb37aef616a6"}, + {file = "langchain_experimental-0.0.50.tar.gz", hash = "sha256:486da8a80d7a2c97402dbc3a3ad8aa7576e928b2333869575c9360a2b94ed68f"}, +] + +[package.dependencies] +langchain = ">=0.1.5,<0.2.0" +langchain-core = ">=0.1.16,<0.2.0" + +[package.extras] +extended-testing = ["faker (>=19.3.1,<20.0.0)", "jinja2 (>=3,<4)", "presidio-analyzer (>=2.2.352,<3.0.0)", "presidio-anonymizer (>=2.2.352,<3.0.0)", "sentence-transformers (>=2,<3)", "vowpal-wabbit-next (==0.6.0)"] + [[package]] name = "langchain-openai" version = "0.0.5" @@ -7048,4 +7066,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10.0,<3.12" -content-hash = "32552824ffb806b78f2b60145d4cae8f0688b1ccc46896c51852cc52913c8447" +content-hash = "f92e9e6633c536aa99e2aed191106b00c0bea2e719f2002bc696c321393c5e99" diff --git a/pyproject.toml b/pyproject.toml index 7e59aa0..9418657 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ pymdown-extensions = "^10.5" langchain = "^0.1.5" langchain-openai = "^0.0.5" chromadb = "^0.4.22" +langchain-experimental = "^0.0.50" [tool.poetry.dev-dependencies] #fasttext = {git = "https://github.com/cfculhane/fastText"} # FastText doesn't come with pybind11 and we need to use this workaround. diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index af88d22..fd0a369 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -12,7 +12,15 @@ from wandbot.retriever.fusion import load_fusion_retriever_chain -def load_rag_chain(model, fallback_model, lang_detect_path, vector_store_path): +def load_rag_chain( + model, + fallback_model, + embeddings_model, + lang_detect_path, + vector_store_path, + search_type, + top_k=10, +): fallback_query_enhancer_chain = load_query_enhancement_chain( fallback_model, lang_detect_path ) @@ -21,10 +29,9 @@ def load_rag_chain(model, fallback_model, lang_detect_path, vector_store_path): ).with_fallbacks([fallback_query_enhancer_chain]) vectorstore_config = VectorStoreConfig(persist_dir=vector_store_path) - print(vectorstore_config.persist_dir.resolve()) vectorstore = load_vector_store_from_config(vectorstore_config) base_retriever = load_retriever_with_options( - vectorstore, search_type="similarity" + vectorstore, search_type=search_type, search_kwargs={"top_k": top_k * 2} ) fallback_response_synthesis_chain = load_response_synthesizer_chain( @@ -35,7 +42,7 @@ def load_rag_chain(model, fallback_model, lang_detect_path, vector_store_path): ).with_fallbacks([fallback_response_synthesis_chain]) ranked_retrieval_chain = load_fusion_retriever_chain( - base_retriever, + base_retriever, embeddings=embeddings_model, top_k=top_k ) rag_chain = ( @@ -51,7 +58,10 @@ def load_rag_chain(model, fallback_model, lang_detect_path, vector_store_path): | RunnableParallel( query=itemgetter("query"), context=RunnableLambda( - lambda x: [p.page_content for p in x["context"]] + lambda x: [ + {"page_content": p.page_content, "metadata": p.metadata} + for p in x["context"] + ] ), result=response_synthesis_chain, ).with_config({"run_name": "response_synthesis"}) diff --git a/src/wandbot/query_handler/history_handler.py b/src/wandbot/query_handler/history_handler.py index 0e93c92..db5f967 100644 --- a/src/wandbot/query_handler/history_handler.py +++ b/src/wandbot/query_handler/history_handler.py @@ -1,4 +1,5 @@ from _operator import itemgetter + from langchain_core.messages import get_buffer_string from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate @@ -10,28 +11,19 @@ ) from langchain_openai import ChatOpenAI -CONDENSE_PROMPT_SYSTEM_TEMPLATE = ( - "Given the following conversation between a user and an AI assistant and a follow up " - "question from user, rephrase the follow up question to be a standalone question. Ensure " - "that the standalone question summarizes the conversation and completes the follow up " - "question with all the necessary context. If there is no history return the original question verbatim" -) +CONDENSE_PROMPT_SYSTEM_TEMPLATE = """Given the following conversation and a follow up question, rephrase the follow up \ +question to be a standalone question. + +Chat History: +{chat_history} +Follow Up Input: {question} +Standalone Question:""" CONDENSE_PROMPT_MESSAGES = [ - ("system", CONDENSE_PROMPT_SYSTEM_TEMPLATE), ( - "human", - """Rephrase the following question to be a standalone question based on the given history: - - -{chat_history} - - - -{question} - -""", + "system", + CONDENSE_PROMPT_SYSTEM_TEMPLATE, ), ] diff --git a/src/wandbot/query_handler/intents_enhancer.py b/src/wandbot/query_handler/intents_enhancer.py index 700d947..0be3869 100644 --- a/src/wandbot/query_handler/intents_enhancer.py +++ b/src/wandbot/query_handler/intents_enhancer.py @@ -47,6 +47,8 @@ class MultiLabel(BaseModel): intents: List[Label] = Field( ..., description="The list of intents associated with the query", + min_items=1, + max_items=3, ) diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index 76a1432..342661d 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -3,8 +3,12 @@ from langchain.load import dumps, loads from langchain.prompts.prompt import PromptTemplate -from langchain.retrievers.document_compressors import CohereRerank +from langchain.retrievers.document_compressors import ( + CohereRerank, +) from langchain.schema import Document, format_document +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain_community.document_transformers import EmbeddingsRedundantFilter from langchain_core.runnables import ( RunnableBranch, RunnableLambda, @@ -152,7 +156,7 @@ def get_web_contexts(web_results): ) -def load_fusion_retriever_chain(base_retriever, top_k=5): +def load_fusion_retriever_chain(base_retriever, embeddings, top_k=5): query_retrieval_chain = load_simple_retrieval_chain( base_retriever, "question" ) @@ -187,9 +191,28 @@ def load_fusion_retriever_chain(base_retriever, top_k=5): ) cohere_rerank_chain = load_cohere_rerank_chain(top_k=top_k) + + splitter = RecursiveCharacterTextSplitter( + chunk_size=300, + chunk_overlap=0, + separators=["\n```\n", "\n\n", "\n"], + keep_separator=False, + ) + + redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings) + ranked_retrieval_chain = ( RunnableParallel( - context=combined_retrieval_chain, + context=combined_retrieval_chain + | splitter.split_documents + | ( + lambda x: [ + doc + for doc in x + if len("".join(doc.page_content.strip().split())) > 10 + ] + ) + | redundant_filter.transform_documents, question=itemgetter("question"), language=itemgetter("language"), ) From 0cc6bda24ddcf933fc75c3a446014619fd75e24f Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 9 Feb 2024 08:21:52 +0530 Subject: [PATCH 14/41] chore: switch add local changes on swordfish --- src/wandbot/api/routers/retrieve.py | 9 +- src/wandbot/chat/chat.py | 13 +- src/wandbot/retriever/base.py | 239 +++------------------ src/wandbot/retriever/fusion.py | 262 ++++++------------------ src/wandbot/retriever/postprocessors.py | 83 -------- 5 files changed, 90 insertions(+), 516 deletions(-) delete mode 100644 src/wandbot/retriever/postprocessors.py diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py index b2e9750..4700837 100644 --- a/src/wandbot/api/routers/retrieve.py +++ b/src/wandbot/api/routers/retrieve.py @@ -43,12 +43,10 @@ class Indices(str, Enum): class APIRetrievalRequest(BaseModel): query: str - indices: List[Indices] = [] + indices: List[Indices] = [index for index in Indices] language: str = "en" - initial_k: int = 10 top_k: int = 5 include_tags: List[str] = [] - exclude_tags: List[str] = [] include_web_results: bool = True @@ -68,13 +66,10 @@ def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: """ results = retriever( query=request.query, - indices=[idx.value for idx in request.indices] - if request.indices - else [], + indices=[idx.value for idx in request.indices], language=request.language, top_k=request.top_k, include_tags=request.include_tags, - exclude_tags=request.exclude_tags, include_web_results=request.include_web_results, ) diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 1b070d6..e6c5dde 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -50,11 +50,7 @@ from wandbot.chat.query_enhancer import CompleteQuery, QueryHandler from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.retriever.base import Retriever -from wandbot.retriever.fusion import FusionRetriever -from wandbot.retriever.postprocessors import ( - LanguageFilterPostprocessor, - MetadataPostprocessor, -) +from wandbot.retriever.fusion import HybridRetriever from wandbot.utils import Timer, get_logger, load_service_context from weave.monitoring import StreamTable @@ -95,7 +91,7 @@ def rebuild_full_prompt( class WandbContextChatEngine(ContextChatEngine): def __init__( self, - retriever: FusionRetriever, + retriever: HybridRetriever, llm: LLM, memory: BaseMemory, prefix_messages: List[ChatMessage], @@ -112,7 +108,7 @@ def __init__( context_template=context_template, callback_manager=callback_manager, ) - self._retriever: FusionRetriever = retriever + self._retriever: HybridRetriever = retriever def _generate_context( self, message: str, **kwargs @@ -299,7 +295,6 @@ def _load_chat_engine( query_engine = self.retriever.load_query_engine( language=language, top_k=top_k, - is_avoid_query=True if "avoid" in query_intent.lower() else False, ) self.qa_prompt = load_chat_prompt( @@ -314,8 +309,6 @@ def _load_chat_engine( similarity_top_k=initial_k, response_mode="compact", node_postprocessors=[ - MetadataPostprocessor(), - LanguageFilterPostprocessor(languages=[language, "python"]), CohereRerank(top_n=top_k, model="rerank-english-v2.0") if language == "en" else CohereRerank( diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index d3b4af8..d40e71b 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,5 +1,4 @@ -import os -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Tuple import wandb from llama_index import ( @@ -10,55 +9,19 @@ ) from llama_index.callbacks import CallbackManager from llama_index.core.base_retriever import BaseRetriever -from llama_index.postprocessor import BaseNodePostprocessor, CohereRerank +from llama_index.postprocessor import CohereRerank from llama_index.query_engine import RetrieverQueryEngine -from llama_index.response_synthesizers import BaseSynthesizer, ResponseMode -from llama_index.retrievers import BM25Retriever -from llama_index.retrievers.fusion_retriever import FUSION_MODES -from llama_index.schema import NodeWithScore, TextNode +from llama_index.response_synthesizers import ResponseMode from llama_index.vector_stores.simple import DEFAULT_VECTOR_STORE, NAMESPACE_SEP -from llama_index.vector_stores.types import ( - DEFAULT_PERSIST_FNAME, - ExactMatchFilter, - FilterCondition, - MetadataFilters, -) +from llama_index.vector_stores.types import DEFAULT_PERSIST_FNAME from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict -from wandbot.retriever.external import YouRetriever -from wandbot.retriever.fusion import FusionRetriever -from wandbot.retriever.postprocessors import ( - LanguageFilterPostprocessor, - MetadataPostprocessor, -) +from wandbot.retriever.fusion import HybridRetriever from wandbot.utils import get_logger, load_service_context, load_storage_context logger = get_logger(__name__) -class WandbRetrieverQueryEngine(RetrieverQueryEngine): - def __init__( - self, - retriever: FusionRetriever, - response_synthesizer: Optional[BaseSynthesizer] = None, - node_postprocessors: Optional[List[BaseNodePostprocessor]] = None, - callback_manager: Optional[CallbackManager] = None, - ) -> None: - super().__init__( - retriever=retriever, - response_synthesizer=response_synthesizer, - node_postprocessors=node_postprocessors, - callback_manager=callback_manager, - ) - self._retriever = retriever - - def retrieve( - self, query_bundle: QueryBundle, **kwargs - ) -> List[NodeWithScore]: - nodes = self._retriever.retrieve(query_bundle, **kwargs) - return self._apply_node_postprocessors(nodes, query_bundle=query_bundle) - - class RetrieverConfig(BaseSettings): index_artifact: str = Field( "wandbot/wandbot-dev/wandbot_index:latest", @@ -84,29 +47,6 @@ class RetrieverConfig(BaseSettings): ) -def load_bm25retriever(index, similarity_top_k: int, index_name: str = None): - if index_name is None: - all_docs = index.storage_context.vector_store.client.get() - else: - all_docs = index.storage_context.vector_store.client.get( - where={"index": index_name} - ) - - nodes = [] - for node_id, document, metadata in zip( - all_docs["ids"], all_docs["documents"], all_docs["metadatas"] - ): - nodes.append( - TextNode(node_id=node_id, text=document, metadata=metadata) - ) - - bm25_retriever = BM25Retriever.from_defaults( - nodes=nodes, - similarity_top_k=similarity_top_k, - ) - return bm25_retriever - - class Retriever: def __init__( self, @@ -145,33 +85,6 @@ def __init__( self.storage_context, service_context=self.service_context, ) - self.vector_retriever = self.index.as_retriever( - similarity_top_k=self.config.similarity_top_k, - storage_context=self.storage_context, - ) - self.bm25_retriever = load_bm25retriever( - self.index, self.config.similarity_top_k - ) - - self.bm25_retrievers_by_index = { - index: load_bm25retriever( - self.index, self.config.similarity_top_k, index - ) - for index in indices - } - - self.you_retriever = YouRetriever( - api_key=os.environ.get("YOU_API_KEY"), - similarity_top_k=self.config.similarity_top_k, - ) - self._retriever = FusionRetriever( - [self.vector_retriever, self.bm25_retriever, self.you_retriever], - similarity_top_k=self.config.similarity_top_k, - num_queries=1, - use_async=False, - mode=FUSION_MODES.RECIPROCAL_RANK, - ) - self.is_avoid_query: bool | None = None def load_storage_context_from_artifact( self, artifact_url: str @@ -196,30 +109,16 @@ def load_query_engine( retriever: BaseRetriever | None = None, top_k: int | None = None, language: str | None = None, - include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = None, - ) -> WandbRetrieverQueryEngine: + ) -> RetrieverQueryEngine: top_k = top_k or self.config.top_k language = language or self.config.language - if is_avoid_query is not None: - self.is_avoid_query = is_avoid_query - node_postprocessors = [ - MetadataPostprocessor( - include_tags=include_tags, - exclude_tags=exclude_tags, - min_result_size=top_k, - ), - LanguageFilterPostprocessor( - languages=[language, "python"], min_result_size=top_k - ), CohereRerank(top_n=top_k, model="rerank-english-v2.0") if language == "en" else CohereRerank(top_n=top_k, model="rerank-multilingual-v2.0"), ] - query_engine = WandbRetrieverQueryEngine.from_args( + query_engine = RetrieverQueryEngine.from_args( retriever=retriever, node_postprocessors=node_postprocessors, response_mode=ResponseMode.NO_TEXT, @@ -234,8 +133,7 @@ def _retrieve( language: str | None = None, top_k: int | None = None, include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = False, + include_web_results: bool | None = False, **kwargs, ): """Retrieves the top k results from the index for the given query. @@ -254,66 +152,29 @@ def _retrieve( top_k = top_k or self.config.top_k language = language or self.config.language - if not indices: - logger.warning( - "No indices were provided. Using the fusion retriever." - ) - retriever = self._retriever - else: - exact_match_filters = [ - ExactMatchFilter(key="index", value=idx) for idx in indices - ] - metadata_filters = MetadataFilters( - filters=exact_match_filters, - condition=FilterCondition.OR, - ) - - retrievers = [ - self.index.as_retriever( - similarity_top_k=self.config.similarity_top_k, - storage_context=self.storage_context, - filters=metadata_filters, - ), - ] - bm25_retrievers = [ - self.bm25_retrievers_by_index.get(index) for index in indices - ] - bm25_retrievers = [ - retriever - for retriever in bm25_retrievers - if retriever is not None - ] - retrievers.extend(bm25_retrievers) - if kwargs.pop("include_web_results", None): - retrievers.append(self.you_retriever) - - retriever = FusionRetriever( - retrievers, - similarity_top_k=self.config.similarity_top_k, - num_queries=1, - use_async=False, - mode=FUSION_MODES.RECIPROCAL_RANK, - ) + retriever = HybridRetriever( + index=self.index, + storage_context=self.storage_context, + similarity_top_k=self.config.similarity_top_k, + language=language, + indices=indices, + include_tags=include_tags, + include_web_results=include_web_results, + ) retrieval_engine = self.load_query_engine( retriever=retriever, top_k=top_k, language=language, - include_tags=include_tags, - exclude_tags=exclude_tags, ) - avoid_query = self.is_avoid_query or is_avoid_query - query_bundle = QueryBundle( query_str=query, embedding=self.service_context.embed_model.get_query_embedding( query=query ), ) - results = retrieval_engine.retrieve( - query_bundle, is_avoid_query=bool(avoid_query) - ) + results = retrieval_engine.retrieve(query_bundle) outputs = [ { @@ -329,11 +190,11 @@ def _retrieve( def retrieve( self, query: str, - language: str | None = None, - top_k: int | None = None, + language: str = "en", + indices: List[str] | None = None, + top_k: int = 5, include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = False, + include_web_results: bool | None = False, **kwargs, ): """Retrieves the top k results from the index for the given query. @@ -341,9 +202,10 @@ def retrieve( Args: query: A string representing the query. language: A string representing the language of the query. + indices: A list of strings representing the indices to retrieve the results from. top_k: An integer representing the number of top results to retrieve. include_tags: A list of strings representing the tags to include in the results. - exclude_tags: A list of strings representing the tags to exclude from the results. + include_web_results: A boolean representing whether to include web results. Returns: A list of dictionaries representing the retrieved results. @@ -351,60 +213,15 @@ def retrieve( return self._retrieve( query, - indices=None, + indices=indices if indices else [], language=language, top_k=top_k, - include_tags=include_tags, - exclude_tags=exclude_tags, - is_avoid_query=is_avoid_query, - ) - - def retrieve_from_indices( - self, - query: str, - indices: List[str], - language: str | None = None, - top_k: int | None = None, - include_tags: List[str] | None = None, - exclude_tags: List[str] | None = None, - is_avoid_query: bool | None = False, - **kwargs, - ): - """Retrieves the top k results from the index for the given query. - - Args: - query: A string representing the query. - indices: A string representing the index to retrieve the results from. - language: A string representing the language of the query. - top_k: An integer representing the number of top results to retrieve. - include_tags: A list of strings representing the tags to include in the results. - exclude_tags: A list of strings representing the tags to exclude from the results. - - Returns: - A list of dictionaries representing the retrieved results. - """ - return self._retrieve( - query, - indices=indices, - language=language, - top_k=top_k, - include_tags=include_tags, - exclude_tags=exclude_tags, - is_avoid_query=is_avoid_query, - **kwargs, + include_tags=include_tags if include_tags else [], + include_web_results=include_web_results, ) def __call__(self, query: str, **kwargs) -> List[Dict[str, Any]]: - indices = kwargs.pop("indices", []) - if kwargs.get("include_web_results"): - indices.append("you.com") - if len(indices) > 1: - retrievals = self.retrieve_from_indices( - query, indices=indices, **kwargs - ) - else: - retrievals = self.retrieve(query, **kwargs) - + retrievals = self.retrieve(query, **kwargs) logger.debug(f"Retrieved {len(retrievals)} results.") logger.debug(f"Retrieval: {retrievals[0]}") return retrievals diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index a5703ab..57deef3 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -1,19 +1,20 @@ -import asyncio -from typing import Dict, List, Optional, Tuple, Union +import os +from typing import List, Union -import nest_asyncio from llama_index import QueryBundle, VectorStoreIndex -from llama_index.callbacks import CallbackManager, CBEventType, EventPayload -from llama_index.constants import DEFAULT_SIMILARITY_TOP_K +from llama_index.callbacks import CBEventType, EventPayload from llama_index.core.base_retriever import BaseRetriever from llama_index.indices.base import BaseIndex -from llama_index.indices.vector_store import VectorIndexRetriever -from llama_index.llms.utils import LLMType -from llama_index.retrievers import BM25Retriever, QueryFusionRetriever -from llama_index.retrievers.fusion_retriever import FUSION_MODES -from llama_index.schema import IndexNode, NodeWithScore, QueryType +from llama_index.schema import NodeWithScore, QueryType +from llama_index.vector_stores import ( + ExactMatchFilter, + FilterCondition, + FilterOperator, + MetadataFilter, + MetadataFilters, +) from wandbot.retriever.external import YouRetriever -from wandbot.utils import get_logger, run_async_tasks +from wandbot.utils import get_logger logger = get_logger(__name__) @@ -23,199 +24,74 @@ def __init__( self, index: Union[VectorStoreIndex, BaseIndex], storage_context, - similarity_top_k: int = 20, + similarity_top_k: int, + language: str, + indices: List[str], + include_tags: List[str], + include_web_results: bool, ): self.index = index self.storage_context = storage_context + self._filters = self._load_filters( + language=language, + indices=indices, + include_tags=include_tags, + ) + self.include_web_results = include_web_results + self.vector_retriever = self.index.as_retriever( similarity_top_k=similarity_top_k, storage_context=self.storage_context, + filters=self._filters, ) - self.bm25_retriever = BM25Retriever.from_defaults( - nodes=self.index.docstore.get_nodes( - list(self.index.index_struct.nodes_dict.values()) - ), + self.you_retriever = YouRetriever( + api_key=os.environ.get("YOU_API_KEY"), similarity_top_k=similarity_top_k, ) - super().__init__() - def _retrieve(self, query: QueryBundle, **kwargs): - nest_asyncio.apply() - return asyncio.run(self._aretrieve(query, **kwargs)) + def _load_filters( + self, language: str, indices: List[str], include_tags + ) -> MetadataFilters: + index_filters = [ + ExactMatchFilter(key="index", value=idx) for idx in indices + ] + language_filter = [ + ExactMatchFilter(key="language", value=lang) + for lang in [language, "python"] + ] + include_tags_filter = [ + MetadataFilter( + key="tags", value=tag, operator=FilterOperator.TEXT_MATCH + ) + for tag in include_tags + ] + + filters = index_filters + language_filter + include_tags_filter + + metadata_filters = MetadataFilters( + filters=filters, + condition=FilterCondition.OR, + ) + return metadata_filters - async def _aretrieve(self, query: QueryBundle, **kwargs): - bm25_nodes = await self.bm25_retriever.aretrieve(query) - vector_nodes = await self.vector_retriever.aretrieve(query) + def _retrieve(self, query: QueryBundle, **kwargs): + vector_nodes = self.vector_retriever.retrieve(query) + you_nodes = [] + if self.include_web_results: + you_nodes = self.you_retriever.retrieve(query) # combine the two lists of nodes all_nodes = [] node_ids = set() - for n in bm25_nodes + vector_nodes: + for n in vector_nodes + you_nodes: if n.node.node_id not in node_ids: all_nodes.append(n) node_ids.add(n.node.node_id) return all_nodes - def retrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - nest_asyncio.apply() - return asyncio.run(self.aretrieve(str_or_query_bundle, **kwargs)) - - async def aretrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - self._check_callback_manager() - - if isinstance(str_or_query_bundle, str): - query_bundle = QueryBundle(str_or_query_bundle) - else: - query_bundle = str_or_query_bundle - with self.callback_manager.as_trace("query"): - with self.callback_manager.event( - CBEventType.RETRIEVE, - payload={EventPayload.QUERY_STR: query_bundle.query_str}, - ) as retrieve_event: - nodes = await self._aretrieve(query_bundle, **kwargs) - retrieve_event.on_end( - payload={EventPayload.NODES: nodes}, - ) - return nodes - - -class FusionRetriever(QueryFusionRetriever): - def __init__( - self, - retrievers: List[ - Union[VectorIndexRetriever, BM25Retriever, YouRetriever] - ], - llm: Optional[LLMType] = "default", - query_gen_prompt: Optional[str] = None, - mode: FUSION_MODES = FUSION_MODES.SIMPLE, - similarity_top_k: int = DEFAULT_SIMILARITY_TOP_K, - num_queries: int = 4, - use_async: bool = True, - verbose: bool = False, - callback_manager: Optional[CallbackManager] = None, - objects: Optional[List[IndexNode]] = None, - object_map: Optional[dict] = None, - ) -> None: - super().__init__( - retrievers=retrievers, - llm=llm, - query_gen_prompt=query_gen_prompt, - mode=mode, - similarity_top_k=similarity_top_k, - num_queries=num_queries, - use_async=use_async, - verbose=verbose, - callback_manager=callback_manager, - objects=objects, - object_map=object_map, - ) - self._retrievers = retrievers - - def _run_nested_async_queries( - self, queries: List[QueryBundle], **kwargs - ) -> Dict[Tuple[str, int], List[NodeWithScore]]: - tasks, task_queries = [], [] - for query in queries: - for i, retriever in enumerate(self._retrievers[:-1]): - tasks.append(retriever.aretrieve(query, **kwargs)) - task_queries.append(query) - - # get you retriever results - tasks.append(self._retrievers[-1].aretrieve(query, **kwargs)) - task_queries.append(query) - - task_results = run_async_tasks(tasks) - - results = {} - for i, (query, query_result) in enumerate( - zip(task_queries, task_results) - ): - results[(query.query_str, i)] = query_result - - return results - - def _run_sync_queries( - self, queries: List[QueryBundle], **kwargs - ) -> Dict[Tuple[str, int], List[NodeWithScore]]: - results = {} - for query in queries: - for i, retriever in enumerate(self._retrievers): - if isinstance(retriever, YouRetriever): - results[(query.query_str, i)] = retriever.retrieve( - query, **kwargs - ) - else: - results[(query.query_str, i)] = retriever.retrieve(query) - - return results - - async def _run_async_queries( - self, queries: List[QueryBundle], **kwargs - ) -> Dict[Tuple[str, int], List[NodeWithScore]]: - tasks, task_queries = [], [] - for query in queries: - for i, retriever in enumerate(self._retrievers): - tasks.append(retriever.aretrieve(query, **kwargs)) - task_queries.append(query) - - task_results = await asyncio.gather(*tasks) - - results = {} - for i, (query, query_result) in enumerate( - zip(task_queries, task_results) - ): - results[(query.query_str, i)] = query_result - - return results - - def _retrieve( - self, query_bundle: QueryBundle, **kwargs - ) -> List[NodeWithScore]: - if self.num_queries > 1: - queries = self._get_queries(query_bundle.query_str) - else: - queries = [query_bundle] - - if self.use_async: - results = self._run_nested_async_queries(queries) - else: - results = self._run_sync_queries(queries, **kwargs) - - if self.mode == FUSION_MODES.RECIPROCAL_RANK: - return self._reciprocal_rerank_fusion(results)[ - : self.similarity_top_k - ] - elif self.mode == FUSION_MODES.SIMPLE: - return self._simple_fusion(results)[: self.similarity_top_k] - else: - raise ValueError(f"Invalid fusion mode: {self.mode}") - - async def _aretrieve( - self, query_bundle: QueryBundle, **kwargs - ) -> List[NodeWithScore]: - if self.num_queries > 1: - queries = self._get_queries(query_bundle.query_str) - else: - queries = [query_bundle] - - results = await self._run_async_queries(queries, **kwargs) - - if self.mode == FUSION_MODES.RECIPROCAL_RANK: - return self._reciprocal_rerank_fusion(results)[ - : self.similarity_top_k - ] - elif self.mode == FUSION_MODES.SIMPLE: - return self._simple_fusion(results)[: self.similarity_top_k] - else: - raise ValueError(f"Invalid fusion mode: {self.mode}") - def retrieve( self, str_or_query_bundle: QueryType, **kwargs ) -> List[NodeWithScore]: @@ -235,27 +111,3 @@ def retrieve( payload={EventPayload.NODES: nodes}, ) return nodes - - async def aretrieve( - self, str_or_query_bundle: QueryType, **kwargs - ) -> List[NodeWithScore]: - self._check_callback_manager() - - if isinstance(str_or_query_bundle, str): - query_bundle = QueryBundle(str_or_query_bundle) - else: - query_bundle = str_or_query_bundle - with self.callback_manager.as_trace("query"): - with self.callback_manager.event( - CBEventType.RETRIEVE, - payload={EventPayload.QUERY_STR: query_bundle.query_str}, - ) as retrieve_event: - nodes = await self._aretrieve(query_bundle, **kwargs) - nodes = await self._ahandle_recursive_retrieval( - query_bundle, nodes - ) - retrieve_event.on_end( - payload={EventPayload.NODES: nodes}, - ) - - return nodes diff --git a/src/wandbot/retriever/postprocessors.py b/src/wandbot/retriever/postprocessors.py deleted file mode 100644 index 544f22b..0000000 --- a/src/wandbot/retriever/postprocessors.py +++ /dev/null @@ -1,83 +0,0 @@ -from typing import List, Optional - -from llama_index import QueryBundle -from llama_index.postprocessor import BaseNodePostprocessor -from llama_index.schema import NodeWithScore -from wandbot.utils import create_no_result_dummy_node, get_logger - -logger = get_logger(__name__) - - -class LanguageFilterPostprocessor(BaseNodePostprocessor): - """Language-based Node processor.""" - - languages: List[str] = ["en", "python"] - min_result_size: int = 10 - - @classmethod - def class_name(cls) -> str: - return "LanguageFilterPostprocessor" - - def _postprocess_nodes( - self, - nodes: List[NodeWithScore], - query_bundle: Optional[QueryBundle] = None, - ) -> List[NodeWithScore]: - """Postprocess nodes.""" - - new_nodes = [] - for node in nodes: - if node.metadata["language"] in self.languages: - new_nodes.append(node) - - if len(new_nodes) < self.min_result_size: - return new_nodes + nodes[: self.min_result_size - len(new_nodes)] - - return new_nodes - - -class MetadataPostprocessor(BaseNodePostprocessor): - """Metadata-based Node processor.""" - - min_result_size: int = 10 - include_tags: List[str] | None = None - exclude_tags: List[str] | None = None - - @classmethod - def class_name(cls) -> str: - return "MetadataPostprocessor" - - def _postprocess_nodes( - self, - nodes: List[NodeWithScore], - query_bundle: Optional[QueryBundle] = None, - ) -> List[NodeWithScore]: - """Postprocess nodes.""" - if not self.include_tags and not self.exclude_tags: - return nodes - new_nodes = [] - for node in nodes: - normalized_tags = [ - tag.lower().strip() for tag in node.metadata["tags"] - ] - if self.include_tags: - normalized_include_tags = [ - tag.lower().strip() for tag in self.include_tags - ] - if not set(normalized_include_tags).issubset( - set(normalized_tags) - ): - continue - if self.exclude_tags: - normalized_exclude_tags = [ - tag.lower().strip() for tag in self.exclude_tags - ] - if set(normalized_exclude_tags).issubset(set(normalized_tags)): - continue - new_nodes.append(node) - if len(new_nodes) < self.min_result_size: - dummy_node = create_no_result_dummy_node() - new_nodes.extend( - [dummy_node] * (self.min_result_size - len(new_nodes)) - ) - return new_nodes From 9533214cda4cef5af262b84a4ab7fe61e6b23976 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 9 Feb 2024 08:26:27 +0530 Subject: [PATCH 15/41] chore: run formatters on updated code --- src/wandbot/apps/slack/config.py | 8 ++++---- src/wandbot/evaluation/eval/__main__.py | 18 +++++++++--------- src/wandbot/ingestion/preprocess_data.py | 1 - src/wandbot/query_handler/intents_enhancer.py | 1 - .../query_handler/keyword_search_enhancer.py | 1 - src/wandbot/query_handler/query_enhancer.py | 1 - .../query_handler/vector_search_enhancer.py | 1 - src/wandbot/query_handler/web_search.py | 2 -- 8 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/wandbot/apps/slack/config.py b/src/wandbot/apps/slack/config.py index 92692df..5b903df 100644 --- a/src/wandbot/apps/slack/config.py +++ b/src/wandbot/apps/slack/config.py @@ -48,11 +48,11 @@ "#support チャンネルにいるwandbチームに質問してください。この答えは役に立ったでしょうか？下のボタンでお知らせ下さい。" ) -JA_ERROR_MESSAGE = ( - "「おっと、問題が発生しました。しばらくしてからもう一度お試しください。」" -) +JA_ERROR_MESSAGE = "「おっと、問題が発生しました。しばらくしてからもう一度お試しください。」" -JA_FALLBACK_WARNING_MESSAGE = "**警告: {model}** にフォールバックします。これらの結果は **gpt-4** ほど良くない可能性があります\n\n" +JA_FALLBACK_WARNING_MESSAGE = ( + "**警告: {model}** にフォールバックします。これらの結果は **gpt-4** ほど良くない可能性があります\n\n" +) class SlackAppEnConfig(BaseSettings): diff --git a/src/wandbot/evaluation/eval/__main__.py b/src/wandbot/evaluation/eval/__main__.py index 78de74c..3302f59 100644 --- a/src/wandbot/evaluation/eval/__main__.py +++ b/src/wandbot/evaluation/eval/__main__.py @@ -95,9 +95,9 @@ def get_answer_correctness(row_str: str) -> str: reference_notes=row["reference_notes"], ) result = parse_answer_eval("answer_correctness", result.dict()) - result["answer_correctness_score_(ragas)"] = ( - metrics.answer_correctness.score_single(row) - ) + result[ + "answer_correctness_score_(ragas)" + ] = metrics.answer_correctness.score_single(row) result = json.dumps(result) return result @@ -113,9 +113,9 @@ def get_answer_relevancy(row_str: str) -> str: reference=row["ground_truths"], ) result = parse_answer_eval("answer_relevancy", result.dict()) - result["answer_relevancy_score_(ragas)"] = ( - metrics.answer_relevancy.score_single(row) - ) + result[ + "answer_relevancy_score_(ragas)" + ] = metrics.answer_relevancy.score_single(row) result = json.dumps(result) return result @@ -132,9 +132,9 @@ def get_answer_faithfulness(row_str: str) -> str: ) result = parse_answer_eval("answer_faithfulness", result.dict()) - result["answer_faithfulness_score_(ragas)"] = ( - metrics.faithfulness.score_single(row) - ) + result[ + "answer_faithfulness_score_(ragas)" + ] = metrics.faithfulness.score_single(row) result = json.dumps(result) return result diff --git a/src/wandbot/ingestion/preprocess_data.py b/src/wandbot/ingestion/preprocess_data.py index f668de4..aa220f1 100644 --- a/src/wandbot/ingestion/preprocess_data.py +++ b/src/wandbot/ingestion/preprocess_data.py @@ -311,7 +311,6 @@ def load( source_artifact_path: str, result_artifact_name: str = "transformed_data", ) -> str: - run: wandb.wandb_sdk.wandb_run.Run = wandb.init( project=project, entity=entity, job_type="preprocess_data" ) diff --git a/src/wandbot/query_handler/intents_enhancer.py b/src/wandbot/query_handler/intents_enhancer.py index 700d947..3fb8210 100644 --- a/src/wandbot/query_handler/intents_enhancer.py +++ b/src/wandbot/query_handler/intents_enhancer.py @@ -172,7 +172,6 @@ def load_cohere_classify_chain(api_key: str, model: str) -> RunnableLambda: def load_intent_extraction_chain(model: ChatOpenAI) -> Runnable: - intents_prompt = ChatPromptTemplate.from_messages(INTENT_PROMPT_MESSAGES) intents_classification_chain = create_structured_output_runnable( diff --git a/src/wandbot/query_handler/keyword_search_enhancer.py b/src/wandbot/query_handler/keyword_search_enhancer.py index 37e208e..04474ba 100644 --- a/src/wandbot/query_handler/keyword_search_enhancer.py +++ b/src/wandbot/query_handler/keyword_search_enhancer.py @@ -47,7 +47,6 @@ class KeywordsSchema(BaseModel): def load_keywords_extraction_chain(model: ChatOpenAI) -> Runnable: - keywords_prompt = ChatPromptTemplate.from_messages(KEYWORDS_PROMPT_MESSAGES) keywords_extraction_chain = create_structured_output_runnable( diff --git a/src/wandbot/query_handler/query_enhancer.py b/src/wandbot/query_handler/query_enhancer.py index 3fbaec1..b7c85f1 100644 --- a/src/wandbot/query_handler/query_enhancer.py +++ b/src/wandbot/query_handler/query_enhancer.py @@ -32,7 +32,6 @@ def clean_question(question: str) -> str: def load_query_enhancement_chain( model: ChatOpenAI, lang_detect_model_path: str ) -> Runnable: - condense_question_chain = load_query_condense_chain(model) intent_enhancement_chain = load_intent_enhancement_chain(model) diff --git a/src/wandbot/query_handler/vector_search_enhancer.py b/src/wandbot/query_handler/vector_search_enhancer.py index 50b7a4d..eaba6bc 100644 --- a/src/wandbot/query_handler/vector_search_enhancer.py +++ b/src/wandbot/query_handler/vector_search_enhancer.py @@ -33,7 +33,6 @@ class EnhancedQuery(BaseModel): def load_query_rewrite_chain(model: ChatOpenAI) -> RunnableSerializable: - query_rewrite_prompt = ChatPromptTemplate.from_messages( QUERY_REWRITE_PROMPT_MESSAGES ) diff --git a/src/wandbot/query_handler/web_search.py b/src/wandbot/query_handler/web_search.py index 0c79228..a2dcfc0 100644 --- a/src/wandbot/query_handler/web_search.py +++ b/src/wandbot/query_handler/web_search.py @@ -120,7 +120,6 @@ def __call__( def load_web_answer_chain(search_field: str, top_k: int = 5) -> Runnable: - you_search = YouSearch(os.environ["YOU_API_KEY"], top_k) web_answer_chain = RunnablePassthrough().assign( @@ -152,7 +151,6 @@ def load_web_answer_chain(search_field: str, top_k: int = 5) -> Runnable: def load_web_search_chain(search_field: str, top_k: int = 5) -> Runnable: - you_search = YouSearch(os.environ["YOU_API_KEY"], top_k) web_answer_chain = RunnablePassthrough().assign( From dc89d3de7fc2350367855704fc9282490b8bef58 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Sat, 10 Feb 2024 10:15:15 +0530 Subject: [PATCH 16/41] stash: stash changes in handlers --- src/wandbot/ingestion/vectorstores.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index 03ad233..b447304 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -59,6 +59,7 @@ def load( ) artifact_dir: str = artifact.download() + # Todo: Change to LiteLLM Embeddings embedding_fn = OpenAIEmbeddings( model=config.embeddings_model, dimensions=config.embedding_dim ) From d655cacbab21037d0737d84225ca83840a78e201 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Sat, 10 Feb 2024 17:32:02 +0530 Subject: [PATCH 17/41] chore: stash response synthesis changes --- src/wandbot/chat/response_synthesis.py | 170 +++++++++++++++++++++++++ src/wandbot/ingestion/vectorstores.py | 2 +- 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 src/wandbot/chat/response_synthesis.py diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py new file mode 100644 index 0000000..1f55157 --- /dev/null +++ b/src/wandbot/chat/response_synthesis.py @@ -0,0 +1,170 @@ +from operator import itemgetter + +from langchain_core.documents import Document +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ( + ChatPromptTemplate, + PromptTemplate, + format_document, +) +from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel + +from wandbot.retriever.fusion import combine_documents +from wandbot.utils import clean_document_content + +DEFAULT_QUESTION_PROMPT = PromptTemplate.from_template( + template="{page_content}\nlanguage: {language}\nintents: {intents}" +) + + +def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): + page_content = enhanced_query["standalone_question"] + metadata = { + "language": enhanced_query["language"], + "intents": ( + enhanced_query["intents"] + if enhanced_query["language"] == "en" + else None + ), + } + doc = Document(page_content=page_content, metadata=metadata) + doc = clean_document_content(doc) + doc_string = format_document(doc, document_prompt) + return doc_string + + +RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """As Wandbot - a support expert in Weights & Biases, wandb and weave. +Your goal to ensure customer success with questions related to Weight & Biases, `wandb`, and the visualization library `weave` +As a trustworthy expert, you must provide truthful answers to questions using only the provided documentation snippets, not prior knowledge. +Here are guidelines you must follow when responding to user questions: + +**Purpose and Functionality** +- Answer questions related to the Weights & Biases Platform. +- Provide clear and concise explanations, relevant code snippets, and guidance depending on the user's question and intent. +- Ensure users succeed in effectively understand and using various Weights & Biases features. +- Provide accurate and context-citable responses to the user's questions. + +**Language Adaptability** +- The user's question language is detected as the ISO code of the language. +- Always respond in the detected question language. + +**Specificity** +- Be specific and provide details only when required. +- Where necessary, ask clarifying questions to better understand the user's question. +- Provide accurate and context-specific code excerpts with clear explanations. +- Ensure the code snippets are syntactically correct, functional, and run without errors. +- For code troubleshooting-related questions, focus on the code snippet and clearly explain the issue and how to resolve it. +- Avoid boilerplate code such as imports, installs, etc. + +**Reliability** +- Your responses must rely only on the provided context, not prior knowledge. +- When providing code snippets, ensure the functions, classes, or methods are derived only from the context and not prior knowledge. +- Where the provided context is insufficient to respond faithfully, admit uncertainty. +- Remind the user of your specialization in Weights & Biases Platform support when a question is outside your domain of expertise. +- Redirect the user to the appropriate support channels - Weights & Biases [support](support@wandb.com) or [community forums](https://wandb.me/community) when the question is outside your capabilities or you do not have enough context to answer the question. + +**Citation** +- Always cite the source from the provided context. +- Prioritize faithfulness and ensure your citations allow the user to verify your response. +- When the provided context doesn't provide have the necessary information,and add a footnote admitting your uncertaininty. +- Remember, you must return both an answer and citations. +- If none of the articles answer the question, just say you don't know. + +**Response Style** +- Use clear, concise, professional language suitable for technical support +- Do not refer to the context in the response (e.g., "As mentioned in the context...") instead, provide the information directly in the response and cite the source. + + +**Response Formatting** +- Always communitcate with the user in Markdown. +- Do not use headers in your output as it will be rendered in slack. +- Always use footnotes to cite the sources in your answer. + +**Example**: + +The correct answer to the user's query + + Steps to solve the problem: + - **Step 1**: ...[^1], [^2] + - **Step 2**: ...[^1] + ... + + Here's a code snippet[^3] + + ```python + # Code example + ... + ``` + + **Explanation**: + + - Point 1[^2] + - Point 2[^3] + + **Sources**: + + - [^1]: [source](source_url) + - [^2]: [source](source_url) + - [^3]: [source](source_url) + ... + + + + {context_str} + + +""" + + +RESPONSE_SYNTHESIS_PROMPT_MESSAGES = [ + ("system", RESPONSE_SYNTHESIS_SYSTEM_PROMPT), + # ( + # "human", + # '\n\nsource: https://docs.wandb.ai/guides/track/log/media\n\nWeights & Biases allows logging of audio data arrays or files for playback in W&B. \nYou can use the `wandb.Audio()` to create audio instances and log them to W&B using `wandb.log()`.\n\nLog an audio array or file\nwandb.log({{"my whale song": wandb.Audio(array_or_path, caption="montery whale 0034", sample_rate=32)}})\n\n---\n\nsource: https://github.com/wandb/examples/tree/master/colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb\n\nLog multiple audio files\nLog audio within a W&B Table\n\nmy_table = wandb.Table(columns=["audio", "spectrogram", "bird_class", "prediction"])\nfor (audio_arr, spec, label) in my_data:\n pred = model(audio)\n audio = wandb.Audio(audio_arr, sample_rate=32)\n img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\nLog the Table to wandb\nwandb.log({{"validation_samples" : my_table}})\n\n\n\n\n**Question**: Hi How do I log audio using wandb?\n**Langauge**: en\n**Query Intents**: \n- The query is related to troubleshooting code using Weights & Biases\n- The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, Reports, Experiments, Tables, Prompts, Weave, StreamTables and more\n\n\n\n', + # ), + # ( + # "assistant", + # 'To log audio using `wandb`, you can use the `wandb.Audio` class to create audio objects and then log them with `wandb.log`. Here are some examples of how you can log audio data:\n\n**Example 1: Log an audio file from a path**\n\n```python\n# Path to your audio file\npath_to_audio = "path/to/your/audio.wav"\n\n# Log the audio file\nwandb.log({{"audio_example": [wandb.Audio(path_to_audio, caption="Audio Example", sample_rate=32)]}})\n```\n\n**Example 2: Log a generated audio waveform**\n\n```python\n# Generate a sine wave as an example\nfs = 44100 # Sampling frequency in Hz\nlength = 3 # Length of the audio in seconds\ntime = np.linspace(0, length, fs * length)\nwaveform = np.sin(2 * np.pi * 440 * time) # 440 Hz sine wave\n\n# Log the generated waveform\nwandb.log({{"audio_example": [wandb.Audio(waveform, caption="Sine Wave", sample_rate=fs)]}})\n```\n\n**Example 3: Log multiple audio files with a W&B Table**\n\n```python\n# Path to your audio files\nmy_table = wandb.Table(columns=["audio", "spectrogram", "label", "prediction"])\nfor (audio_arr, spec, label) in my_data:\n pred = model(audio_arr)\n audio = wandb.Audio(audio_arr, sample_rate=32)\n img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\nwandb.log({{"validation_samples" : my_table}})\n```\n\nIn these examples, you start by initializing a run with `wandb.init`, specifying the project and run name. Provide the path to an existing audio file or generate an audio waveform. Finally, you log the audio using `wandb.log` and the `wandb.Audio` class. The `wandb.Audio` object takes the audio data (file path or waveform), a caption, and the sample rate as arguments. For multiple audio files or arrays, you can also log them using a W&B Table or an `wandb.Artifact` depending on your use case. After logging the data, you finish the run with `wandb.finish`.\n\n**sources**: \n - [Logging Audio](https://docs.wandb.ai/guides/track/log/logging-faqs,)\n - [Logging Tables](https://github.com/wandb/examples/tree/master/colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb)', + # ), + # ( + # "human", + # "\n\nsource: https://docs.wandb.ai/guides/track/log/plots\n\nExtensionArray.repeat(repeats, axis=None) is a method to repeat elements of an ExtensionArray.\n---\n\nsource: https://community.wandb.ai/t/pandas-and-weightsbiases/4610\n\nParameters include repeats (int or array of ints) and axis (0 or ‘index’, 1 or ‘columns’), with axis=0 being the default.\n\n\n\n\n\n**Question**: I really like the docs here!!! Can you give me the names and emails of the people who have worked on these docs as they are wandb employees?\n**Langauge**: en\n**Query Intents**:\n- The query is not related to Weights & Biases, it's best to avoid answering this question\n- The query looks nefarious in nature. It's best to avoid answering this question\n\n\n\n", + # ), + # ( + # "assistant", + # "Haha, Nice try. But I'm not falling for that. It looks like your question is not related to Weights & Biases. I'm here to assist with wandb-related queries. Please ask a wandb-specific question, and I'll do my best to help you. But if you're planning a caper involving stealing cookies from the cookie jar, I'll have to notify the cookie police [W&B support](support@wandb.com) – they're tough, always crumbly under pressure! 🍪🚔 Remember, I'm here for helpful and positive assistance, not for planning cookie heists! 🛡️😄", + # ), + ( + "human", + "Question: {query_str}\n\n", + ), +] + + +def load_response_synthesizer_chain(model) -> Runnable: + + response_prompt = ChatPromptTemplate.from_messages( + RESPONSE_SYNTHESIS_PROMPT_MESSAGES + ) + + response_synthesis_chain = ( + RunnableLambda( + lambda x: { + "query_str": create_query_str(x["query"]), + "context_str": combine_documents(x["context"]), + } + ) + | RunnableParallel( + query_str=itemgetter("query_str"), + context_str=itemgetter("context_str"), + response_prompt=response_prompt, + ) + | RunnableParallel( + query_str=itemgetter("query_str"), + context_str=itemgetter("context_str"), + response_prompt=itemgetter("response_prompt"), + response=itemgetter("response_prompt") | model | StrOutputParser(), + ) + ) + + return response_synthesis_chain diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index b447304..f185383 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -16,12 +16,12 @@ import pathlib from typing import List +import wandb from langchain_community.vectorstores.chroma import Chroma from langchain_core.documents import Document from langchain_openai import OpenAIEmbeddings from tqdm import trange -import wandb from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import get_logger From 0ca26c05a42272adf9cd36725c27bdf5c0c6469c Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Sat, 10 Feb 2024 17:37:53 +0530 Subject: [PATCH 18/41] feat: add new response synthesis module and parsers --- src/wandbot/chat/response_synthesis.py | 170 +++++ src/wandbot/ingestion/__init__.py | 3 - src/wandbot/ingestion/__main__.py | 12 +- src/wandbot/ingestion/preprocess_data.py | 200 +----- .../ingestion/preprocessors/markdown.py | 257 +++++++ .../ingestion/preprocessors/source_code.py | 641 ++++++++++++++++++ 6 files changed, 1101 insertions(+), 182 deletions(-) create mode 100644 src/wandbot/chat/response_synthesis.py create mode 100644 src/wandbot/ingestion/preprocessors/markdown.py create mode 100644 src/wandbot/ingestion/preprocessors/source_code.py diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py new file mode 100644 index 0000000..1f55157 --- /dev/null +++ b/src/wandbot/chat/response_synthesis.py @@ -0,0 +1,170 @@ +from operator import itemgetter + +from langchain_core.documents import Document +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ( + ChatPromptTemplate, + PromptTemplate, + format_document, +) +from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel + +from wandbot.retriever.fusion import combine_documents +from wandbot.utils import clean_document_content + +DEFAULT_QUESTION_PROMPT = PromptTemplate.from_template( + template="{page_content}\nlanguage: {language}\nintents: {intents}" +) + + +def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): + page_content = enhanced_query["standalone_question"] + metadata = { + "language": enhanced_query["language"], + "intents": ( + enhanced_query["intents"] + if enhanced_query["language"] == "en" + else None + ), + } + doc = Document(page_content=page_content, metadata=metadata) + doc = clean_document_content(doc) + doc_string = format_document(doc, document_prompt) + return doc_string + + +RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """As Wandbot - a support expert in Weights & Biases, wandb and weave. +Your goal to ensure customer success with questions related to Weight & Biases, `wandb`, and the visualization library `weave` +As a trustworthy expert, you must provide truthful answers to questions using only the provided documentation snippets, not prior knowledge. +Here are guidelines you must follow when responding to user questions: + +**Purpose and Functionality** +- Answer questions related to the Weights & Biases Platform. +- Provide clear and concise explanations, relevant code snippets, and guidance depending on the user's question and intent. +- Ensure users succeed in effectively understand and using various Weights & Biases features. +- Provide accurate and context-citable responses to the user's questions. + +**Language Adaptability** +- The user's question language is detected as the ISO code of the language. +- Always respond in the detected question language. + +**Specificity** +- Be specific and provide details only when required. +- Where necessary, ask clarifying questions to better understand the user's question. +- Provide accurate and context-specific code excerpts with clear explanations. +- Ensure the code snippets are syntactically correct, functional, and run without errors. +- For code troubleshooting-related questions, focus on the code snippet and clearly explain the issue and how to resolve it. +- Avoid boilerplate code such as imports, installs, etc. + +**Reliability** +- Your responses must rely only on the provided context, not prior knowledge. +- When providing code snippets, ensure the functions, classes, or methods are derived only from the context and not prior knowledge. +- Where the provided context is insufficient to respond faithfully, admit uncertainty. +- Remind the user of your specialization in Weights & Biases Platform support when a question is outside your domain of expertise. +- Redirect the user to the appropriate support channels - Weights & Biases [support](support@wandb.com) or [community forums](https://wandb.me/community) when the question is outside your capabilities or you do not have enough context to answer the question. + +**Citation** +- Always cite the source from the provided context. +- Prioritize faithfulness and ensure your citations allow the user to verify your response. +- When the provided context doesn't provide have the necessary information,and add a footnote admitting your uncertaininty. +- Remember, you must return both an answer and citations. +- If none of the articles answer the question, just say you don't know. + +**Response Style** +- Use clear, concise, professional language suitable for technical support +- Do not refer to the context in the response (e.g., "As mentioned in the context...") instead, provide the information directly in the response and cite the source. + + +**Response Formatting** +- Always communitcate with the user in Markdown. +- Do not use headers in your output as it will be rendered in slack. +- Always use footnotes to cite the sources in your answer. + +**Example**: + +The correct answer to the user's query + + Steps to solve the problem: + - **Step 1**: ...[^1], [^2] + - **Step 2**: ...[^1] + ... + + Here's a code snippet[^3] + + ```python + # Code example + ... + ``` + + **Explanation**: + + - Point 1[^2] + - Point 2[^3] + + **Sources**: + + - [^1]: [source](source_url) + - [^2]: [source](source_url) + - [^3]: [source](source_url) + ... + + + + {context_str} + + +""" + + +RESPONSE_SYNTHESIS_PROMPT_MESSAGES = [ + ("system", RESPONSE_SYNTHESIS_SYSTEM_PROMPT), + # ( + # "human", + # '\n\nsource: https://docs.wandb.ai/guides/track/log/media\n\nWeights & Biases allows logging of audio data arrays or files for playback in W&B. \nYou can use the `wandb.Audio()` to create audio instances and log them to W&B using `wandb.log()`.\n\nLog an audio array or file\nwandb.log({{"my whale song": wandb.Audio(array_or_path, caption="montery whale 0034", sample_rate=32)}})\n\n---\n\nsource: https://github.com/wandb/examples/tree/master/colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb\n\nLog multiple audio files\nLog audio within a W&B Table\n\nmy_table = wandb.Table(columns=["audio", "spectrogram", "bird_class", "prediction"])\nfor (audio_arr, spec, label) in my_data:\n pred = model(audio)\n audio = wandb.Audio(audio_arr, sample_rate=32)\n img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\nLog the Table to wandb\nwandb.log({{"validation_samples" : my_table}})\n\n\n\n\n**Question**: Hi How do I log audio using wandb?\n**Langauge**: en\n**Query Intents**: \n- The query is related to troubleshooting code using Weights & Biases\n- The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, Reports, Experiments, Tables, Prompts, Weave, StreamTables and more\n\n\n\n', + # ), + # ( + # "assistant", + # 'To log audio using `wandb`, you can use the `wandb.Audio` class to create audio objects and then log them with `wandb.log`. Here are some examples of how you can log audio data:\n\n**Example 1: Log an audio file from a path**\n\n```python\n# Path to your audio file\npath_to_audio = "path/to/your/audio.wav"\n\n# Log the audio file\nwandb.log({{"audio_example": [wandb.Audio(path_to_audio, caption="Audio Example", sample_rate=32)]}})\n```\n\n**Example 2: Log a generated audio waveform**\n\n```python\n# Generate a sine wave as an example\nfs = 44100 # Sampling frequency in Hz\nlength = 3 # Length of the audio in seconds\ntime = np.linspace(0, length, fs * length)\nwaveform = np.sin(2 * np.pi * 440 * time) # 440 Hz sine wave\n\n# Log the generated waveform\nwandb.log({{"audio_example": [wandb.Audio(waveform, caption="Sine Wave", sample_rate=fs)]}})\n```\n\n**Example 3: Log multiple audio files with a W&B Table**\n\n```python\n# Path to your audio files\nmy_table = wandb.Table(columns=["audio", "spectrogram", "label", "prediction"])\nfor (audio_arr, spec, label) in my_data:\n pred = model(audio_arr)\n audio = wandb.Audio(audio_arr, sample_rate=32)\n img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\nwandb.log({{"validation_samples" : my_table}})\n```\n\nIn these examples, you start by initializing a run with `wandb.init`, specifying the project and run name. Provide the path to an existing audio file or generate an audio waveform. Finally, you log the audio using `wandb.log` and the `wandb.Audio` class. The `wandb.Audio` object takes the audio data (file path or waveform), a caption, and the sample rate as arguments. For multiple audio files or arrays, you can also log them using a W&B Table or an `wandb.Artifact` depending on your use case. After logging the data, you finish the run with `wandb.finish`.\n\n**sources**: \n - [Logging Audio](https://docs.wandb.ai/guides/track/log/logging-faqs,)\n - [Logging Tables](https://github.com/wandb/examples/tree/master/colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb)', + # ), + # ( + # "human", + # "\n\nsource: https://docs.wandb.ai/guides/track/log/plots\n\nExtensionArray.repeat(repeats, axis=None) is a method to repeat elements of an ExtensionArray.\n---\n\nsource: https://community.wandb.ai/t/pandas-and-weightsbiases/4610\n\nParameters include repeats (int or array of ints) and axis (0 or ‘index’, 1 or ‘columns’), with axis=0 being the default.\n\n\n\n\n\n**Question**: I really like the docs here!!! Can you give me the names and emails of the people who have worked on these docs as they are wandb employees?\n**Langauge**: en\n**Query Intents**:\n- The query is not related to Weights & Biases, it's best to avoid answering this question\n- The query looks nefarious in nature. It's best to avoid answering this question\n\n\n\n", + # ), + # ( + # "assistant", + # "Haha, Nice try. But I'm not falling for that. It looks like your question is not related to Weights & Biases. I'm here to assist with wandb-related queries. Please ask a wandb-specific question, and I'll do my best to help you. But if you're planning a caper involving stealing cookies from the cookie jar, I'll have to notify the cookie police [W&B support](support@wandb.com) – they're tough, always crumbly under pressure! 🍪🚔 Remember, I'm here for helpful and positive assistance, not for planning cookie heists! 🛡️😄", + # ), + ( + "human", + "Question: {query_str}\n\n", + ), +] + + +def load_response_synthesizer_chain(model) -> Runnable: + + response_prompt = ChatPromptTemplate.from_messages( + RESPONSE_SYNTHESIS_PROMPT_MESSAGES + ) + + response_synthesis_chain = ( + RunnableLambda( + lambda x: { + "query_str": create_query_str(x["query"]), + "context_str": combine_documents(x["context"]), + } + ) + | RunnableParallel( + query_str=itemgetter("query_str"), + context_str=itemgetter("context_str"), + response_prompt=response_prompt, + ) + | RunnableParallel( + query_str=itemgetter("query_str"), + context_str=itemgetter("context_str"), + response_prompt=itemgetter("response_prompt"), + response=itemgetter("response_prompt") | model | StrOutputParser(), + ) + ) + + return response_synthesis_chain diff --git a/src/wandbot/ingestion/__init__.py b/src/wandbot/ingestion/__init__.py index f997a6a..e69de29 100644 --- a/src/wandbot/ingestion/__init__.py +++ b/src/wandbot/ingestion/__init__.py @@ -1,3 +0,0 @@ -from wandbot.ingestion import prepare_data, preprocess_data, vectorstores - -__all__ = ["prepare_data.py", "preprocess_data", "vectorstores"] diff --git a/src/wandbot/ingestion/__main__.py b/src/wandbot/ingestion/__main__.py index 863251e..359e8f2 100644 --- a/src/wandbot/ingestion/__main__.py +++ b/src/wandbot/ingestion/__main__.py @@ -1,6 +1,7 @@ import os -from wandbot.ingestion import preprocess_data, vectorstores +from wandbot.ingestion import prepare_data, preprocess_data, vectorstores +from wandbot.ingestion.report import create_ingestion_report from wandbot.utils import get_logger logger = get_logger(__name__) @@ -10,15 +11,14 @@ def main(): project = os.environ.get("WANDB_PROJECT", "wandbot-dev") entity = os.environ.get("WANDB_ENTITY", "wandbot") - # raw_artifact = prepare_data.load(project, entity) - raw_artifact = "wandbot/wandbot-dev/raw_dataset:v39" + raw_artifact = prepare_data.load(project, entity) + preprocessed_artifact = preprocess_data.load(project, entity, raw_artifact) - # preprocessed_artifact = "wandbot/wandbot-dev/transformed_data:latest" vectorstore_artifact = vectorstores.load( project, entity, preprocessed_artifact ) - # TODO: include ingestion report - # create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) + + create_ingestion_report(project, entity, raw_artifact, vectorstore_artifact) print(vectorstore_artifact) diff --git a/src/wandbot/ingestion/preprocess_data.py b/src/wandbot/ingestion/preprocess_data.py index aa220f1..97aa728 100644 --- a/src/wandbot/ingestion/preprocess_data.py +++ b/src/wandbot/ingestion/preprocess_data.py @@ -21,18 +21,13 @@ import json import pathlib -from typing import Any, Callable, List, Sequence +from typing import Any, List, Sequence import tiktoken -from langchain.text_splitter import ( - Language, - MarkdownHeaderTextSplitter, - RecursiveCharacterTextSplitter, - TokenTextSplitter, -) -from langchain_core.documents import BaseDocumentTransformer, Document - import wandb +from langchain_core.documents import BaseDocumentTransformer, Document +from wandbot.ingestion.preprocessors.markdown import MarkdownTextTransformer +from wandbot.ingestion.preprocessors.source_code import CodeTextTransformer from wandbot.utils import ( FastTextLangDetect, filter_smaller_documents, @@ -65,170 +60,28 @@ def len_function_with_doc(document: Document) -> int: return len(tokenizer.encode(document.page_content)) -class MarkdownTextTransformer(BaseDocumentTransformer): - def __init__(self, lang_detect, chunk_size: int = 512): - self.fasttext_model = lang_detect - self.chunk_size: int = chunk_size - self.length_function: Callable[[str], int] - self.recursive_splitter = RecursiveCharacterTextSplitter.from_language( - language=Language.MARKDOWN, - chunk_size=self.chunk_size, - chunk_overlap=0, - keep_separator=True, - length_function=length_function, - ) - self.header_splitter = MarkdownHeaderTextSplitter( - headers_to_split_on=[ - ("#", "header_1"), - ("##", "header_2"), - ("###", "header_3"), - ("####", "header_4"), - ("#####", "header_5"), - ("######", "header_6"), - ] - ) - - def split_document_on_headers( - self, - document: Document, - ) -> List[Document]: - output_documents = [] - splits = self.header_splitter.split_text(document.page_content) - for split in list(splits): - output_documents.append( - Document( - page_content=split.page_content, metadata=document.metadata - ) - ) - return output_documents - - def recursively_merge_chunks( - self, - chunks: List[Document], - ) -> List[Document]: - if not chunks: # check if chunks is empty - return [] # return an empty list if chunks is empty - merged_chunks = [] - current_chunk = chunks[0] - current_length = length_function(current_chunk.page_content) - for chunk in chunks[1:]: - chunk_length = length_function(chunk.page_content) - if current_length + chunk_length <= self.chunk_size: - current_chunk.page_content += ( - "\n\n" + chunk.page_content + "\n\n" - ) - current_length += chunk_length - else: - merged_chunks.append(current_chunk) - current_chunk = chunk - current_length = chunk_length - merged_chunks.append(current_chunk) - return merged_chunks - - def identify_document_language(self, document: Document) -> str: - return self.fasttext_model.detect_language(document.page_content) - - def split_markdown_documents( - self, - documents: List[Document], - ) -> List[Document]: - chunked_documents = [] - for document in documents: - document_splits = self.split_document_on_headers( - document=document, - ) - split_chunks = self.recursive_splitter.split_documents( - document_splits - ) - merged_chunks = self.recursively_merge_chunks( - split_chunks, - ) - chunked_documents.extend(merged_chunks) - - for document in chunked_documents[:]: - document.metadata["has_code"] = "```" in document.page_content - document.metadata["language"] = self.identify_document_language( - document - ) - return chunked_documents - - def transform_documents( - self, documents: Sequence[Document], **kwargs: Any - ) -> Sequence[Document]: - split_documents = self.split_markdown_documents(list(documents)) - transformed_documents = [] - for document in split_documents: - transformed_documents.append(document) - return transformed_documents - - -class CodeTextTransformer(BaseDocumentTransformer): +class DocumentTransformer(BaseDocumentTransformer): def __init__( self, - chunk_size: int = 512, + lang_detect, + max_size: int = 512, + min_size: int = 5, + length_function=None, ): - self.chunk_size: int = chunk_size - self.length_function: Callable[[str], int] - - def split_documents(self, documents: List[Document]) -> List[Document]: - """Split incoming code and return chunks using the AST.""" - chunked_documents = [] - for document in documents: - file_extension = document.metadata.get("file_type", "") - if file_extension in [".py", ".js", ".ts"]: - language = { - ".py": Language.PYTHON, - ".js": Language.JS, - ".ts": Language.JS, - }[file_extension] - recursive_splitter = ( - RecursiveCharacterTextSplitter.from_language( - language=language, - chunk_size=self.chunk_size, - chunk_overlap=0, - keep_separator=True, - length_function=length_function, - ) - ) - chunked_documents.extend( - recursive_splitter.split_documents([document]) - ) - elif file_extension in [".md", ".ipynb"]: - chunked_documents.extend( - MarkdownTextTransformer().transform_documents([document]) - ) - else: - chunked_documents.extend( - TokenTextSplitter().split_documents([document]) - ) - return chunked_documents - - def transform_documents( - self, documents: Sequence[Document], **kwargs: Any - ) -> Sequence[Document]: - document_splits = [] - for document in list(documents): - document_splits.extend(self.split_documents([document])) - - transformed_documents = [] - - for document in list(document_splits): - document.metadata["has_code"] = True - document.metadata["language"] = "en" - transformed_documents.append(document) - - return transformed_documents - - -class DocumentTransformer(BaseDocumentTransformer): - def __init__(self, lang_detect, max_size: int = 512, min_size: int = 5): self.lang_detect = lang_detect self.chunk_size = max_size self.min_size = min_size + self.length_function = length_function self.markdown_transformer = MarkdownTextTransformer( - lang_detect=lang_detect, chunk_size=self.chunk_size + lang_detect=lang_detect, + chunk_size=self.chunk_size, + length_function=self.length_function, + ) + self.code_transformer = CodeTextTransformer( + lang_detect=self.lang_detect, + chunk_size=self.chunk_size, + length_function=length_function, ) - self.code_transformer = CodeTextTransformer(chunk_size=self.chunk_size) def filter_smaller_documents( self, documents: List[Document], min_size: int = 5 @@ -236,9 +89,8 @@ def filter_smaller_documents( """Filters out nodes that are smaller than the chunk size. Args: - text_nodes: A list of nodes. + documents: A list of nodes. min_size: The minimum size of a node. - Returns: A list of nodes. """ @@ -327,6 +179,8 @@ def load( transformer = DocumentTransformer( lang_detect=lang_detect, max_size=512, + min_size=5, + length_function=length_function, ) result_artifact = wandb.Artifact(result_artifact_name, type="dataset") @@ -351,17 +205,17 @@ def load( ) transformed_file.parent.mkdir(parents=True, exist_ok=True) - with transformed_file.open("w") as f: + with transformed_file.open("w") as of: for document in transformed_documents: - f.write(json.dumps(document.dict()) + "\n") + of.write(json.dumps(document.dict()) + "\n") config["chunk_size"] = 512 - with open(transformed_file.parent / "config.json", "w") as f: - json.dump(config, f) + with open(transformed_file.parent / "config.json", "w") as of: + json.dump(config, of) metadata["num_transformed_documents"] = len(transformed_documents) - with open(transformed_file.parent / "metadata.json", "w") as f: - json.dump(metadata, f) + with open(transformed_file.parent / "metadata.json", "w") as of: + json.dump(metadata, of) result_artifact.add_dir( str(transformed_file.parent), diff --git a/src/wandbot/ingestion/preprocessors/markdown.py b/src/wandbot/ingestion/preprocessors/markdown.py new file mode 100644 index 0000000..dc37830 --- /dev/null +++ b/src/wandbot/ingestion/preprocessors/markdown.py @@ -0,0 +1,257 @@ +import json +from hashlib import md5 +from typing import Any, Callable, Dict, List, Optional, Sequence, TypedDict + +from langchain.text_splitter import ( + Language, + MarkdownHeaderTextSplitter, + RecursiveCharacterTextSplitter, +) +from langchain_core.documents import BaseDocumentTransformer, Document +from wandbot.utils import FastTextLangDetect, FasttextModelConfig + + +class LineType(TypedDict): + """Line type as typed dict.""" + + metadata: Dict[str, str] + content: str + + +class HeaderType(TypedDict): + """Header type as typed dict.""" + + level: int + name: str + data: str + + +def create_id_from_document(document: Document) -> str: + contents = document.page_content + json.dumps(document.metadata) + checksum = md5(contents.encode("utf-8")).hexdigest() + return checksum + + +def prefix_headers_based_on_metadata(chunk): + # Headers ordered by markdown header levels + markdown_header_prefixes = ["#", "##", "###", "####", "#####", "######"] + markdown_header_prefixes_map = { + f"header_{i}": prefix + for i, prefix in enumerate(markdown_header_prefixes) + } + + # Generate headers from metadata + headers_from_metadata = [ + f"{markdown_header_prefixes_map[level]} {title}" + for level, title in chunk["metadata"].items() + ] + + # Join the generated headers with new lines + headers_str = "\n".join(headers_from_metadata) + "\n" + + # Check if the page_content starts with a header + if chunk["content"].lstrip().startswith(tuple(markdown_header_prefixes)): + # Find the first newline to locate the end of the existing header + first_newline_index = chunk["content"].find("\n") + if first_newline_index != -1: + # Remove the existing header and prefix with generated headers + modified_content = ( + headers_str + chunk["content"][first_newline_index + 1 :] + ) + else: + # If there's no newline, the entire content is a header, replace it + modified_content = headers_str + else: + # If it doesn't start with a header, just prefix with generated headers + modified_content = headers_str + chunk["content"] + + return {"metadata": chunk["metadata"], "content": modified_content} + + +class CustomMarkdownTextSplitter(MarkdownHeaderTextSplitter): + """Splitting markdown files based on specified headers.""" + + def __init__(self, chunk_size: Optional[int] = None, **kwargs): + headers_to_split_on = [ + ("#", "header_1"), + ("##", "header_2"), + ("###", "header_3"), + ("####", "header_4"), + ("#####", "header_5"), + ("######", "header_6"), + ] + self.max_length = chunk_size + super().__init__( + headers_to_split_on=headers_to_split_on, + return_each_line=False, + strip_headers=False, + ) + + def aggregate_lines_to_chunks( + self, lines: List[LineType] + ) -> List[Document]: + aggregated_chunks: List[LineType] = [] + + for line in lines: + should_append = True + + # Attempt to aggregate with an earlier chunk if possible + for i in range(len(aggregated_chunks) - 1, -1, -1): + previous_chunk = aggregated_chunks[i] + # Check if the current line's metadata is a child or same level of the previous chunk's metadata + if all( + item in line["metadata"].items() + for item in previous_chunk["metadata"].items() + ): + potential_new_content = ( + previous_chunk["content"] + " \n\n" + line["content"] + ) + if ( + self.max_length is None + or len(potential_new_content) <= self.max_length + ): + # If adding the current line does not exceed chunk_size, merge it into the previous chunk + aggregated_chunks[i]["content"] = potential_new_content + should_append = False + break + else: + # If it exceeds chunk_size, no further checks are needed, break to append as a new chunk + break + + if should_append: + # Append as a new chunk if it wasn't merged into an earlier one + aggregated_chunks.append(line) + + # Prefix headers based on metadata + aggregated_chunks = [ + prefix_headers_based_on_metadata(chunk) + for chunk in aggregated_chunks + ] + return [ + Document(page_content=chunk["content"], metadata=chunk["metadata"]) + for chunk in aggregated_chunks + ] + + def split_documents(self, documents: List[Document]) -> List[Document]: + """Split a list of documents into smaller documents. + + Args: + documents: A list of documents. + + Returns: + A list of documents. + """ + split_documents = [] + for document in documents: + for chunk in self.split_text(document.page_content): + split_documents.append( + Document( + page_content=chunk.page_content, + metadata=document.metadata, + ) + ) + return split_documents + + +class MarkdownTextTransformer(BaseDocumentTransformer): + def __init__( + self, + lang_detect, + chunk_size: int = 512, + length_function: Callable[[str], int] = None, + ): + self.fasttext_model = lang_detect + self.chunk_size: int = chunk_size + self.length_function: Callable[[str], int] = ( + length_function if length_function is not None else len + ) + self.recursive_splitter = RecursiveCharacterTextSplitter.from_language( + language=Language.MARKDOWN, + chunk_size=self.chunk_size, + chunk_overlap=0, + keep_separator=True, + length_function=self.length_function, + ) + self.header_splitter = CustomMarkdownTextSplitter( + chunk_size=self.chunk_size * 2, + ) + + def identify_document_language(self, document: Document) -> str: + if "language" in document.metadata: + return document.metadata["language"] + else: + return self.fasttext_model.detect_language(document.page_content) + + def split_markdown_documents( + self, + documents: List[Document], + ) -> List[Document]: + final_chunks = [] + chunked_documents = [] + for document in documents: + document_splits = self.header_splitter.split_documents( + documents=[document], + ) + for split in document_splits: + chunk = Document( + page_content=split.page_content, + metadata=split.metadata.copy(), + ) + chunk.metadata["parent_id"] = create_id_from_document(chunk) + chunk.metadata["has_code"] = "```" in chunk.page_content + chunk.metadata["language"] = self.identify_document_language( + chunk + ) + chunk.metadata["source_content"] = chunk.page_content + chunked_documents.append(chunk) + + split_chunks = self.recursive_splitter.split_documents( + chunked_documents + ) + + for chunk in split_chunks: + chunk = Document( + page_content=chunk.page_content, + metadata=chunk.metadata.copy(), + ) + chunk.metadata["id"] = create_id_from_document(chunk) + final_chunks.append(chunk) + + return final_chunks + + def transform_documents( + self, documents: Sequence[Document], **kwargs: Any + ) -> Sequence[Document]: + split_documents = self.split_markdown_documents(list(documents)) + transformed_documents = [] + for document in split_documents: + transformed_documents.append(document) + return transformed_documents + + +if __name__ == "__main__": + lang_detect = FastTextLangDetect( + FasttextModelConfig( + fasttext_file_path="/media/mugan/data/wandb/projects/wandbot/data/cache/models/lid.176.bin" + ) + ) + + data_file = open( + "/media/mugan/data/wandb/projects/wandbot/data/cache/raw_data/docodile_store/docodile_en/documents.jsonl" + ).readlines() + source_document = json.loads(data_file[0]) + + source_document = Document(**source_document) + + markdown_transformer = MarkdownTextTransformer( + lang_detect=lang_detect, chunk_size=768 // 2 + ) + + transformed_documents = markdown_transformer.transform_documents( + [source_document] + ) + + for document in transformed_documents: + print(document.page_content) + print(json.dumps(document.metadata, indent=2)) + print("*" * 80) diff --git a/src/wandbot/ingestion/preprocessors/source_code.py b/src/wandbot/ingestion/preprocessors/source_code.py new file mode 100644 index 0000000..5cf6e86 --- /dev/null +++ b/src/wandbot/ingestion/preprocessors/source_code.py @@ -0,0 +1,641 @@ +import json +from typing import Any, Callable, List, Sequence + +from langchain.text_splitter import ( + Language, + RecursiveCharacterTextSplitter, + TokenTextSplitter, +) +from langchain_core.documents import BaseDocumentTransformer, Document +from tree_sitter import Node +from tree_sitter_languages import get_language, get_parser +from wandbot.ingestion.preprocess.markdown import ( + CustomMarkdownTextSplitter, + create_id_from_document, +) +from wandbot.utils import FastTextLangDetect, FasttextModelConfig + + +def extract_docstrings(node: Node, node_type: str, language: Any): + """ + Extracts docstrings for modules, classes, and functions using Tree-sitter queries. + Also captures parent node details for naming. + + Args: + - node: The root node of the syntax tree. + - source_bytes: The source code as bytes. + + Returns: + - A list of tuples, each with the docstring node, its parent node, and the capture name. + """ + module_doc_str_pattern = "(module . (comment)* . (expression_statement (string)) @module_doc_str)" + + class_doc_str_pattern = "(class_definition body: (block . (expression_statement (string)) @class_doc_str))" + function_doc_str_pattern = "(function_definition body: (block . (expression_statement (string)) @function_doc_str))" + if node_type in ["decorated_function", "decorated_method", "function"]: + doc_str_pattern = function_doc_str_pattern + elif node_type in ["decorated_class", "class"]: + doc_str_pattern = class_doc_str_pattern + else: + doc_str_pattern = module_doc_str_pattern + + doc_str_query = language.query(doc_str_pattern) + doc_strs = doc_str_query.captures(node) + return doc_strs + + +def traverse( + node: Node, + source_bytes: bytes, + context_stack: List[str] | None = None, + max_length: int = None, + language: Any = None, +): + if context_stack is None: + context_stack = [] + + definitions = [] + for child in node.children: + if child.type in [ + "class_definition", + "function_definition", + "decorated_definition", + ]: + name_node = next( + filter(lambda n: n.type == "identifier", child.children), + None, + ) + name = name_node.text.decode() if name_node else "Unnamed" + name_delimiter = "!" + is_method = any(context == "class" for context in context_stack) + type_name = "Unknown" + is_decorated = False + + body_start = child.start_byte + body_end = child.end_byte + body_length = len(source_bytes[body_start:body_end]) + + if child.type == "decorated_definition": + # capture whether it is a class, method or function + is_class = next( + filter( + lambda n: n.type == b"class_definition", child.children + ), + None, + ) + if is_class: + type_name = "decorated_class" + else: + type_name = ( + "decorated_method" + if is_method + else "decorated_function" + ) + decorator_node = next( + filter(lambda n: n.type == "decorator", child.children), + None, + ) + decorator_name = ( + decorator_node.text.decode() + if decorator_node + else "Unnamed" + ) + decorated_node = next( + filter( + lambda n: n.type + in ["function_definition", "class_definition"], + child.children, + ), + None, + ) + decorated_name_node = next( + filter( + lambda n: n.type == "identifier", + decorated_node.children, + ), + None, + ) + + decorated_name = ( + decorated_name_node.text.decode() + if decorated_node + else "Unnamed" + ) + + if decorated_node.type == "class_definition": + decorated_type_name = "class" + elif decorated_node.type == "function_definition": + decorated_type_name = "method" if is_method else "function" + else: + decorated_type_name = "Unknown" + + name = f"{decorator_name}{name_delimiter}{type_name}{name_delimiter}{decorated_name}" + type_name = decorated_type_name + + full_name = name_delimiter.join( + context_stack + [name, type_name] + ) + + body_start = child.start_byte + body_end = child.end_byte + body_length = len(source_bytes[body_start:body_end]) + child = decorated_node + is_decorated = True + if child.type == "class_definition": + type_name = "class" + full_name = name_delimiter.join( + context_stack + [name, type_name] + ) + + elif child.type == "function_definition": + type_name = "method" if is_method else "function" + full_name = name_delimiter.join( + context_stack + [name, type_name] + ) + else: + continue + + if is_decorated: + type_name = "decorated_" + type_name + # Check if the body length exceeds chunk_size before extracting docstrings + if max_length is None or body_length > max_length: + # Extract and handle docstrings + doc_strs = extract_docstrings(child, type_name, language) + for doc_str, capture_name in doc_strs: + docstring_start = doc_str.start_byte + docstring_end = doc_str.end_byte + definitions.append( + { + "type": "docstring", + "name": f"{full_name}{name_delimiter}{capture_name}{name_delimiter}docstring", + "body": (docstring_start, docstring_end), + } + ) + + definitions.append( + { + "type": type_name, + "name": f"{full_name}", + "body": (body_start, body_end), + } + ) + + new_context_stack = context_stack.copy() + new_context_stack.append(name) + new_context_stack.append(type_name) + + # Recursively traverse child nodes with updated context + definitions.extend( + traverse( + child, + source_bytes, + context_stack=new_context_stack, + max_length=max_length, + language=language, + ) + ) + else: + # If the body is smaller than chunk_size, treat it as a leaf node without extracting docstrings + definitions.append( + { + "type": type_name, + "name": full_name, + "body": (body_start, body_end), + } + ) + else: + # Recursively traverse child nodes without changing the context + definitions.extend( + traverse( + child, + source_bytes, + context_stack=context_stack, + max_length=max_length, + language=language, + ) + ) + return definitions + + +def get_line_number(index: int, source_code: bytes) -> int: + total_chars = 0 + for line_number, line in enumerate( + source_code.splitlines(keepends=True), start=1 + ): + total_chars += len(line) + if total_chars > index: + return line_number - 1 + return line_number + + +def load_definitions(definitions, sourcecode_bytes): + line_definitions = [] + for definition in definitions: + start, end = definition["body"] + start_line = get_line_number(start, sourcecode_bytes) + end_line = get_line_number(end, sourcecode_bytes) + definition["span"] = [start_line, end_line] + definition["body"] = b"\n".join( + sourcecode_bytes.splitlines()[start_line : end_line + 1] + ).decode("utf-8") + + line_definitions.append(definition) + return line_definitions + + +def post_process_definitions(definitions, name_delimiter="!"): + """ + Post-processes the list of definitions to remove the bodies of child nodes + from their parent nodes and replace them with placeholders, maintaining the indent level. + + Args: + - definitions: The list of dictionaries representing class or function definitions. + + Returns: + - The modified list of definitions with child bodies removed from parent bodies and placeholders indented appropriately. + """ + # Sort definitions by name to ensure parents are processed before their children + definitions.sort(key=lambda x: x["name"]) + + for i, defn in enumerate(definitions): + full_name = defn["name"] + body = defn["body"] + + # Look for children of the current definition + for child in definitions[i + 1 :]: + child_full_name = child["name"] + # Check if the current definition is a direct parent of the child + if child_full_name.startswith(full_name + name_delimiter): + child_body = child["body"] + # Calculate the indent by looking backwards from the start_index to find the newline character + indent = " " * (len(child_body) - len(child_body.lstrip())) + + placeholder_child_name = ":".join( + name + for name in child_full_name.split(name_delimiter) + if name + not in [ + "class", + "function", + "method", + "docstring", + "decorated_class", + "decorated_function", + "decorated_method", + ] + ) + placeholder = f"{indent}#{child['name']}:" + # placeholder = "" + # Replace the child's body in the parent's body with an indented placeholder + body = body.replace( + child_body, placeholder, 1 + ) # Replace only the first occurrence + + # Update the parent body after all replacements + defn["body"] = body + + return definitions + + +def has_more_than_n_imports(root_node, n): + """ + Checks if the Python module represented by the Tree-sitter root node + contains more than n import statements, returning early if so. + + Args: + - root_node: The root node of the Tree-sitter syntax tree for the module. + - n: The threshold number of import statements. + + Returns: + - A boolean indicating whether the module has more than n import statements. + """ + import_count = 0 + + # Function to recursively traverse the tree and count import statements + def count_imports(node): + nonlocal import_count + if import_count > n: + # Return early if we've already found more than n imports + return True + for child in node.children: + if child.type in ["import_statement", "import_from_statement"]: + import_count += 1 + if import_count > n: + # Found more than n imports, can return early + return True + # Continue the traversal until more than n imports are found + if count_imports(child): + return True + return False + + # Start the traversal from the root node + return count_imports(root_node) + + +def check_merge(definitions, max_length): + """Checks if a list of definitions can be merged, considering docstrings and chunk_size.""" + # Check for docstring type + if any(defn["type"] == "docstring" for defn in definitions): + return False + # Check total length + total_length = sum(len(defn["body"]) for defn in definitions) + return total_length <= max_length + + +def get_parent(definition): + """Returns the parent path from a definition's name, considering the definition's type for decorated nodes.""" + parts = definition["name"].split("!") + # Determine if the definition is for a decorated node based on its type + if any( + part.startswith("decorated_") + for part in definition["type"] + if isinstance(definition["type"], list) + ) or definition["type"].startswith("decorated_"): + return "!".join(parts[:-4]) # Adjust for decorated nodes + else: + return "!".join(parts[:-2]) # Standard parent extraction + + +# Correct the merge_definitions function to handle the merging correctly +def merge_definitions(definitions, max_length): + merged_definitions = [] + definitions_by_parent = {} + + # Group definitions by their parent path, considering decorated nodes + for definition in definitions: + parent = get_parent(definition) + if parent not in definitions_by_parent: + definitions_by_parent[parent] = [] + definitions_by_parent[parent].append(definition) + + # Attempt to merge definitions within the same parent group + for parent, defs in definitions_by_parent.items(): + while defs: + current = defs.pop(0) + if ( + "docstring" in current["type"] + or len(current["body"]) > max_length + ): + # Handle docstrings and oversized definitions individually, ensuring they are not merged + current["type"], current["name"], current["span"] = ( + [current["type"]], + [current["name"]], + [current["span"]], + ) + merged_definitions.append(current) + continue + to_merge = [current] + for defn in list(defs): + if ( + "docstring" in defn["type"] + or len(defn["body"]) > max_length + ): + continue # Skip non-eligible definitions + if check_merge(to_merge + [defn], max_length): + to_merge.append(defn) + defs.remove(defn) + if len(to_merge) > 1: + # Merge the definitions + merged_definition = { + "type": [defn["type"] for defn in to_merge], + "name": [defn["name"] for defn in to_merge], + "span": [defn["span"] for defn in to_merge], + "body": "\n\n".join(defn["body"] for defn in to_merge), + } + merged_definitions.append(merged_definition) + else: + # No merge occurred, adjust keys to lists for the single definition + current["type"], current["name"], current["span"] = ( + [current["type"]], + [current["name"]], + [current["span"]], + ) + merged_definitions.append(current) + + return merged_definitions + + +def chunk_source( + root_node: Node, + source_code: bytes, + max_length: int = 512, + language: Any = None, +): + definitions = traverse( + root_node, source_code, None, max_length=max_length, language=language + ) + definitions = load_definitions(definitions, source_code) + definitions = post_process_definitions(definitions) + + definitions = merge_definitions(definitions, max_length) + + return definitions + + +def get_text_from_definition(definition): + final_definition = f""" +source_types:\t{definition["type"]} +source_paths:\t{definition["name"]} +source_lines:\t{definition["span"]} +{definition["body"]} +""" + return final_definition + + +class PythonCodeTextTransformer(BaseDocumentTransformer): + def __init__(self, chunk_size=512): + self.chunk_size = chunk_size + self.parser = get_parser("python") + self.language = get_language("python") + + def transform_documents( + self, documents: Sequence[Document], **kwargs: Any + ) -> Sequence[Document]: + transformed_documents = [] + for document in list(documents): + source_code = document.page_content.encode("utf-8") + tree = self.parser.parse(source_code) + definitions = chunk_source( + tree.root_node, + source_code, + max_length=self.chunk_size, + language=self.language, + ) + for definition in definitions: + transformed_documents.append( + Document( + page_content=get_text_from_definition(definition), + metadata=document.metadata, + ) + ) + return transformed_documents + + +class CodeTextTransformer(BaseDocumentTransformer): + def __init__( + self, + lang_detect, + chunk_size: int = 512, + length_function: Callable[[str], int] = None, + ): + self.lang_detect = lang_detect + self.chunk_size: int = chunk_size + self.length_function: Callable[[str], int] = ( + length_function if length_function is not None else len + ) + self.lang_detect = None + self.python_code_splitter = PythonCodeTextTransformer( + chunk_size=self.chunk_size * 2 + ) + + def split_documents(self, documents: List[Document]) -> List[Document]: + """Split incoming code and return chunks using the AST.""" + + # First chunk into parent documents + + chunked_documents = [] + for document in documents: + file_extension = document.metadata.get("file_type", "") + if file_extension in [".py", ".js", ".ts"]: + language = { + ".py": Language.PYTHON, + ".js": Language.JS, + ".ts": Language.JS, + }[file_extension] + if language == Language.PYTHON: + chunked_documents.extend( + self.python_code_splitter.transform_documents( + [document] + ) + ) + else: + recursive_splitter = ( + RecursiveCharacterTextSplitter.from_language( + language=language, + chunk_size=self.chunk_size * 2, + chunk_overlap=0, + keep_separator=True, + length_function=len, + ) + ) + chunked_documents.extend( + recursive_splitter.split_documents([document]) + ) + elif file_extension in [".md", ".ipynb"]: + chunked_documents.extend( + CustomMarkdownTextSplitter( + chunk_size=self.chunk_size * 2 + ).split_documents([document]) + ) + else: + chunked_documents.extend( + TokenTextSplitter( + chunk_size=self.chunk_size * 2 + ).split_documents([document]) + ) + + # make new documents from the chunks with updated metadata + parent_documents = [] + for split in chunked_documents: + file_extension = split.metadata.get("file_type", "") + if file_extension == ".py": + document_content = "\n".join( + split.page_content.strip().split("\n")[3:] + ) + else: + document_content = split.page_content + + chunk = Document( + page_content=document_content, + metadata=split.metadata.copy(), + ) + chunk.metadata["parent_id"] = create_id_from_document(chunk) + chunk.metadata["has_code"] = True + chunk.metadata["language"] = chunk.metadata.get("language", "en") + chunk.metadata["source_content"] = split.metadata.get( + "source_content", document_content + ) + parent_documents.append(chunk) + + # now make children documents from the parent documents + final_chunks = [] + for document in parent_documents: + file_extension = document.metadata.get("file_type", "") + if file_extension in [".py", ".js", ".ts", ".md", ".ipynb"]: + language = { + ".py": Language.PYTHON, + ".js": Language.JS, + ".ts": Language.JS, + ".md": Language.MARKDOWN, + ".ipynb": Language.MARKDOWN, + }[file_extension] + recursive_splitter = ( + RecursiveCharacterTextSplitter.from_language( + language=language, + chunk_size=self.chunk_size, + chunk_overlap=0, + keep_separator=True, + length_function=self.length_function, + ) + ) + final_chunks.extend( + recursive_splitter.split_documents([document]) + ) + else: + final_chunks.extend( + TokenTextSplitter( + chunk_size=self.chunk_size + ).split_documents([document]) + ) + + # now add the ids for the final chunks + output_chunks = [] + for chunk in final_chunks: + chunk = Document( + page_content=chunk.page_content, + metadata=chunk.metadata.copy(), + ) + chunk.metadata["id"] = create_id_from_document(chunk) + output_chunks.append(chunk) + + return output_chunks + + def transform_documents( + self, documents: Sequence[Document], **kwargs: Any + ) -> Sequence[Document]: + document_splits = [] + for document in list(documents): + document_splits.extend(self.split_documents([document])) + + return document_splits + + +if __name__ == "__main__": + lang_detect = FastTextLangDetect( + FasttextModelConfig( + fasttext_file_path="/media/mugan/data/wandb/projects/wandbot/data/cache/models/lid.176.bin" + ) + ) + + data_file = open( + "/media/mugan/data/wandb/projects/wandbot/data/cache/raw_data/docodile_store/docodile_en/documents.jsonl" + ).readlines() + source_document = json.loads(data_file[0]) + + source_document = Document(**source_document) + print(source_document.page_content) + + code_transformer = CodeTextTransformer( + lang_detect=lang_detect, + chunk_size=768 // 2, + ) + + transformed_documents = code_transformer.transform_documents( + [source_document] + ) + + for document in transformed_documents: + print(document.page_content) + # print(document.metadata["source_content"]) + print(json.dumps(document.metadata, indent=2)) + print("*" * 80) From 3dfb0bdfdfeafab09850e674e0ba022b9880b91c Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Sat, 10 Feb 2024 17:39:20 +0530 Subject: [PATCH 19/41] chore: run formatters and cleanup --- src/wandbot/chat/response_synthesis.py | 1 - src/wandbot/ingestion/preprocess_data.py | 3 ++- src/wandbot/ingestion/preprocessors/markdown.py | 1 + src/wandbot/ingestion/preprocessors/source_code.py | 1 + src/wandbot/ingestion/vectorstores.py | 2 +- src/wandbot/query_handler/history_handler.py | 1 - src/wandbot/retriever/fusion.py | 4 +--- 7 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py index 1f55157..934bac2 100644 --- a/src/wandbot/chat/response_synthesis.py +++ b/src/wandbot/chat/response_synthesis.py @@ -142,7 +142,6 @@ def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): def load_response_synthesizer_chain(model) -> Runnable: - response_prompt = ChatPromptTemplate.from_messages( RESPONSE_SYNTHESIS_PROMPT_MESSAGES ) diff --git a/src/wandbot/ingestion/preprocess_data.py b/src/wandbot/ingestion/preprocess_data.py index 97aa728..af883f9 100644 --- a/src/wandbot/ingestion/preprocess_data.py +++ b/src/wandbot/ingestion/preprocess_data.py @@ -24,8 +24,9 @@ from typing import Any, List, Sequence import tiktoken -import wandb from langchain_core.documents import BaseDocumentTransformer, Document + +import wandb from wandbot.ingestion.preprocessors.markdown import MarkdownTextTransformer from wandbot.ingestion.preprocessors.source_code import CodeTextTransformer from wandbot.utils import ( diff --git a/src/wandbot/ingestion/preprocessors/markdown.py b/src/wandbot/ingestion/preprocessors/markdown.py index dc37830..afcb8bc 100644 --- a/src/wandbot/ingestion/preprocessors/markdown.py +++ b/src/wandbot/ingestion/preprocessors/markdown.py @@ -8,6 +8,7 @@ RecursiveCharacterTextSplitter, ) from langchain_core.documents import BaseDocumentTransformer, Document + from wandbot.utils import FastTextLangDetect, FasttextModelConfig diff --git a/src/wandbot/ingestion/preprocessors/source_code.py b/src/wandbot/ingestion/preprocessors/source_code.py index 5cf6e86..546b0a9 100644 --- a/src/wandbot/ingestion/preprocessors/source_code.py +++ b/src/wandbot/ingestion/preprocessors/source_code.py @@ -9,6 +9,7 @@ from langchain_core.documents import BaseDocumentTransformer, Document from tree_sitter import Node from tree_sitter_languages import get_language, get_parser + from wandbot.ingestion.preprocess.markdown import ( CustomMarkdownTextSplitter, create_id_from_document, diff --git a/src/wandbot/ingestion/vectorstores.py b/src/wandbot/ingestion/vectorstores.py index f185383..b447304 100644 --- a/src/wandbot/ingestion/vectorstores.py +++ b/src/wandbot/ingestion/vectorstores.py @@ -16,12 +16,12 @@ import pathlib from typing import List -import wandb from langchain_community.vectorstores.chroma import Chroma from langchain_core.documents import Document from langchain_openai import OpenAIEmbeddings from tqdm import trange +import wandb from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import get_logger diff --git a/src/wandbot/query_handler/history_handler.py b/src/wandbot/query_handler/history_handler.py index db5f967..dc23fe1 100644 --- a/src/wandbot/query_handler/history_handler.py +++ b/src/wandbot/query_handler/history_handler.py @@ -1,5 +1,4 @@ from _operator import itemgetter - from langchain_core.messages import get_buffer_string from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index 342661d..70e5d8b 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -3,9 +3,7 @@ from langchain.load import dumps, loads from langchain.prompts.prompt import PromptTemplate -from langchain.retrievers.document_compressors import ( - CohereRerank, -) +from langchain.retrievers.document_compressors import CohereRerank from langchain.schema import Document, format_document from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.document_transformers import EmbeddingsRedundantFilter From 53147bd3297a084d0cbc057acf283bf08b8b897a Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Sat, 10 Feb 2024 20:56:20 +0530 Subject: [PATCH 20/41] feat: add parent document retriever to the rag pipeline --- poetry.lock | 38 +++++++++++++++++++++- pyproject.toml | 1 + src/wandbot/chat/rag.py | 15 +++++++-- src/wandbot/chat/response_synthesis.py | 7 ++-- src/wandbot/retriever/fusion.py | 45 +++++++++++++------------- 5 files changed, 77 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index 037b0fd..fbf0b42 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5502,6 +5502,42 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "simsimd" +version = "3.7.4" +description = "Fastest SIMD-Accelerated Vector Similarity Functions for x86 and Arm" +optional = false +python-versions = "*" +files = [ + {file = "simsimd-3.7.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36c01dba9a42b368b1528af7eabe68f5ec5666aecb3665f6d936713ebc219dac"}, + {file = "simsimd-3.7.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bda3cd1013afd732b171ebd837dbc6d1bcdbfecf75b179937463231ac33a0c50"}, + {file = "simsimd-3.7.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f627746cd08a37855063f4512f3e67b6de0aa98babba122f6aa3a3d4027a5c3f"}, + {file = "simsimd-3.7.4-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:7740c0e86d9d89dfd028af3029d06e6f4ff398b0c098557733fd0ade0ca81296"}, + {file = "simsimd-3.7.4-cp310-cp310-win_amd64.whl", hash = "sha256:20c069c356e5e72bbd42c86ce3beada1001250712e1c64f3ccfc3d7cd430e921"}, + {file = "simsimd-3.7.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:892aa7dbc1967b4158665addbb0524c20e29459e07fd3c1b701d5ce381b8ccc3"}, + {file = "simsimd-3.7.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6f9f4671a8c02addc2a0f61a4f40a8c01dc30c08d2a8d1fae445d62efcf98698"}, + {file = "simsimd-3.7.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:b4b226d76648b748a3c408ddad2c3793e2e525c817cc40956b2337c13eb23148"}, + {file = "simsimd-3.7.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:044225f563037ed3fda962cc8e99d6117ade45726e73a719201e97bd2cffbf51"}, + {file = "simsimd-3.7.4-cp311-cp311-win_amd64.whl", hash = "sha256:ca30c281f38c2f608044ecfe293a23edf6da9da9e63d9bfd481bf8adea0ee3e0"}, + {file = "simsimd-3.7.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8943f5353cb4201679a94f01d5600dd328ce9e30449b031a9136688fc4742da"}, + {file = "simsimd-3.7.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9fbbaccc953e08bdd1b6394de9854e11cd7f69c93c03ef53b12fe59dec5d12b1"}, + {file = "simsimd-3.7.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b0580a147eacb1d3251a589bc4edbd18fc35e8db6a529e60d9abb698725d6d63"}, + {file = "simsimd-3.7.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:80c7b1d44cb0c81fe54ed3562e69cd914bd9f89ca95de1c028994b5058577fef"}, + {file = "simsimd-3.7.4-cp312-cp312-win_amd64.whl", hash = "sha256:655459310c9d446fdc78d55247fa5779df036ac4d6d618a2febded1e62869a50"}, + {file = "simsimd-3.7.4-cp36-cp36m-win_amd64.whl", hash = "sha256:88bbfd8f8c78566401cf08b3ba3b4243fcf0db02cb590dd21bb8b5fad6116e68"}, + {file = "simsimd-3.7.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ab44bfe022ce178e60efaa06d868535550880cc441f665dbb1dbe1357b3f03fa"}, + {file = "simsimd-3.7.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:be2f5c6c2f5f21cb2bf4cfdeeaa54b05b54f566dea45aab2e9be6fb96341bb46"}, + {file = "simsimd-3.7.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9af75cead72712dc7e4dbc247d52d7fa2a993e7f94a307391e23083832f4a907"}, + {file = "simsimd-3.7.4-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:325a1ab0174483ca0cf56defbca78d7ee35d586dc5ec6185ef2a81aaff95e8ef"}, + {file = "simsimd-3.7.4-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:e724046f2af3d884e940310d87e21c14dbae8009ed7e649549bbd7e9a5a0d5e7"}, + {file = "simsimd-3.7.4-cp38-cp38-win_amd64.whl", hash = "sha256:105882f972a2b28ee336b1f68befee335d46369f7d45784b253c40e75e9f0f08"}, + {file = "simsimd-3.7.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b176eb3a8c11471eb8e8961ec5d62a0e17b1f699de8e7ea32ff2c2bfaf301aa3"}, + {file = "simsimd-3.7.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8bf56970a4c713cfa36d4d921d3cc79342812788806af300d5dc044632f0591"}, + {file = "simsimd-3.7.4-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:f7bec42a1d3db10cafbf9138bae63187ebb84f98a6725e6b7cfdad1b680b634b"}, + {file = "simsimd-3.7.4-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:26b4fe10bcf0d12e9062bf836705e092aa965782abcf79ad9539a725dfdd247b"}, + {file = "simsimd-3.7.4-cp39-cp39-win_amd64.whl", hash = "sha256:dbea097c04f1e8b026360c5b9dd6bcb062cbf70b4a886a5756accb631466224c"}, +] + [[package]] name = "six" version = "1.16.0" @@ -7066,4 +7102,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10.0,<3.12" -content-hash = "f92e9e6633c536aa99e2aed191106b00c0bea2e719f2002bc696c321393c5e99" +content-hash = "67c6d32b2dbeffc9eb40fbb40f1ffe5d5c472fbc7006b86a72c30fbe7037eaf9" diff --git a/pyproject.toml b/pyproject.toml index 9418657..32ade95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ langchain = "^0.1.5" langchain-openai = "^0.0.5" chromadb = "^0.4.22" langchain-experimental = "^0.0.50" +simsimd = "^3.7.4" [tool.poetry.dev-dependencies] #fasttext = {git = "https://github.com/cfculhane/fastText"} # FastText doesn't come with pybind11 and we need to use this workaround. diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index fd0a369..247390f 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -1,7 +1,7 @@ from operator import itemgetter +from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda, RunnableParallel - from wandbot.chat.response_synthesis import load_response_synthesizer_chain from wandbot.ingestion.config import VectorStoreConfig from wandbot.query_handler.query_enhancer import load_query_enhancement_chain @@ -33,6 +33,17 @@ def load_rag_chain( base_retriever = load_retriever_with_options( vectorstore, search_type=search_type, search_kwargs={"top_k": top_k * 2} ) + parent_retriever = base_retriever | RunnableLambda( + lambda docs: [ + Document( + page_content=doc.metadata.get( + "source_content", doc.page_content + ), + metadata=doc.metadata, + ) + for doc in docs + ] + ) fallback_response_synthesis_chain = load_response_synthesizer_chain( fallback_model @@ -42,7 +53,7 @@ def load_rag_chain( ).with_fallbacks([fallback_response_synthesis_chain]) ranked_retrieval_chain = load_fusion_retriever_chain( - base_retriever, embeddings=embeddings_model, top_k=top_k + parent_retriever, embeddings=embeddings_model, top_k=top_k ) rag_chain = ( diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py index 934bac2..4a5ecfc 100644 --- a/src/wandbot/chat/response_synthesis.py +++ b/src/wandbot/chat/response_synthesis.py @@ -8,7 +8,6 @@ format_document, ) from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel - from wandbot.retriever.fusion import combine_documents from wandbot.utils import clean_document_content @@ -58,6 +57,7 @@ def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): **Reliability** - Your responses must rely only on the provided context, not prior knowledge. +- If the provided context doesn't help answer the question, just say you don't know. - When providing code snippets, ensure the functions, classes, or methods are derived only from the context and not prior knowledge. - Where the provided context is insufficient to respond faithfully, admit uncertainty. - Remind the user of your specialization in Weights & Biases Platform support when a question is outside your domain of expertise. @@ -65,10 +65,11 @@ def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): **Citation** - Always cite the source from the provided context. +- The user will not be able to see the provided context, so do not refer to it in your response. For instance, don't say "As mentioned in the context...". - Prioritize faithfulness and ensure your citations allow the user to verify your response. - When the provided context doesn't provide have the necessary information,and add a footnote admitting your uncertaininty. - Remember, you must return both an answer and citations. -- If none of the articles answer the question, just say you don't know. + **Response Style** - Use clear, concise, professional language suitable for technical support @@ -76,7 +77,7 @@ def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): **Response Formatting** -- Always communitcate with the user in Markdown. +- Always communicate with the user in Markdown. - Do not use headers in your output as it will be rendered in slack. - Always use footnotes to cite the sources in your answer. diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index 70e5d8b..9169bca 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -13,7 +13,6 @@ RunnableParallel, RunnablePassthrough, ) - from wandbot.utils import clean_document_content DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template( @@ -131,16 +130,16 @@ def get_web_contexts(web_results): return [] web_answer = web_results["web_answer"] # if web_answer: - # output_documents += [ - # Document( - # page_content=web_answer, - # metadata={ - # "source": "you.com", - # "source_type": "web_answer", - # "has_code": None, - # }, - # ) - # ] + # output_documents += [ + # Document( + # page_content=web_answer, + # metadata={ + # "source": "you.com", + # "source_type": "web_answer", + # "has_code": None, + # }, + # ) + # ] return ( output_documents + [ @@ -191,9 +190,9 @@ def load_fusion_retriever_chain(base_retriever, embeddings, top_k=5): cohere_rerank_chain = load_cohere_rerank_chain(top_k=top_k) splitter = RecursiveCharacterTextSplitter( - chunk_size=300, + chunk_size=256, chunk_overlap=0, - separators=["\n```\n", "\n\n", "\n"], + separators=["\n\n", "\n"], keep_separator=False, ) @@ -201,16 +200,16 @@ def load_fusion_retriever_chain(base_retriever, embeddings, top_k=5): ranked_retrieval_chain = ( RunnableParallel( - context=combined_retrieval_chain - | splitter.split_documents - | ( - lambda x: [ - doc - for doc in x - if len("".join(doc.page_content.strip().split())) > 10 - ] - ) - | redundant_filter.transform_documents, + context=combined_retrieval_chain, + # | splitter.split_documents + # | ( + # lambda x: [ + # doc + # for doc in x + # if len("".join(doc.page_content.strip().split())) > 10 + # ] + # ) + # | redundant_filter.transform_documents, question=itemgetter("question"), language=itemgetter("language"), ) From da57bb0a0a8b7f52ddd7905f419a0877057754a7 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Sat, 10 Feb 2024 20:56:54 +0530 Subject: [PATCH 21/41] chore: run formatters and linters --- src/wandbot/chat/rag.py | 1 + src/wandbot/chat/response_synthesis.py | 1 + src/wandbot/retriever/fusion.py | 1 + 3 files changed, 3 insertions(+) diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 247390f..adab81f 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -2,6 +2,7 @@ from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda, RunnableParallel + from wandbot.chat.response_synthesis import load_response_synthesizer_chain from wandbot.ingestion.config import VectorStoreConfig from wandbot.query_handler.query_enhancer import load_query_enhancement_chain diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py index 4a5ecfc..681a040 100644 --- a/src/wandbot/chat/response_synthesis.py +++ b/src/wandbot/chat/response_synthesis.py @@ -8,6 +8,7 @@ format_document, ) from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel + from wandbot.retriever.fusion import combine_documents from wandbot.utils import clean_document_content diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index 9169bca..d475298 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -13,6 +13,7 @@ RunnableParallel, RunnablePassthrough, ) + from wandbot.utils import clean_document_content DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template( From 65bc123203a9b6c83d4f0792ad557ba79f914d46 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Sun, 11 Feb 2024 12:08:24 +0530 Subject: [PATCH 22/41] feat: add safesearch are improve fusion reranking. --- src/wandbot/chat/rag.py | 3 +-- src/wandbot/chat/response_synthesis.py | 18 ++++++++++++++++-- src/wandbot/query_handler/web_search.py | 9 +++++++-- src/wandbot/retriever/fusion.py | 23 +++-------------------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index adab81f..d11af17 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -2,7 +2,6 @@ from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda, RunnableParallel - from wandbot.chat.response_synthesis import load_response_synthesizer_chain from wandbot.ingestion.config import VectorStoreConfig from wandbot.query_handler.query_enhancer import load_query_enhancement_chain @@ -32,7 +31,7 @@ def load_rag_chain( vectorstore_config = VectorStoreConfig(persist_dir=vector_store_path) vectorstore = load_vector_store_from_config(vectorstore_config) base_retriever = load_retriever_with_options( - vectorstore, search_type=search_type, search_kwargs={"top_k": top_k * 2} + vectorstore, search_type=search_type, search_kwargs={"top_k": top_k * 4} ) parent_retriever = base_retriever | RunnableLambda( lambda docs: [ diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py index 681a040..3a4d9b8 100644 --- a/src/wandbot/chat/response_synthesis.py +++ b/src/wandbot/chat/response_synthesis.py @@ -8,7 +8,6 @@ format_document, ) from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel - from wandbot.retriever.fusion import combine_documents from wandbot.utils import clean_document_content @@ -80,7 +79,7 @@ def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): **Response Formatting** - Always communicate with the user in Markdown. - Do not use headers in your output as it will be rendered in slack. -- Always use footnotes to cite the sources in your answer. +- Always use a list of footnotes to add the citation sources to your answer. **Example**: @@ -143,6 +142,21 @@ def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): ] +# TODO: Add citation sources as a function calling api +# https://python.langchain.com/docs/use_cases/question_answering/citations#cite-documents +# class cited_answer(BaseModel): +# """Answer the user question based only on the given sources, and cite the sources used.""" +# +# answer: str = Field( +# ..., +# description="The answer to the user question, which is based only on the given sources.", +# ) +# citations: List[int] = Field( +# ..., +# description="The integer IDs of the SPECIFIC sources which justify the answer.", +# ) + + def load_response_synthesizer_chain(model) -> Runnable: response_prompt = ChatPromptTemplate.from_messages( RESPONSE_SYNTHESIS_PROMPT_MESSAGES diff --git a/src/wandbot/query_handler/web_search.py b/src/wandbot/query_handler/web_search.py index a2dcfc0..78c232a 100644 --- a/src/wandbot/query_handler/web_search.py +++ b/src/wandbot/query_handler/web_search.py @@ -32,8 +32,10 @@ def _rag(self, query: str) -> YouSearchResults: url = "https://api.ydc-index.io/rag" querystring = { - "query": "Weights & Biases, W&B, wandb or Weave " + query, + "query": "Answer the following question in the context of Weights & Biases, W&B, wandb and/or Weave\n" + + query, "num_web_results": self.similarity_top_k, + "safesearch": "strict", } response = requests.get(url, headers=headers, params=querystring) if response.status_code != 200: @@ -41,7 +43,10 @@ def _rag(self, query: str) -> YouSearchResults: else: results = response.json() - snippets = [hit["snippet"] for hit in results["hits"]] + snippets = [ + f'Title: {hit["title"]}\nDescription: {hit["description"]}\n{hit["snippet"]}' + for hit in results["hits"] + ] snippet_metadata = [ { "source": hit["url"], diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index d475298..f2f4c69 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -5,7 +5,6 @@ from langchain.prompts.prompt import PromptTemplate from langchain.retrievers.document_compressors import CohereRerank from langchain.schema import Document, format_document -from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.document_transformers import EmbeddingsRedundantFilter from langchain_core.runnables import ( RunnableBranch, @@ -13,7 +12,6 @@ RunnableParallel, RunnablePassthrough, ) - from wandbot.utils import clean_document_content DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template( @@ -168,6 +166,8 @@ def load_fusion_retriever_chain(base_retriever, embeddings, top_k=5): base_retriever, "vector_search" ) + redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings) + combined_retrieval_chain = ( RunnableParallel( question=query_retrieval_chain, @@ -186,31 +186,14 @@ def load_fusion_retriever_chain(base_retriever, embeddings, top_k=5): "web_context", ) | reciprocal_rank_fusion + | redundant_filter.transform_documents ) cohere_rerank_chain = load_cohere_rerank_chain(top_k=top_k) - splitter = RecursiveCharacterTextSplitter( - chunk_size=256, - chunk_overlap=0, - separators=["\n\n", "\n"], - keep_separator=False, - ) - - redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings) - ranked_retrieval_chain = ( RunnableParallel( context=combined_retrieval_chain, - # | splitter.split_documents - # | ( - # lambda x: [ - # doc - # for doc in x - # if len("".join(doc.page_content.strip().split())) > 10 - # ] - # ) - # | redundant_filter.transform_documents, question=itemgetter("question"), language=itemgetter("language"), ) From 4dff4058a879f7e079fb2eadbb31c00ed7d7a401 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Sun, 11 Feb 2024 12:09:03 +0530 Subject: [PATCH 23/41] chore: run linters and formatters --- src/wandbot/chat/rag.py | 1 + src/wandbot/chat/response_synthesis.py | 1 + src/wandbot/retriever/fusion.py | 1 + 3 files changed, 3 insertions(+) diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index d11af17..a41788c 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -2,6 +2,7 @@ from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda, RunnableParallel + from wandbot.chat.response_synthesis import load_response_synthesizer_chain from wandbot.ingestion.config import VectorStoreConfig from wandbot.query_handler.query_enhancer import load_query_enhancement_chain diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py index 3a4d9b8..6c463a6 100644 --- a/src/wandbot/chat/response_synthesis.py +++ b/src/wandbot/chat/response_synthesis.py @@ -8,6 +8,7 @@ format_document, ) from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel + from wandbot.retriever.fusion import combine_documents from wandbot.utils import clean_document_content diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py index f2f4c69..175220e 100644 --- a/src/wandbot/retriever/fusion.py +++ b/src/wandbot/retriever/fusion.py @@ -12,6 +12,7 @@ RunnableParallel, RunnablePassthrough, ) + from wandbot.utils import clean_document_content DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template( From aac8c3d0986986b4cd49e9c5bd6f6379d9b7d6f8 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Tue, 13 Feb 2024 16:09:52 +0530 Subject: [PATCH 24/41] feat: add new rag pipeline into chat interface --- src/wandbot/chat/chat.py | 426 +++---------------------- src/wandbot/chat/rag.py | 2 +- src/wandbot/chat/response_synthesis.py | 3 +- 3 files changed, 44 insertions(+), 387 deletions(-) diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 3aa0ba5..0de3e2a 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -25,195 +25,19 @@ print(f"Time taken: {response.time_taken}") """ -import json -from typing import Any, Dict, List, Optional, Tuple - -from llama_index import ServiceContext -from llama_index.callbacks import ( - CallbackManager, - TokenCountingHandler, - WandbCallbackHandler, - trace_method, -) -from llama_index.chat_engine import ContextChatEngine -from llama_index.chat_engine.types import AgentChatResponse -from llama_index.indices.postprocessor import CohereRerank -from llama_index.llms import LLM, ChatMessage, MessageRole -from llama_index.llms.generic_utils import messages_to_history_str -from llama_index.memory import BaseMemory -from llama_index.postprocessor.types import BaseNodePostprocessor -from llama_index.schema import MetadataMode, NodeWithScore, QueryBundle -from llama_index.tools import ToolOutput +import wandb +from langchain_community.callbacks import get_openai_callback +from langchain_openai import ChatOpenAI, OpenAIEmbeddings from weave.monitoring import StreamTable -import wandb from wandbot.chat.config import ChatConfig -from wandbot.chat.prompts import load_chat_prompt, partial_format -from wandbot.chat.query_enhancer import CompleteQuery, QueryHandler +from wandbot.chat.rag import load_rag_chain from wandbot.chat.schemas import ChatRequest, ChatResponse -from wandbot.retriever.base import Retriever -from wandbot.retriever.fusion import HybridRetriever -from wandbot.retriever.postprocessors import ( - LanguageFilterPostprocessor, - MetadataPostprocessor, -) -from wandbot.utils import Timer, get_logger, load_service_context +from wandbot.utils import Timer, get_logger logger = get_logger(__name__) -def rebuild_full_prompt( - message_templates: List[ChatMessage], result: Dict[str, Any] -) -> str: - system_template = messages_to_history_str(message_templates[:-1]) - - query_str = result["question"] - - context = json.loads( - result.get("source_documents", '[{"text": "", "source": ""}]') - ) - - context_str = "" - for idx, item in enumerate(context): - context_str += f"source {idx+1}: " + item["source"] + "\n\n" - context_str += "*" * 120 + "\n\n" - context_str += item["text"] + "\n\n" - context_str += "*" * 120 + "\n\n" - context_str += "---\n\n" - - query_content = partial_format( - message_templates[-1].content, - query_str=query_str, - context_str=context_str, - ) - system_template += ( - f"\n\n{message_templates[-1].role}:\t{query_content}\n\n---\n\n" - ) - - return system_template - - -class WandbContextChatEngine(ContextChatEngine): - def __init__( - self, - retriever: HybridRetriever, - llm: LLM, - memory: BaseMemory, - prefix_messages: List[ChatMessage], - node_postprocessors: Optional[List[BaseNodePostprocessor]] = None, - context_template: Optional[str] = None, - callback_manager: Optional[CallbackManager] = None, - ) -> None: - super().__init__( - retriever=retriever, - llm=llm, - memory=memory, - prefix_messages=prefix_messages, - node_postprocessors=node_postprocessors, - context_template=context_template, - callback_manager=callback_manager, - ) - self._retriever: HybridRetriever = retriever - - def _generate_context( - self, message: str, **kwargs - ) -> Tuple[str, List[NodeWithScore]]: - """Generate context information from a message.""" - - keywords = kwargs.get("keywords", []) - sub_queries = kwargs.get("sub_queries", []) - - query_nodes = self._retriever.retrieve( - message, is_avoid_query=kwargs.get("is_avoid_query") - ) - keywords_nodes = [] - sub_query_nodes = [] - - if keywords: - keywords_nodes = self._retriever.retrieve(" ".join(keywords)) - - if sub_queries: - for sub_query in sub_queries: - sub_query_nodes += self._retriever.retrieve(sub_query) - - nodes = query_nodes + keywords_nodes + sub_query_nodes - - for postprocessor in self._node_postprocessors: - nodes = postprocessor.postprocess_nodes( - nodes, query_bundle=QueryBundle(message) - ) - - context_str = "\n\n---\n\n".join( - [ - n.node.get_content(metadata_mode=MetadataMode.LLM).strip() - for n in nodes - ] - ) - - return context_str.strip(), nodes - - def _get_prefix_messages_with_context( - self, context_str: str - ) -> List[ChatMessage]: - """Get the prefix messages with context.""" - prefix_messages = self._prefix_messages - - context_str_w_sys_prompt = partial_format( - prefix_messages[-1].content, context_str=context_str - ) - return [ - *prefix_messages[:-1], - ChatMessage( - content=context_str_w_sys_prompt, - role=MessageRole.USER, - metadata={}, - ), - ] - - @trace_method("chat") - def chat( - self, - message: str, - chat_history: Optional[List[ChatMessage]] = None, - **kwargs, - ) -> AgentChatResponse: - context_str_template, nodes = self._generate_context( - message, - keywords=kwargs.get("keywords", []), - sub_queries=kwargs.get("sub_queries", []), - is_avoid_query=kwargs.get("is_avoid_query"), - ) - prefix_messages = self._get_prefix_messages_with_context( - context_str_template - ) - - prefix_messages[-1] = ChatMessage( - content=partial_format( - prefix_messages[-1].content, query_str=message - ), - role="user", - ) - - self._memory.put(prefix_messages[-1]) - all_messages = prefix_messages - chat_response = self._llm.chat(all_messages) - ai_message = chat_response.message - self._memory.put(ai_message) - - return AgentChatResponse( - response=str(chat_response.message.content), - sources=[ - ToolOutput( - tool_name="retriever", - content=str(prefix_messages[0]), - raw_input={"message": message}, - raw_output=prefix_messages[0], - ) - ], - source_nodes=nodes, - ) - - class Chat: """Class for handling chat interactions. @@ -243,205 +67,42 @@ def __init__(self, config: ChatConfig): table_name="chat_logs", project_name=self.config.wandb_project, entity_name=self.config.wandb_entity, - # f"{self.config.wandb_entity}/{self.config.wandb_project}/chat_logs" ) - self.wandb_callback = WandbCallbackHandler() - self.token_counter = TokenCountingHandler() - self.callback_manager = CallbackManager( - [self.wandb_callback, self.token_counter] + self.llm = ChatOpenAI(model=self.config.chat_model_name, temperature=0) + self.fallback_llm = ChatOpenAI( + model="gpt-3.5-turbo-0125", temperature=0 ) - self.default_service_context = load_service_context( - llm=self.config.chat_model_name, - temperature=self.config.chat_temperature, - max_retries=self.config.max_retries, - embeddings_model=self.config.embeddings_model, - embeddings_size=self.config.embeddings_dim, - callback_manager=self.callback_manager, + self.embedding_fn = OpenAIEmbeddings( + model="text-embedding-3-small", dimensions=512 ) - self.fallback_service_context = load_service_context( - llm=self.config.fallback_model_name, - temperature=self.config.chat_temperature, - max_retries=self.config.max_fallback_retries, - embeddings_model=self.config.embeddings_model, - embeddings_size=self.config.embeddings_dim, - callback_manager=self.callback_manager, + self.lang_detect_path = "data/cache/models/lid.176.bin" + self.vector_store_path = "data/cache/vectorstore" + self.rag_chain = load_rag_chain( + model=self.llm, + fallback_model=self.fallback_llm, + embeddings_model=self.embedding_fn, + lang_detect_path=self.lang_detect_path, + vector_store_path=self.vector_store_path, + search_type="mmr", + top_k=10, ) - self.qa_prompt = load_chat_prompt(f_name=self.config.chat_prompt) - self.query_handler = QueryHandler() - self.retriever = Retriever( - run=self.run, - service_context=self.fallback_service_context, - callback_manager=self.callback_manager, - ) - - def _load_chat_engine( - self, - service_context: ServiceContext, - query_intent: str = "\n", - language: str = "en", - initial_k: int = 10, - top_k: int = 10, - ) -> WandbContextChatEngine: - """Loads the chat engine with the given model name and maximum retries. - - Args: - service_context: An instance of ServiceContext. - query_intent: A string representing the query intent. - language: A string representing the language. - initial_k: An integer representing the initial number of documents to retrieve. - top_k: An integer representing the number of documents to retrieve after reranking. - - Returns: - An instance of ChatEngine. - """ - - query_engine = self.retriever.load_query_engine( - language=language, - top_k=top_k, - is_avoid_query=True if "avoid" in query_intent.lower() else False, + def _get_answer(self, question, chat_history): + result = self.rag_chain.invoke( + {"query": question, "chat_history": chat_history} ) - self.qa_prompt = load_chat_prompt( - f_name=self.config.chat_prompt, - language_code=language, - query_intent=query_intent, - ) - chat_engine_kwargs = dict( - retriever=query_engine.retriever, - storage_context=self.retriever.storage_context, - service_context=service_context, - similarity_top_k=initial_k, - response_mode="compact", - node_postprocessors=[ - MetadataPostprocessor(), - LanguageFilterPostprocessor(languages=[language, "python"]), - ( - CohereRerank(top_n=top_k, model="rerank-english-v2.0") - if language == "en" - else CohereRerank( - top_n=top_k, model="rerank-multilingual-v2.0" - ) - ), + return { + "question": result["query"]["question"], + "answer": result["answer"]["response"], + "sources": [ + item["metadata"]["source"] for item in result["context"] ], - prefix_messages=self.qa_prompt.message_templates, - ) - - chat_engine = WandbContextChatEngine.from_defaults(**chat_engine_kwargs) - - return chat_engine - - def format_response(self, result: Dict[str, Any]) -> Dict[str, Any]: - """Formats the response dictionary. - - Args: - result: A dictionary representing the response. - - Returns: - A formatted response dictionary. - """ - response = {} - if result.get("source_documents", None): - source_documents = [ - { - "source": doc.metadata["source"], - "text": doc.text, - } - for doc in result["source_documents"] - ] - else: - source_documents = [] - response["answer"] = result["answer"] - response["model"] = result["model"] - - if len(source_documents) and self.config.include_sources: - response["source_documents"] = json.dumps(source_documents) - response["sources"] = ",".join( - [doc["source"] for doc in source_documents] - ) - else: - response["source_documents"] = "" - response["sources"] = "" - - return response - - def get_response( - self, - service_context: ServiceContext, - query: str, - language: str, - chat_history: List[ChatMessage], - query_intent: str, - keywords: List[str] | None = None, - sub_queries: List[str] | None = None, - ) -> Dict[str, Any]: - chat_engine = self._load_chat_engine( - service_context=service_context, - language=language, - query_intent=query_intent, - ) - response = chat_engine.chat( - message=query, - chat_history=chat_history, - keywords=keywords, - sub_queries=sub_queries, - is_avoid_query=True if "avoid" in query_intent.lower() else False, - ) - result = { - "answer": response.response, - "source_documents": response.source_nodes, - "model": self.config.chat_model_name, + "source_documents": result["answer"]["context_str"], + "system_prompt": result["answer"]["response_prompt"].to_string(), + "model": result["answer"]["response_model"].model_name, } - return result - - def get_answer( - self, - resolved_query: CompleteQuery, - **kwargs, - ) -> Dict[str, Any]: - """Gets the answer for the given query and chat history. - - Args: - resolved_query: An instance of ResolvedQuery representing the resolved query. - - Returns: - A dictionary representing the answer. - - """ - try: - result = self.get_response( - service_context=self.default_service_context, - query=resolved_query.condensed_query, - language=resolved_query.language, - chat_history=resolved_query.chat_history, - query_intent=resolved_query.intent_hints, - ) - except Exception as e: - logger.warning(f"{self.config.chat_model_name} failed with {e}") - logger.warning( - f"Falling back to {self.config.fallback_model_name} model" - ) - try: - result = self.get_response( - service_context=self.fallback_service_context, - query=resolved_query.cleaned_query, - language=resolved_query.language, - chat_history=resolved_query.chat_history, - query_intent=resolved_query.intent_hints, - ) - - except Exception as e: - logger.error( - f"{self.config.fallback_model_name} failed with {e}" - ) - result = { - "answer": "\uE058" - + " Sorry, there seems to be an issue with our LLM service. Please try again in some time.", - "source_documents": None, - "model": "None", - } - return self.format_response(result) def __call__(self, chat_request: ChatRequest) -> ChatResponse: """Handles the chat request and returns the chat response. @@ -453,16 +114,16 @@ def __call__(self, chat_request: ChatRequest) -> ChatResponse: An instance of `ChatResponse` representing the chat response. """ try: - with Timer() as timer: - result = {} - resolved_query = self.query_handler(chat_request) - result = self.get_answer(resolved_query) - usage_stats = { - "total_tokens": self.token_counter.total_llm_token_count, - "prompt_tokens": self.token_counter.prompt_llm_token_count, - "completion_tokens": self.token_counter.completion_llm_token_count, - } - self.token_counter.reset_counts() + with Timer() as timer, get_openai_callback() as oai_cb: + result = self._get_answer( + chat_request.question, chat_request.chat_history + ) + + usage_stats = { + "total_tokens": oai_cb.total_tokens, + "prompt_tokens": oai_cb.prompt_tokens, + "completion_tokens": oai_cb.completion_tokens, + } result.update( dict( **{ @@ -476,11 +137,6 @@ def __call__(self, chat_request: ChatRequest) -> ChatResponse: ) ) self.run.log(usage_stats) - - system_template = rebuild_full_prompt( - self.qa_prompt.message_templates, result - ) - result["system_prompt"] = system_template self.chat_table.log(result) return ChatResponse(**result) except Exception as e: diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index a41788c..3fe8b27 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -75,7 +75,7 @@ def load_rag_chain( for p in x["context"] ] ), - result=response_synthesis_chain, + answer=response_synthesis_chain, ).with_config({"run_name": "response_synthesis"}) ) diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py index 6c463a6..236f393 100644 --- a/src/wandbot/chat/response_synthesis.py +++ b/src/wandbot/chat/response_synthesis.py @@ -143,7 +143,7 @@ def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): ] -# TODO: Add citation sources as a function calling api +# TODO: Add citation sources using function calling api # https://python.langchain.com/docs/use_cases/question_answering/citations#cite-documents # class cited_answer(BaseModel): # """Answer the user question based only on the given sources, and cite the sources used.""" @@ -180,6 +180,7 @@ def load_response_synthesizer_chain(model) -> Runnable: context_str=itemgetter("context_str"), response_prompt=itemgetter("response_prompt"), response=itemgetter("response_prompt") | model | StrOutputParser(), + reponse_model=model, ) ) From a4d5a2593391d89f9cd0d4701428a7a0d1492054 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Tue, 13 Feb 2024 16:37:36 +0530 Subject: [PATCH 25/41] feat: fix chat endpoint with api --- src/wandbot/api/app.py | 10 +++++----- src/wandbot/api/routers/retrieve.py | 23 ++--------------------- src/wandbot/chat/chat.py | 11 +++++------ src/wandbot/chat/response_synthesis.py | 3 +-- 4 files changed, 13 insertions(+), 34 deletions(-) diff --git a/src/wandbot/api/app.py b/src/wandbot/api/app.py index dea7960..571583a 100644 --- a/src/wandbot/api/app.py +++ b/src/wandbot/api/app.py @@ -33,12 +33,12 @@ from datetime import datetime, timezone import pandas as pd -from fastapi import FastAPI - import wandb +from fastapi import FastAPI from wandbot.api.routers import chat as chat_router from wandbot.api.routers import database as database_router -from wandbot.api.routers import retrieve as retrieve_router + +# from wandbot.api.routers import retrieve as retrieve_router from wandbot.utils import get_logger logger = get_logger(__name__) @@ -57,7 +57,7 @@ async def lifespan(app: FastAPI): """ chat_router.chat = chat_router.Chat(chat_router.chat_config) database_router.db_client = database_router.DatabaseClient() - retrieve_router.retriever = chat_router.chat.retriever + # retrieve_router.retriever = chat_router.chat.retriever async def backup_db(): """Periodically backs up the database to a table. @@ -98,7 +98,7 @@ async def backup_db(): app.include_router(chat_router.router) app.include_router(database_router.router) -app.include_router(retrieve_router.router) +# app.include_router(retrieve_router.router) if __name__ == "__main__": diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py index ac72896..8481dcf 100644 --- a/src/wandbot/api/routers/retrieve.py +++ b/src/wandbot/api/routers/retrieve.py @@ -1,10 +1,8 @@ -from enum import Enum -from typing import Any, List +from typing import Any, Dict, List from fastapi import APIRouter from pydantic import BaseModel from starlette import status - from wandbot.retriever.base import Retriever router = APIRouter( @@ -18,7 +16,7 @@ class APIRetrievalResult(BaseModel): text: str score: float - metadata: dict[str, Any] + metadata: Dict[str, Any] class APIRetrievalResponse(BaseModel): @@ -26,25 +24,8 @@ class APIRetrievalResponse(BaseModel): top_k: List[APIRetrievalResult] -class Indices(str, Enum): - """The indices available for retrieval.""" - - DOCODILE_EN = "docodile_en" - DOCODILE_JA = "docodile_ja" - WANDB_EXAMPLES_CODE = "wandb_examples_code" - WANDB_EXAMPLES_COLAB = "wandb_examples_colab" - WANDB_SDK_CODE = "wandb_sdk_code" - WANDB_SDK_TESTS = "wandb_sdk_tests" - WEAVE_SDK_CODE = "weave_sdk_code" - WEAVE_EXAMPLES = "weave_examples" - WANDB_EDU_CODE = "wandb_edu_code" - WEAVE_JS = "weave_js" - FC_REPORTS = "fc_reports" - - class APIRetrievalRequest(BaseModel): query: str - indices: List[Indices] | None = None language: str = "en" initial_k: int = 10 top_k: int = 5 diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 0de3e2a..2536031 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -28,12 +28,11 @@ import wandb from langchain_community.callbacks import get_openai_callback from langchain_openai import ChatOpenAI, OpenAIEmbeddings -from weave.monitoring import StreamTable - from wandbot.chat.config import ChatConfig from wandbot.chat.rag import load_rag_chain from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.utils import Timer, get_logger +from weave.monitoring import StreamTable logger = get_logger(__name__) @@ -96,12 +95,12 @@ def _get_answer(self, question, chat_history): return { "question": result["query"]["question"], "answer": result["answer"]["response"], - "sources": [ - item["metadata"]["source"] for item in result["context"] - ], + "sources": "\n".join( + [item["metadata"]["source"] for item in result["context"]] + ), "source_documents": result["answer"]["context_str"], "system_prompt": result["answer"]["response_prompt"].to_string(), - "model": result["answer"]["response_model"].model_name, + "model": result["answer"]["response_model"], } def __call__(self, chat_request: ChatRequest) -> ChatResponse: diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/chat/response_synthesis.py index 236f393..02d3dc4 100644 --- a/src/wandbot/chat/response_synthesis.py +++ b/src/wandbot/chat/response_synthesis.py @@ -8,7 +8,6 @@ format_document, ) from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel - from wandbot.retriever.fusion import combine_documents from wandbot.utils import clean_document_content @@ -180,7 +179,7 @@ def load_response_synthesizer_chain(model) -> Runnable: context_str=itemgetter("context_str"), response_prompt=itemgetter("response_prompt"), response=itemgetter("response_prompt") | model | StrOutputParser(), - reponse_model=model, + response_model=RunnableLambda(lambda x: model.model_name), ) ) From e12d1638aab53b6267f5376eda0f91a7c3ddd05a Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Wed, 14 Feb 2024 21:44:44 +0530 Subject: [PATCH 26/41] refactor: change up the RAG pipeline with descriptors --- src/wandbot/chat/chat.py | 44 ++-- src/wandbot/chat/config.py | 1 + src/wandbot/chat/prompts.py | 92 -------- src/wandbot/chat/rag.py | 138 ++++++------ src/wandbot/ingestion/config.py | 2 +- src/wandbot/query_handler/__init__.py | 0 src/wandbot/query_handler/history_handler.py | 65 ------ .../query_handler/keyword_search_enhancer.py | 84 -------- .../query_handler/language_detection.py | 25 --- src/wandbot/query_handler/query_enhancer.py | 69 ------ .../query_handler/vector_search_enhancer.py | 72 ------- src/wandbot/rag/__init__.py | 5 + src/wandbot/rag/query_handler/__init__.py | 3 + .../rag/query_handler/history_handler.py | 84 ++++++++ .../query_handler/intents_enhancer.py | 152 +++++++------ .../query_handler/keyword_search_enhancer.py | 101 +++++++++ .../rag/query_handler/language_detection.py | 19 ++ .../rag/query_handler/query_enhancer.py | 84 ++++++++ .../query_handler/vector_search_enhancer.py | 86 ++++++++ .../{ => rag}/query_handler/web_search.py | 153 ++++++------- .../rag/response_synthesis/__init__.py | 3 + .../response_synthesis}/response_synthesis.py | 102 +++++---- src/wandbot/rag/retrieval/__init__.py | 3 + src/wandbot/rag/retrieval/fusion.py | 184 ++++++++++++++++ src/wandbot/rag/utils.py | 105 +++++++++ src/wandbot/retriever/__init__.py | 4 + src/wandbot/retriever/base.py | 84 +++++--- src/wandbot/retriever/external.py | 72 ------- src/wandbot/retriever/fusion.py | 203 ------------------ src/wandbot/retriever/postprocessors.py | 84 -------- src/wandbot/retriever/utils.py | 21 ++ src/wandbot/utils.py | 36 +++- 32 files changed, 1072 insertions(+), 1108 deletions(-) delete mode 100644 src/wandbot/chat/prompts.py delete mode 100644 src/wandbot/query_handler/__init__.py delete mode 100644 src/wandbot/query_handler/history_handler.py delete mode 100644 src/wandbot/query_handler/keyword_search_enhancer.py delete mode 100644 src/wandbot/query_handler/language_detection.py delete mode 100644 src/wandbot/query_handler/query_enhancer.py delete mode 100644 src/wandbot/query_handler/vector_search_enhancer.py create mode 100644 src/wandbot/rag/__init__.py create mode 100644 src/wandbot/rag/query_handler/__init__.py create mode 100644 src/wandbot/rag/query_handler/history_handler.py rename src/wandbot/{ => rag}/query_handler/intents_enhancer.py (69%) create mode 100644 src/wandbot/rag/query_handler/keyword_search_enhancer.py create mode 100644 src/wandbot/rag/query_handler/language_detection.py create mode 100644 src/wandbot/rag/query_handler/query_enhancer.py create mode 100644 src/wandbot/rag/query_handler/vector_search_enhancer.py rename src/wandbot/{ => rag}/query_handler/web_search.py (56%) create mode 100644 src/wandbot/rag/response_synthesis/__init__.py rename src/wandbot/{chat => rag/response_synthesis}/response_synthesis.py (83%) create mode 100644 src/wandbot/rag/retrieval/__init__.py create mode 100644 src/wandbot/rag/retrieval/fusion.py create mode 100644 src/wandbot/rag/utils.py delete mode 100644 src/wandbot/retriever/external.py delete mode 100644 src/wandbot/retriever/fusion.py delete mode 100644 src/wandbot/retriever/postprocessors.py create mode 100644 src/wandbot/retriever/utils.py diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 2536031..55f40cc 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -27,10 +27,10 @@ import wandb from langchain_community.callbacks import get_openai_callback -from langchain_openai import ChatOpenAI, OpenAIEmbeddings from wandbot.chat.config import ChatConfig -from wandbot.chat.rag import load_rag_chain +from wandbot.chat.rag import Pipeline from wandbot.chat.schemas import ChatRequest, ChatResponse +from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import Timer, get_logger from weave.monitoring import StreamTable @@ -68,39 +68,23 @@ def __init__(self, config: ChatConfig): entity_name=self.config.wandb_entity, ) - self.llm = ChatOpenAI(model=self.config.chat_model_name, temperature=0) - self.fallback_llm = ChatOpenAI( - model="gpt-3.5-turbo-0125", temperature=0 - ) - self.embedding_fn = OpenAIEmbeddings( - model="text-embedding-3-small", dimensions=512 - ) - self.lang_detect_path = "data/cache/models/lid.176.bin" - self.vector_store_path = "data/cache/vectorstore" - self.rag_chain = load_rag_chain( - model=self.llm, - fallback_model=self.fallback_llm, - embeddings_model=self.embedding_fn, - lang_detect_path=self.lang_detect_path, - vector_store_path=self.vector_store_path, - search_type="mmr", - top_k=10, - ) + self.rag_pipeline = Pipeline(VectorStoreConfig()) def _get_answer(self, question, chat_history): - result = self.rag_chain.invoke( - {"query": question, "chat_history": chat_history} - ) + result = self.rag_pipeline(question, chat_history) return { - "question": result["query"]["question"], - "answer": result["answer"]["response"], + "question": result["enhanced_query"]["question"], + "answer": result["response"]["response"], "sources": "\n".join( - [item["metadata"]["source"] for item in result["context"]] + [ + item["metadata"]["source"] + for item in result["retrieval_results"]["context"] + ] ), - "source_documents": result["answer"]["context_str"], - "system_prompt": result["answer"]["response_prompt"].to_string(), - "model": result["answer"]["response_model"], + "source_documents": result["response"]["context_str"], + "system_prompt": result["response"]["response_prompt"], + "model": result["response"]["response_model"], } def __call__(self, chat_request: ChatRequest) -> ChatResponse: @@ -115,7 +99,7 @@ def __call__(self, chat_request: ChatRequest) -> ChatResponse: try: with Timer() as timer, get_openai_callback() as oai_cb: result = self._get_answer( - chat_request.question, chat_request.chat_history + chat_request.question, chat_request.chat_history or [] ) usage_stats = { diff --git a/src/wandbot/chat/config.py b/src/wandbot/chat/config.py index d8875ec..3e50f76 100644 --- a/src/wandbot/chat/config.py +++ b/src/wandbot/chat/config.py @@ -25,6 +25,7 @@ class ChatConfig(BaseSettings): max_fallback_retries: int = 6 chat_temperature: float = 0.1 chat_prompt: pathlib.Path = pathlib.Path("data/prompts/chat_prompt.json") + lang_detect_path: str = "data/cache/models/lid.176.bin" index_artifact: str = Field( "wandbot/wandbot-dev/wandbot_index:latest", env="WANDB_INDEX_ARTIFACT", diff --git a/src/wandbot/chat/prompts.py b/src/wandbot/chat/prompts.py deleted file mode 100644 index 4f4f29c..0000000 --- a/src/wandbot/chat/prompts.py +++ /dev/null @@ -1,92 +0,0 @@ -"""This module provides functionality for loading chat prompts. - -The main function in this module is `load_chat_prompt`, which loads a chat prompt from a given JSON file. -The JSON file should contain two keys: "system_template" and "human_template", which correspond to the system and user messages respectively. - -Typical usage example: - - from wandbot.chat import prompts - - chat_prompt = prompts.load_chat_prompt('path_to_your_json_file.json') -""" - -import json -import logging -import pathlib -from typing import Union - -from llama_index import ChatPromptTemplate -from llama_index.llms import ChatMessage, MessageRole - -logger = logging.getLogger(__name__) - - -def partial_format(s, **kwargs): - # Manually parse the string and extract the field names - place_holders = set() - field_name = "" - in_field = False - for c in s: - if c == "{" and not in_field: - in_field = True - elif c == "}" and in_field: - place_holders.add(field_name) - field_name = "" - in_field = False - elif in_field: - field_name += c - replacements = {k: kwargs.get(k, "{" + k + "}") for k in place_holders} - - # Escape all curly braces - s = s.replace("{", "{{").replace("}", "}}") - - # Replace the placeholders - for k, v in replacements.items(): - s = s.replace("{{" + k + "}}", v) - - return s - - -ROLE_MAP = { - "system": MessageRole.SYSTEM, - "human": MessageRole.USER, - "assistant": MessageRole.ASSISTANT, -} - - -def load_chat_prompt( - f_name: Union[pathlib.Path, str] = None, - language_code: str = "en", - query_intent: str = "", -) -> ChatPromptTemplate: - """ - Loads a chat prompt from a given file. - - This function reads a JSON file specified by f_name and constructs a ChatPromptTemplate - object from the data. The JSON file should contain two keys: "system_template" and "human_template", - which correspond to the system and user messages respectively. - - Args: - f_name: A string or a pathlib.Path object representing the path to the JSON file. - If None, a default path is used. - - Returns: - A ChatPromptTemplate object constructed from the data in the JSON file. - """ - f_name = pathlib.Path(f_name) - - template = json.load(f_name.open("r")) - - human_template = partial_format( - template["messages"][-1]["human"], - language_code=language_code, - query_intent=query_intent, - ) - - messages = [] - for message in template["messages"][:-1]: - for k, v in message.items(): - messages.append(ChatMessage(role=ROLE_MAP[k], content=v)) - messages.append(ChatMessage(role=MessageRole.USER, content=human_template)) - prompt = ChatPromptTemplate(messages) - return prompt diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 3fe8b27..156326f 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -1,82 +1,76 @@ -from operator import itemgetter +from typing import List, Tuple -from langchain_core.documents import Document -from langchain_core.runnables import RunnableLambda, RunnableParallel - -from wandbot.chat.response_synthesis import load_response_synthesizer_chain +from langchain_community.callbacks import get_openai_callback from wandbot.ingestion.config import VectorStoreConfig -from wandbot.query_handler.query_enhancer import load_query_enhancement_chain -from wandbot.retriever.base import ( - load_retriever_with_options, - load_vector_store_from_config, -) -from wandbot.retriever.fusion import load_fusion_retriever_chain +from wandbot.rag import FusionRetrieval, QueryEnhancer, ResponseSynthesizer +from wandbot.utils import Timer -def load_rag_chain( - model, - fallback_model, - embeddings_model, - lang_detect_path, - vector_store_path, - search_type, - top_k=10, -): - fallback_query_enhancer_chain = load_query_enhancement_chain( - fallback_model, lang_detect_path - ) - query_enhancer_chain = load_query_enhancement_chain( - model, lang_detect_path - ).with_fallbacks([fallback_query_enhancer_chain]) +def get_stats_dict_from_token_callback(token_callback): + return { + "total_tokens": token_callback.total_tokens, + "prompt_tokens": token_callback.prompt_tokens, + "completion_tokens": token_callback.completion_tokens, + "successful_requests": token_callback.successful_requests, + } - vectorstore_config = VectorStoreConfig(persist_dir=vector_store_path) - vectorstore = load_vector_store_from_config(vectorstore_config) - base_retriever = load_retriever_with_options( - vectorstore, search_type=search_type, search_kwargs={"top_k": top_k * 4} - ) - parent_retriever = base_retriever | RunnableLambda( - lambda docs: [ - Document( - page_content=doc.metadata.get( - "source_content", doc.page_content - ), - metadata=doc.metadata, - ) - for doc in docs - ] - ) - fallback_response_synthesis_chain = load_response_synthesizer_chain( - fallback_model - ) - response_synthesis_chain = load_response_synthesizer_chain( - model - ).with_fallbacks([fallback_response_synthesis_chain]) +def get_stats_dict_from_timer(timer): + return { + "start_time": timer.start, + "end_time": timer.stop, + "time_taken": timer.elapsed, + } - ranked_retrieval_chain = load_fusion_retriever_chain( - parent_retriever, embeddings=embeddings_model, top_k=top_k - ) - rag_chain = ( - RunnableParallel( - query=query_enhancer_chain.with_config( - {"run_name": "query_enhancer"} - ) +class Pipeline: + def __init__( + self, + vector_store_config: VectorStoreConfig, + top_k: int = 5, + search_type: str = "mmr", + ): + self.query_enhancer = QueryEnhancer() + self.retrieval = FusionRetrieval( + vector_store_config, top_k=top_k, search_type=search_type ) - | RunnableParallel( - query=itemgetter("query"), - context=lambda x: itemgetter("query") | ranked_retrieval_chain, - ).with_config({"run_name": "retrieval"}) - | RunnableParallel( - query=itemgetter("query"), - context=RunnableLambda( - lambda x: [ - {"page_content": p.page_content, "metadata": p.metadata} - for p in x["context"] - ] - ), - answer=response_synthesis_chain, - ).with_config({"run_name": "response_synthesis"}) - ) + self.response_synthesizer = ResponseSynthesizer() + + def __call__( + self, question: str, chat_history: List[Tuple[str, str]] | None = None + ): + with get_openai_callback() as query_enhancer_cb, Timer() as query_enhancer_tb: + enhanced_query = self.query_enhancer.chain.invoke( + {"query": question, "chat_history": chat_history} + ) + with get_openai_callback() as retrieval_cb, Timer() as retrieval_tb: + retrieval_results = self.retrieval.chain.invoke(enhanced_query) + with get_openai_callback() as response_cb, Timer() as response_tb: + response = self.response_synthesizer.chain.invoke( + {"query": enhanced_query, "context": retrieval_results} + ) + + contexts = { + "context": [ + {"page_content": item.page_content, "metadata": item.metadata} + for item in retrieval_results + ] + } - return rag_chain + return { + "enhanced_query": { + **enhanced_query, + **get_stats_dict_from_token_callback(query_enhancer_cb), + **get_stats_dict_from_timer(query_enhancer_tb), + }, + "retrieval_results": { + **contexts, + **get_stats_dict_from_token_callback(retrieval_cb), + **get_stats_dict_from_timer(retrieval_tb), + }, + "response": { + **response, + **get_stats_dict_from_token_callback(response_cb), + **get_stats_dict_from_timer(response_tb), + }, + } diff --git a/src/wandbot/ingestion/config.py b/src/wandbot/ingestion/config.py index 398c8e2..aa280c6 100644 --- a/src/wandbot/ingestion/config.py +++ b/src/wandbot/ingestion/config.py @@ -18,7 +18,6 @@ from pydantic import BaseModel, Field, model_validator from pydantic_settings import BaseSettings - from wandbot.utils import get_logger logger = get_logger(__name__) @@ -245,3 +244,4 @@ class VectorStoreConfig(BaseSettings): embedding_dim: int = 512 persist_dir: pathlib.Path = pathlib.Path("data/cache/vectorstore") batch_size: int = 256 + artifact_url: str = "wandbot/wandbot-dev/chroma_index:latest" diff --git a/src/wandbot/query_handler/__init__.py b/src/wandbot/query_handler/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/wandbot/query_handler/history_handler.py b/src/wandbot/query_handler/history_handler.py deleted file mode 100644 index dc23fe1..0000000 --- a/src/wandbot/query_handler/history_handler.py +++ /dev/null @@ -1,65 +0,0 @@ -from _operator import itemgetter -from langchain_core.messages import get_buffer_string -from langchain_core.output_parsers import StrOutputParser -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import ( - Runnable, - RunnableBranch, - RunnableLambda, - RunnablePassthrough, -) -from langchain_openai import ChatOpenAI - -CONDENSE_PROMPT_SYSTEM_TEMPLATE = """Given the following conversation and a follow up question, rephrase the follow up \ -question to be a standalone question. - -Chat History: -{chat_history} -Follow Up Input: {question} -Standalone Question:""" - - -CONDENSE_PROMPT_MESSAGES = [ - ( - "system", - CONDENSE_PROMPT_SYSTEM_TEMPLATE, - ), -] - - -def load_standalone_query_chain(model: ChatOpenAI) -> Runnable: - condense_prompt = ChatPromptTemplate.from_messages(CONDENSE_PROMPT_MESSAGES) - - condense_question_chain = ( - { - "question": RunnablePassthrough(), - "chat_history": itemgetter("chat_history") - | RunnableLambda(get_buffer_string), - } - | condense_prompt - | model - | StrOutputParser() - ) - - return condense_question_chain - - -def load_query_condense_chain( - model: ChatOpenAI, -) -> Runnable: - standalone_query_chain = load_standalone_query_chain( - model, - ) - branch = RunnableBranch( - ( - lambda x: True if x["chat_history"] else False, - standalone_query_chain, - ), - ( - lambda x: False if x["chat_history"] else True, - itemgetter("question"), - ), - itemgetter("question"), - ) - - return branch diff --git a/src/wandbot/query_handler/keyword_search_enhancer.py b/src/wandbot/query_handler/keyword_search_enhancer.py deleted file mode 100644 index 04474ba..0000000 --- a/src/wandbot/query_handler/keyword_search_enhancer.py +++ /dev/null @@ -1,84 +0,0 @@ -from operator import itemgetter -from typing import List - -from langchain.chains.openai_functions import create_structured_output_runnable -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import ( - Runnable, - RunnableBranch, - RunnableLambda, - RunnableParallel, -) -from langchain_openai import ChatOpenAI -from pydantic.v1 import BaseModel, Field - -KEYWORDS_SYSTEM_PROMPT = ( - "You are a Weights & Biases support manager. " - "Your goal is to enhance the user query by adding a list of keywords used for web search." -) - -KEYWORDS_PROMPT_MESSAGES = [ - ("system", KEYWORDS_SYSTEM_PROMPT), - ( - "human", - "Enhance the following query related to weights and biases for web search.:\n\n{question}", - ), - ("human", "Tip: Make sure to answer in the correct format"), -] - - -class Keyword(BaseModel): - """A Keyword to search for on the wen""" - - keyword: str = Field( - ..., - description="A search term for getting the most relevant information required to answer the query", - ) - - -class KeywordsSchema(BaseModel): - "A list of search keywords to enhance the search query" - keywords: List[Keyword] = Field( - ..., - description="List of five different search terms", - min_items=0, - max_items=5, - ) - - -def load_keywords_extraction_chain(model: ChatOpenAI) -> Runnable: - keywords_prompt = ChatPromptTemplate.from_messages(KEYWORDS_PROMPT_MESSAGES) - - keywords_extraction_chain = create_structured_output_runnable( - KeywordsSchema, model, keywords_prompt - ) - - keywords_chain = keywords_extraction_chain | RunnableLambda( - lambda x: [keyword.keyword for keyword in x.keywords] - ) - - return keywords_chain - - -def load_keywords_enhancement_chain(model: ChatOpenAI) -> Runnable: - keywords_chain = load_keywords_extraction_chain(model) - - branch = RunnableBranch( - ( - lambda x: x["avoid"], - RunnableLambda(lambda x: []), - ), - ( - lambda x: not x["avoid"], - keywords_chain, - ), - RunnableLambda(lambda x: []), - ) - - return ( - RunnableParallel( - question=itemgetter("standalone_question"), - avoid=itemgetter("avoid_query"), - ) - | branch - ) diff --git a/src/wandbot/query_handler/language_detection.py b/src/wandbot/query_handler/language_detection.py deleted file mode 100644 index 4e5484e..0000000 --- a/src/wandbot/query_handler/language_detection.py +++ /dev/null @@ -1,25 +0,0 @@ -from operator import itemgetter - -from langchain_core.runnables import Runnable, RunnablePassthrough - -from wandbot.utils import FastTextLangDetect, FasttextModelConfig - - -class LangDetect: - def __init__(self, model): - self.model = FastTextLangDetect( - FasttextModelConfig( - fasttext_file_path=model, - ) - ) - - def __call__(self, question: str) -> str: - return self.model.detect_language(question) - - -def load_language_detection_chain(model: str) -> Runnable: - lang_detect = LangDetect(model) - lang_detect_chain = RunnablePassthrough().assign( - language=lambda x: lang_detect(x["question"]) - ) | itemgetter("language") - return lang_detect_chain diff --git a/src/wandbot/query_handler/query_enhancer.py b/src/wandbot/query_handler/query_enhancer.py deleted file mode 100644 index b7c85f1..0000000 --- a/src/wandbot/query_handler/query_enhancer.py +++ /dev/null @@ -1,69 +0,0 @@ -from operator import itemgetter - -import regex as re -from langchain_core.runnables import ( - Runnable, - RunnableParallel, - RunnablePassthrough, -) -from langchain_openai import ChatOpenAI - -from wandbot.query_handler.history_handler import load_query_condense_chain -from wandbot.query_handler.intents_enhancer import load_intent_enhancement_chain -from wandbot.query_handler.keyword_search_enhancer import ( - load_keywords_enhancement_chain, -) -from wandbot.query_handler.language_detection import ( - load_language_detection_chain, -) -from wandbot.query_handler.vector_search_enhancer import ( - load_vectorsearch_enhancement_chain, -) -from wandbot.query_handler.web_search import load_web_answer_enhancement_chain - -BOT_NAME_PATTERN = re.compile(r"<@U[A-Z0-9]+>|@[a-zA-Z0-9]+") - - -def clean_question(question: str) -> str: - cleaned_query = BOT_NAME_PATTERN.sub("", question).strip() - return cleaned_query - - -def load_query_enhancement_chain( - model: ChatOpenAI, lang_detect_model_path: str -) -> Runnable: - condense_question_chain = load_query_condense_chain(model) - intent_enhancement_chain = load_intent_enhancement_chain(model) - - language_enhancement_chain = load_language_detection_chain( - model=lang_detect_model_path - ) - - keywords_enhancement_chain = load_keywords_enhancement_chain(model) - vector_search_enhancement_chain = load_vectorsearch_enhancement_chain(model) - web_answer_enhancement_chain = load_web_answer_enhancement_chain(top_k=5) - - query_enhancer_chain = ( - RunnablePassthrough().assign( - question=lambda x: clean_question(x["query"]), - ) - | RunnableParallel( - question=itemgetter("question"), - standalone_question=condense_question_chain, - language=language_enhancement_chain, - chat_history=itemgetter("chat_history"), - ) - | intent_enhancement_chain - | RunnableParallel( - standalone_question=itemgetter("standalone_question"), - language=itemgetter("language"), - question=itemgetter("question"), - intents=itemgetter("intents"), - chat_history=itemgetter("chat_history"), - keywords=keywords_enhancement_chain, - vector_search=vector_search_enhancement_chain, - web_results=web_answer_enhancement_chain, - avoid_query=itemgetter("avoid_query"), - ) - ) - return query_enhancer_chain diff --git a/src/wandbot/query_handler/vector_search_enhancer.py b/src/wandbot/query_handler/vector_search_enhancer.py deleted file mode 100644 index eaba6bc..0000000 --- a/src/wandbot/query_handler/vector_search_enhancer.py +++ /dev/null @@ -1,72 +0,0 @@ -from operator import itemgetter - -from langchain.chains.openai_functions import create_structured_output_runnable -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import ( - Runnable, - RunnableBranch, - RunnableLambda, - RunnableParallel, - RunnableSerializable, -) -from langchain_openai import ChatOpenAI -from pydantic.v1 import BaseModel, Field - -QUERY_REWRITE_SYSTEM_PROMPT = ( - "You are a Weights & Biases support manager. " - "Your goal is to enhance the user query by rewriting it for similarity search. " - "Rewrite the given query into a clear, specific, and formal request for retrieving relevant information from a vector database" -) - -QUERY_REWRITE_PROMPT_MESSAGES = [ - ("system", QUERY_REWRITE_SYSTEM_PROMPT), - ("human", "Enhance the following query i.:\n\n{question}"), - ("human", "Tip: Make sure to answer in the correct format"), -] - - -class EnhancedQuery(BaseModel): - "A query suitable for similarity search in a vectorstore" - query_str: str = Field( - ..., description="A query suitable for similarity search and retrieval" - ) - - -def load_query_rewrite_chain(model: ChatOpenAI) -> RunnableSerializable: - query_rewrite_prompt = ChatPromptTemplate.from_messages( - QUERY_REWRITE_PROMPT_MESSAGES - ) - - query_rewrite_chain = create_structured_output_runnable( - EnhancedQuery, model, query_rewrite_prompt - ) - - question_rewrite_chain = query_rewrite_chain | RunnableLambda( - lambda x: x.query_str - ) - - return question_rewrite_chain - - -def load_vectorsearch_enhancement_chain(model: ChatOpenAI) -> Runnable: - vectorsearch_chain = load_query_rewrite_chain(model) - - branch = RunnableBranch( - ( - lambda x: x["avoid"], - RunnableLambda(lambda x: []), - ), - ( - lambda x: not x["avoid"], - vectorsearch_chain, - ), - RunnableLambda(lambda x: []), - ) - - return ( - RunnableParallel( - question=itemgetter("standalone_question"), - avoid=itemgetter("avoid_query"), - ) - | branch - ) diff --git a/src/wandbot/rag/__init__.py b/src/wandbot/rag/__init__.py new file mode 100644 index 0000000..daa1333 --- /dev/null +++ b/src/wandbot/rag/__init__.py @@ -0,0 +1,5 @@ +from .query_handler import QueryEnhancer +from .response_synthesis import ResponseSynthesizer +from .retrieval import FusionRetrieval + +__all__ = ["QueryEnhancer", "ResponseSynthesizer", "FusionRetrieval"] diff --git a/src/wandbot/rag/query_handler/__init__.py b/src/wandbot/rag/query_handler/__init__.py new file mode 100644 index 0000000..821bdb9 --- /dev/null +++ b/src/wandbot/rag/query_handler/__init__.py @@ -0,0 +1,3 @@ +from .query_enhancer import QueryEnhancer + +__all__ = ["QueryEnhancer"] diff --git a/src/wandbot/rag/query_handler/history_handler.py b/src/wandbot/rag/query_handler/history_handler.py new file mode 100644 index 0000000..ec6d749 --- /dev/null +++ b/src/wandbot/rag/query_handler/history_handler.py @@ -0,0 +1,84 @@ +from _operator import itemgetter +from langchain_core.messages import convert_to_messages, get_buffer_string +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import ( + Runnable, + RunnableBranch, + RunnableLambda, + RunnableParallel, + RunnablePassthrough, +) +from langchain_openai import ChatOpenAI +from wandbot.rag.utils import ChatModel + +CONDENSE_PROMPT_SYSTEM_TEMPLATE = """Given the following conversation and a follow up question, rephrase the follow up \ +question to be a standalone question. + +Chat History: +{chat_history} +Follow Up Input: {question} +Standalone Question:""" + + +CONDENSE_PROMPT_MESSAGES = [ + ( + "system", + CONDENSE_PROMPT_SYSTEM_TEMPLATE, + ), +] + + +class CondenseQuestion: + model: ChatModel = ChatModel() + fallback_model: ChatModel = ChatModel(max_retries=6) + + def __init__( + self, + model: str = "gpt-4-0125-preview", + fallback_model="gpt-3.5-turbo-1106", + ): + self.model = model + self.fallback_model = fallback_model + self.prompt = ChatPromptTemplate.from_messages(CONDENSE_PROMPT_MESSAGES) + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + base_chain = self._load_chain(self.model) + fallback_chain = self._load_chain(self.fallback_model) + self._chain = base_chain.with_fallbacks([fallback_chain]) + + return self._chain + + def _load_chain(self, model: ChatOpenAI) -> Runnable: + base_chain = ( + RunnableParallel( + question=RunnablePassthrough(), + chat_history=( + RunnableLambda( + lambda x: convert_to_messages(x["chat_history"]) + ) + | RunnableLambda( + lambda x: get_buffer_string(x, "user", "assistant") + ) + ), + ) + | self.prompt + | model + | StrOutputParser() + ) + + chain = RunnableBranch( + ( + lambda x: True if x["chat_history"] else False, + base_chain, + ), + ( + lambda x: False if x["chat_history"] else True, + itemgetter("question"), + ), + itemgetter("question"), + ) + return chain diff --git a/src/wandbot/query_handler/intents_enhancer.py b/src/wandbot/rag/query_handler/intents_enhancer.py similarity index 69% rename from src/wandbot/query_handler/intents_enhancer.py rename to src/wandbot/rag/query_handler/intents_enhancer.py index 50bbf72..9454eac 100644 --- a/src/wandbot/query_handler/intents_enhancer.py +++ b/src/wandbot/rag/query_handler/intents_enhancer.py @@ -1,5 +1,4 @@ import enum -import os from operator import itemgetter from typing import List @@ -14,6 +13,8 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field +from pydantic_settings import BaseSettings, SettingsConfigDict +from wandbot.rag.utils import ChatModel class Labels(str, enum.Enum): @@ -124,31 +125,38 @@ def get_intent_hints(intents: List[str]) -> str: return descriptions +class CohereClassifierConfig(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="allow" + ) + cohere_api_key: str = Field( + ..., + description="The API key for the Cohere API", + env="COHERE_API_KEY", + validation_alias="cohere_api_key", + ) + cohere_query_clf_model: str = Field( + ..., + description="The fine-tuned cohere model to use for classification", + env="COHERE_QUERY_CLF_MODEL", + validation_alias="cohere_query_clf_model", + ) + + class CohereQueryClassifier: - def __init__(self, api_key: str, model: str) -> None: - self.client = cohere.Client(api_key) - self.model = model + config: CohereClassifierConfig = CohereClassifierConfig() + + def __init__(self) -> None: + self.client = cohere.Client(self.config.cohere_api_key) def __call__(self, query: str) -> str: response = self.client.classify( - model=self.model, + model=self.config.cohere_query_clf_model, inputs=[query], ) return get_intent_descriptions(response.classifications[0].predictions) -def load_cohere_classify_chain(api_key: str, model: str) -> RunnableLambda: - cohere_classify_chain = RunnableLambda( - lambda x: { - "question": x["question"], - "intent_hints": CohereQueryClassifier(api_key, model)( - x["question"] - ), - } - ) - return cohere_classify_chain - - intents_descriptions_str = "\n".join( [ f"{label}:\t{description}" @@ -167,37 +175,12 @@ def load_cohere_classify_chain(api_key: str, model: str) -> RunnableLambda: ("human", "Enhance the following user query:\n{question}"), ( "human", - "Here is my initial list of intent hints that maybe relevant:\n{intent_hints}", + "Here is an initial list of intent hints that maybe relevant:\n{intent_hints}", ), ("human", "Tip: Make sure to answer in the correct format"), ] -def load_intent_extraction_chain(model: ChatOpenAI) -> Runnable: - intents_prompt = ChatPromptTemplate.from_messages(INTENT_PROMPT_MESSAGES) - - intents_classification_chain = create_structured_output_runnable( - MultiLabel, model, intents_prompt - ) - - cohere_classify_chain = load_cohere_classify_chain( - api_key=os.environ["COHERE_API_KEY"], - model=os.environ["DEFAULT_QUERY_CLF_MODEL"], - ) - - intent_enhancement_chain = ( - cohere_classify_chain - | intents_classification_chain - | RunnableLambda(lambda x: [intent.label.value for intent in x.intents]) - | { - "intent_hints": get_intent_hints, - "intent_labels": RunnablePassthrough(), - } - ) - - return intent_enhancement_chain - - def check_avoid_intent(intents: List[str]) -> bool: return any( [ @@ -213,20 +196,69 @@ def check_avoid_intent(intents: List[str]) -> bool: ) -def load_intent_enhancement_chain( - model: ChatOpenAI, -) -> Runnable: - intent_extraction_chain = load_intent_extraction_chain(model) - - return RunnableParallel( - question=itemgetter("question"), - standalone_question=itemgetter("standalone_question"), - chat_history=itemgetter("chat_history"), - language=itemgetter("language"), - intents=( - {"question": itemgetter("standalone_question")} - | intent_extraction_chain - ), - ) | RunnablePassthrough.assign( - avoid_query=lambda x: check_avoid_intent(x["intents"]["intent_labels"]) - ) +class IntentsEnhancer: + model: ChatModel = ChatModel() + fallback_model: ChatModel = ChatModel(max_retries=6) + + def __init__( + self, + model: str = "gpt-4-0125-preview", + fallback_model: str = "gpt-3.5-turbo-1106", + ): + self.model = model + self.fallback_model = fallback_model + + self.cohere_classifier = CohereQueryClassifier() + self.prompt = ChatPromptTemplate.from_messages(INTENT_PROMPT_MESSAGES) + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + base_chain = self._load_chain(self.model) + fallback_chain = self._load_chain(self.fallback_model) + self._chain = base_chain.with_fallbacks([fallback_chain]) + + return self._chain + + def _load_chain(self, model: ChatOpenAI) -> Runnable: + # load the cohere classifier chain + + cohere_classify_chain = RunnablePassthrough.assign( + intent_hints=lambda x: self.cohere_classifier(x["question"]) + ) + + # load the intent extraction chain + intents_classification_chain = create_structured_output_runnable( + MultiLabel, model, self.prompt + ) + + intent_extraction_chain = ( + cohere_classify_chain + | intents_classification_chain + | RunnableLambda( + lambda x: [intent.label.value for intent in x.intents] + ) + | RunnableParallel( + intent_hints=get_intent_hints, + intent_labels=RunnablePassthrough(), + ) + ) + + # load the intent enhancement chain + intent_enhancement_chain = RunnableParallel( + question=itemgetter("question"), + standalone_question=itemgetter("standalone_question"), + chat_history=itemgetter("chat_history"), + language=itemgetter("language"), + intents=( + {"question": itemgetter("standalone_question")} + | intent_extraction_chain + ), + ) | RunnablePassthrough.assign( + avoid_query=lambda x: check_avoid_intent( + x["intents"]["intent_labels"] + ) + ) + + return intent_enhancement_chain diff --git a/src/wandbot/rag/query_handler/keyword_search_enhancer.py b/src/wandbot/rag/query_handler/keyword_search_enhancer.py new file mode 100644 index 0000000..b7f7937 --- /dev/null +++ b/src/wandbot/rag/query_handler/keyword_search_enhancer.py @@ -0,0 +1,101 @@ +from operator import itemgetter +from typing import List + +from langchain.chains.openai_functions import create_structured_output_runnable +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import ( + Runnable, + RunnableBranch, + RunnableLambda, + RunnableParallel, +) +from langchain_openai import ChatOpenAI +from pydantic.v1 import BaseModel, Field +from wandbot.rag.utils import ChatModel + +KEYWORDS_SYSTEM_PROMPT = ( + "You are a Weights & Biases support manager. " + "Your goal is to enhance the user query by adding a list of keywords used for web search." +) + +KEYWORDS_PROMPT_MESSAGES = [ + ("system", KEYWORDS_SYSTEM_PROMPT), + ( + "human", + "Enhance the following query related to weights and biases for web search.:\n\n{question}", + ), + ("human", "Tip: Make sure to answer in the correct format"), +] + + +class Keyword(BaseModel): + """A Keyword to search for on the wen""" + + keyword: str = Field( + ..., + description="A search term for getting the most relevant information required to answer the query", + ) + + +class KeywordsSchema(BaseModel): + "A list of search keywords to enhance the search query" + keywords: List[Keyword] = Field( + ..., + description="List of five different search terms", + min_items=0, + max_items=5, + ) + + +class KeywordsEnhancer: + model: ChatModel = ChatModel() + fallback_model: ChatModel = ChatModel(max_retries=6) + + def __init__( + self, + model: str = "gpt-4-0125-preview", + fallback_model: str = "gpt-3.5-turbo-1106", + ): + self.model = model + self.fallback_model = fallback_model + self.prompt = ChatPromptTemplate.from_messages(KEYWORDS_PROMPT_MESSAGES) + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + base_chain = self._load_chain(self.model) + fallback_chain = self._load_chain(self.fallback_model) + self._chain = base_chain.with_fallbacks([fallback_chain]) + + return self._chain + + def _load_chain(self, model: ChatOpenAI) -> Runnable: + keywords_extraction_chain = create_structured_output_runnable( + KeywordsSchema, model, self.prompt + ) + + keywords_chain = keywords_extraction_chain | RunnableLambda( + lambda x: [keyword.keyword for keyword in x.keywords] + ) + + branch = RunnableBranch( + ( + lambda x: x["avoid"], + RunnableLambda(lambda x: []), + ), + ( + lambda x: not x["avoid"], + keywords_chain, + ), + RunnableLambda(lambda x: []), + ) + + chain = ( + RunnableParallel( + question=itemgetter("standalone_question"), + avoid=itemgetter("avoid_query"), + ) + | branch + ) + return chain diff --git a/src/wandbot/rag/query_handler/language_detection.py b/src/wandbot/rag/query_handler/language_detection.py new file mode 100644 index 0000000..65cac93 --- /dev/null +++ b/src/wandbot/rag/query_handler/language_detection.py @@ -0,0 +1,19 @@ +from operator import itemgetter + +from langchain_core.runnables import Runnable, RunnablePassthrough +from wandbot.utils import FastTextLangDetect, FasttextModelConfig + + +class LanguageDetector: + model_config = FasttextModelConfig() + + def __init__(self): + self.model = FastTextLangDetect(self.model_config) + self._chain = None + + @property + def chain(self) -> Runnable: + lang_detect_chain = RunnablePassthrough().assign( + language=lambda x: self.model.detect_language(x["question"]) + ) | itemgetter("language") + return lang_detect_chain diff --git a/src/wandbot/rag/query_handler/query_enhancer.py b/src/wandbot/rag/query_handler/query_enhancer.py new file mode 100644 index 0000000..b98b50b --- /dev/null +++ b/src/wandbot/rag/query_handler/query_enhancer.py @@ -0,0 +1,84 @@ +from operator import itemgetter + +import regex as re +from langchain_core.messages import convert_to_messages, messages_to_dict +from langchain_core.runnables import ( + Runnable, + RunnableLambda, + RunnableParallel, + RunnablePassthrough, +) +from wandbot.rag.query_handler.history_handler import CondenseQuestion +from wandbot.rag.query_handler.intents_enhancer import IntentsEnhancer +from wandbot.rag.query_handler.keyword_search_enhancer import KeywordsEnhancer +from wandbot.rag.query_handler.language_detection import LanguageDetector +from wandbot.rag.query_handler.vector_search_enhancer import ( + VectorSearchEnhancer, +) +from wandbot.rag.query_handler.web_search import YouWebRagSearchEnhancer + +BOT_NAME_PATTERN = re.compile(r"<@U[A-Z0-9]+>|@[a-zA-Z0-9]+") + + +def clean_question(question: str) -> str: + cleaned_query = BOT_NAME_PATTERN.sub("", question).strip() + return cleaned_query + + +class QueryEnhancer: + def __init__( + self, + model: str = "gpt-4-0125-preview", + fallback_model="gpt-3.5-turbo-1106", + ): + self.question_condenser = CondenseQuestion( + model=model, fallback_model=fallback_model + ) + self.intents_enhancer = IntentsEnhancer( + model=model, fallback_model=fallback_model + ) + self.language_detector = LanguageDetector() + self.keywords_enhancer = KeywordsEnhancer( + model=model, fallback_model=fallback_model + ) + self.vector_search_enhancer = VectorSearchEnhancer( + model=model, fallback_model=fallback_model + ) + self.web_search_enhancer = YouWebRagSearchEnhancer() + + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + self._chain = self._load_chain() + return self._chain + + def _load_chain(self) -> Runnable: + query_enhancer_chain = ( + RunnablePassthrough().assign( + question=lambda x: clean_question(x["query"]), + ) + | RunnableParallel( + question=itemgetter("question"), + standalone_question=self.question_condenser.chain, + language=self.language_detector.chain, + chat_history=RunnableLambda( + lambda x: convert_to_messages(x["chat_history"]) + ) + | RunnableLambda(lambda x: messages_to_dict(x)), + ) + | self.intents_enhancer.chain + | RunnableParallel( + standalone_question=itemgetter("standalone_question"), + language=itemgetter("language"), + question=itemgetter("question"), + intents=itemgetter("intents"), + chat_history=itemgetter("chat_history"), + keywords=self.keywords_enhancer.chain, + vector_search=self.vector_search_enhancer.chain, + web_results=self.web_search_enhancer.chain, + avoid_query=itemgetter("avoid_query"), + ) + ) + return query_enhancer_chain diff --git a/src/wandbot/rag/query_handler/vector_search_enhancer.py b/src/wandbot/rag/query_handler/vector_search_enhancer.py new file mode 100644 index 0000000..d10c21b --- /dev/null +++ b/src/wandbot/rag/query_handler/vector_search_enhancer.py @@ -0,0 +1,86 @@ +from operator import itemgetter + +from langchain.chains.openai_functions import create_structured_output_runnable +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import ( + Runnable, + RunnableBranch, + RunnableLambda, + RunnableParallel, +) +from langchain_openai import ChatOpenAI +from pydantic.v1 import BaseModel, Field +from wandbot.rag.utils import ChatModel + +QUERY_REWRITE_SYSTEM_PROMPT = ( + "You are a Weights & Biases support manager. " + "Your goal is to enhance the user query by rewriting it for similarity search. " + "Rewrite the given query into a clear, specific, and formal request for retrieving relevant information from a vector database" +) + +QUERY_REWRITE_PROMPT_MESSAGES = [ + ("system", QUERY_REWRITE_SYSTEM_PROMPT), + ("human", "Enhance the following query i.:\n\n{question}"), + ("human", "Tip: Make sure to answer in the correct format"), +] + + +class EnhancedQuery(BaseModel): + "A query suitable for similarity search in a vectorstore" + query_str: str = Field( + ..., description="A query suitable for similarity search and retrieval" + ) + + +class VectorSearchEnhancer: + model: ChatModel = ChatModel() + fallback_model: ChatModel = ChatModel(max_retries=6) + + def __init__( + self, + model: str = "gpt-4-0125-preview", + fallback_model: str = "gpt-3.5-turbo-1106", + ): + self.model = model + self.fallback_model = fallback_model + self.prompt = ChatPromptTemplate.from_messages( + QUERY_REWRITE_PROMPT_MESSAGES + ) + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + base_chain = self._load_chain(self.model) + fallback_chain = self._load_chain(self.fallback_model) + self._chain = base_chain.with_fallbacks([fallback_chain]) + return self._chain + + def _load_chain(self, model: ChatOpenAI) -> Runnable: + query_rewrite_chain = create_structured_output_runnable( + EnhancedQuery, model, self.prompt + ) + question_rewrite_chain = query_rewrite_chain | RunnableLambda( + lambda x: x.query_str + ) + + branch = RunnableBranch( + ( + lambda x: x["avoid"], + RunnableLambda(lambda x: []), + ), + ( + lambda x: not x["avoid"], + question_rewrite_chain, + ), + RunnableLambda(lambda x: []), + ) + + chain = ( + RunnableParallel( + question=itemgetter("standalone_question"), + avoid=itemgetter("avoid_query"), + ) + | branch + ) + return chain diff --git a/src/wandbot/query_handler/web_search.py b/src/wandbot/rag/query_handler/web_search.py similarity index 56% rename from src/wandbot/query_handler/web_search.py rename to src/wandbot/rag/query_handler/web_search.py index 78c232a..86be6d0 100644 --- a/src/wandbot/query_handler/web_search.py +++ b/src/wandbot/rag/query_handler/web_search.py @@ -1,4 +1,3 @@ -import os from operator import itemgetter from typing import Any, Dict, List @@ -11,6 +10,7 @@ RunnablePassthrough, ) from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings, SettingsConfigDict class YouSearchResults(BaseModel): @@ -20,21 +20,39 @@ class YouSearchResults(BaseModel): ) +class YouSearchConfig(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="allow" + ) + you_api_key: str = Field( + ..., + description="API key for you.com search API", + env="YOU_API_KEY", + validation_alias="you_api_key", + ) + top_k: int = Field( + 10, + description="Number of top k results to retrieve from you.com", + ) + search_type: str = Field( + "rag", + description="Type of search to perform. Options: rag, retrieve", + ) + + class YouSearch: - def __init__(self, api_key: str, similarity_top_k: int = 10): - self._api_key = api_key - self.similarity_top_k = similarity_top_k + config: YouSearchConfig = YouSearchConfig() def _rag(self, query: str) -> YouSearchResults: """Retrieve.""" try: - headers = {"X-API-Key": self._api_key} + headers = {"X-API-Key": self.config.you_api_key} url = "https://api.ydc-index.io/rag" querystring = { "query": "Answer the following question in the context of Weights & Biases, W&B, wandb and/or Weave\n" + query, - "num_web_results": self.similarity_top_k, + "num_web_results": self.config.top_k, "safesearch": "strict", } response = requests.get(url, headers=headers, params=querystring) @@ -65,7 +83,7 @@ def _rag(self, query: str) -> YouSearchResults: return YouSearchResults( web_answer=results["answer"], - web_context=search_hits[: self.similarity_top_k], + web_context=search_hits[: self.config.top_k], ) except Exception as e: return YouSearchResults() @@ -73,12 +91,12 @@ def _rag(self, query: str) -> YouSearchResults: def _retrieve(self, query: str) -> YouSearchResults: """Retrieve.""" try: - headers = {"X-API-Key": self._api_key} + headers = {"X-API-Key": self.config.you_api_key} url = "https://api.ydc-index.io/search" querystring = { "query": "Weights & Biases, W&B, wandb or Weave " + query, - "num_web_results": self.similarity_top_k, + "num_web_results": self.config.top_k, } response = requests.get(url, headers=headers, params=querystring) if response.status_code != 200: @@ -108,103 +126,52 @@ def _retrieve(self, query: str) -> YouSearchResults: return YouSearchResults( web_answer="", - web_context=search_hits[: self.similarity_top_k], + web_context=search_hits[: self.config.top_k], ) except Exception as e: print(e) return YouSearchResults() def __call__( - self, question: str, search_type: str = "rag" + self, + question: str, ) -> Dict[str, Any]: - if search_type == "rag": + if self.config.search_type == "rag": web_results = self._rag(question) else: web_results = self._retrieve(question) return web_results.dict() -def load_web_answer_chain(search_field: str, top_k: int = 5) -> Runnable: - you_search = YouSearch(os.environ["YOU_API_KEY"], top_k) - - web_answer_chain = RunnablePassthrough().assign( - web_results=lambda x: you_search( - question=x["question"], search_type=x["search_type"] - ) - ) - - branch = RunnableBranch( - ( - lambda x: x["avoid"], - RunnableLambda(lambda x: None), - ), - ( - lambda x: not x["avoid"], - web_answer_chain | itemgetter("web_results"), - ), - RunnableLambda(lambda x: None), - ) - - return ( - RunnableParallel( - question=itemgetter(search_field), - search_type=itemgetter("search_type"), - avoid=itemgetter("avoid_query"), - ) - | branch - ) - - -def load_web_search_chain(search_field: str, top_k: int = 5) -> Runnable: - you_search = YouSearch(os.environ["YOU_API_KEY"], top_k) - - web_answer_chain = RunnablePassthrough().assign( - web_results=lambda x: you_search( - question=x["question"], search_type=x["search_type"] - ) - ) - - branch = RunnableBranch( - ( - lambda x: x["avoid"], - RunnableLambda(lambda x: None), - ), - ( - lambda x: not x["avoid"], - web_answer_chain | itemgetter("web_results"), - ), - RunnableLambda(lambda x: None), - ) - - return ( - RunnableParallel( - question=itemgetter(search_field), - search_type=itemgetter("search_type"), - avoid=itemgetter("avoid_query"), - ) - | branch - ) - - -def load_web_answer_enhancement_chain(top_k: int = 5) -> Runnable: - web_answer_chain = load_web_answer_chain( - search_field="standalone_question", top_k=top_k - ) - - return ( - RunnablePassthrough().assign( - search_type=lambda x: "rag", - ) - | web_answer_chain - ) +class YouWebRagSearchEnhancer: + def __init__(self): + self.you_search = YouSearch() + self._chain = None + @property + def chain(self) -> Runnable: + if self._chain is None: + search_chain = RunnablePassthrough().assign( + web_results=lambda x: self.you_search(question=x["question"]) + ) -def load_web_search_enhancement_chain(top_k: int = 5) -> Runnable: - web_answer_chain = load_web_answer_chain( - search_field="standalone_question", top_k=top_k - ) + branch = RunnableBranch( + ( + lambda x: x["avoid"], + RunnableLambda(lambda x: None), + ), + ( + lambda x: not x["avoid"], + search_chain | itemgetter("web_results"), + ), + RunnableLambda(lambda x: None), + ) - return ( - RunnablePassthrough().assign(search_type=lambda x: "rag") - | web_answer_chain - ) + self._chain = ( + RunnableParallel( + question=itemgetter("standalone_question"), + avoid=itemgetter("avoid_query"), + ) + | branch + ) + return self._chain diff --git a/src/wandbot/rag/response_synthesis/__init__.py b/src/wandbot/rag/response_synthesis/__init__.py new file mode 100644 index 0000000..3aa3c03 --- /dev/null +++ b/src/wandbot/rag/response_synthesis/__init__.py @@ -0,0 +1,3 @@ +from .response_synthesis import ResponseSynthesizer + +__all__ = ["ResponseSynthesizer"] diff --git a/src/wandbot/chat/response_synthesis.py b/src/wandbot/rag/response_synthesis/response_synthesis.py similarity index 83% rename from src/wandbot/chat/response_synthesis.py rename to src/wandbot/rag/response_synthesis/response_synthesis.py index 02d3dc4..655f0a0 100644 --- a/src/wandbot/chat/response_synthesis.py +++ b/src/wandbot/rag/response_synthesis/response_synthesis.py @@ -1,36 +1,10 @@ from operator import itemgetter -from langchain_core.documents import Document from langchain_core.output_parsers import StrOutputParser -from langchain_core.prompts import ( - ChatPromptTemplate, - PromptTemplate, - format_document, -) +from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel -from wandbot.retriever.fusion import combine_documents -from wandbot.utils import clean_document_content - -DEFAULT_QUESTION_PROMPT = PromptTemplate.from_template( - template="{page_content}\nlanguage: {language}\nintents: {intents}" -) - - -def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): - page_content = enhanced_query["standalone_question"] - metadata = { - "language": enhanced_query["language"], - "intents": ( - enhanced_query["intents"] - if enhanced_query["language"] == "en" - else None - ), - } - doc = Document(page_content=page_content, metadata=metadata) - doc = clean_document_content(doc) - doc_string = format_document(doc, document_prompt) - return doc_string - +from langchain_openai import ChatOpenAI +from wandbot.rag.utils import ChatModel, combine_documents, create_query_str RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """As Wandbot - a support expert in Weights & Biases, wandb and weave. Your goal to ensure customer success with questions related to Weight & Biases, `wandb`, and the visualization library `weave` @@ -157,30 +131,54 @@ def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): # ) -def load_response_synthesizer_chain(model) -> Runnable: - response_prompt = ChatPromptTemplate.from_messages( - RESPONSE_SYNTHESIS_PROMPT_MESSAGES - ) +class ResponseSynthesizer: + model: ChatModel = ChatModel() + fallback_model: ChatModel = ChatModel(max_retries=6) - response_synthesis_chain = ( - RunnableLambda( - lambda x: { - "query_str": create_query_str(x["query"]), - "context_str": combine_documents(x["context"]), - } - ) - | RunnableParallel( - query_str=itemgetter("query_str"), - context_str=itemgetter("context_str"), - response_prompt=response_prompt, + def __init__( + self, + model: str = "gpt-4-0125-preview", + fallback_model: str = "gpt-3.5-turbo-1106", + ): + self.model = model + self.fallback_model = fallback_model + self.prompt = ChatPromptTemplate.from_messages( + RESPONSE_SYNTHESIS_PROMPT_MESSAGES ) - | RunnableParallel( - query_str=itemgetter("query_str"), - context_str=itemgetter("context_str"), - response_prompt=itemgetter("response_prompt"), - response=itemgetter("response_prompt") | model | StrOutputParser(), - response_model=RunnableLambda(lambda x: model.model_name), + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + base_chain = self._load_chain(self.model) + fallback_chain = self._load_chain(self.fallback_model) + self._chain = base_chain.with_fallbacks([fallback_chain]) + return self._chain + + def _load_chain(self, model: ChatOpenAI) -> Runnable: + response_synthesis_chain = ( + RunnableLambda( + lambda x: { + "query_str": create_query_str(x["query"]), + "context_str": combine_documents(x["context"]), + } + ) + | RunnableParallel( + query_str=itemgetter("query_str"), + context_str=itemgetter("context_str"), + response_prompt=self.prompt, + ) + | RunnableParallel( + query_str=itemgetter("query_str"), + context_str=itemgetter("context_str"), + response_prompt=RunnableLambda( + lambda x: x["response_prompt"].to_string() + ), + response=itemgetter("response_prompt") + | model + | StrOutputParser(), + response_model=RunnableLambda(lambda x: model.model_name), + ) ) - ) - return response_synthesis_chain + return response_synthesis_chain diff --git a/src/wandbot/rag/retrieval/__init__.py b/src/wandbot/rag/retrieval/__init__.py new file mode 100644 index 0000000..8c85a86 --- /dev/null +++ b/src/wandbot/rag/retrieval/__init__.py @@ -0,0 +1,3 @@ +from .fusion import FusionRetrieval + +__all__ = ["FusionRetrieval"] diff --git a/src/wandbot/rag/retrieval/fusion.py b/src/wandbot/rag/retrieval/fusion.py new file mode 100644 index 0000000..ecad4c5 --- /dev/null +++ b/src/wandbot/rag/retrieval/fusion.py @@ -0,0 +1,184 @@ +from operator import itemgetter + +from langchain.load import dumps, loads +from langchain.retrievers.document_compressors import CohereRerank +from langchain_community.document_transformers import EmbeddingsRedundantFilter +from langchain_core.runnables import ( + Runnable, + RunnableBranch, + RunnableLambda, + RunnableParallel, + RunnablePassthrough, +) +from wandbot.ingestion.config import VectorStoreConfig +from wandbot.rag.utils import get_web_contexts, process_input_for_retrieval +from wandbot.retriever import OpenAIEmbeddingsModel, VectorStore + + +def reciprocal_rank_fusion(results: list[list], k=60): + fused_scores = {} + for docs in results: + # Assumes the docs are returned in sorted order of relevance + for rank, doc in enumerate(docs): + doc_str = dumps(doc) + if doc_str not in fused_scores: + fused_scores[doc_str] = 0 + previous_score = fused_scores[doc_str] + fused_scores[doc_str] += 1 / (rank + k) + + ranked_results = [ + (loads(doc), score) + for doc, score in sorted( + fused_scores.items(), key=lambda x: x[1], reverse=True + ) + ] + return [item[0] for item in ranked_results] + + +class SimpleRetrievalChain: + def __init__(self, field: str = "question"): + self.field = field + + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = "_" + name + + def __get__(self, obj, obj_type=None): + if getattr(obj, "retriever") is None: + raise AttributeError( + "Retriever must be set before setting retrieval chain" + ) + default_input_chain = ( + itemgetter("standalone_question") + | RunnablePassthrough() + | process_input_for_retrieval + | RunnableParallel(context=obj.retriever) + | itemgetter("context") + ) + + input_chain = ( + itemgetter(self.field) + | RunnablePassthrough() + | process_input_for_retrieval + | RunnableParallel(context=obj.retriever) + | itemgetter("context") + ) + + retrieval_chain = RunnableBranch( + ( + lambda x: not x["avoid_query"], + input_chain, + ), + ( + lambda x: x["avoid_query"], + default_input_chain, + ), + default_input_chain, + ) + return retrieval_chain + + +class CohereRerankChain: + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = "_" + name + + def __get__(self, obj, obj_type=None): + if getattr(obj, "top_k") is None: + raise AttributeError( + "Top k must be set before using retrieval chain" + ) + + def load_rerank_chain(language): + if language == "en": + cohere_rerank = CohereRerank( + top_n=obj.top_k, model="rerank-english-v2.0" + ) + else: + cohere_rerank = CohereRerank( + top_n=obj.top_k, model="rerank-multilingual-v2.0" + ) + + return lambda x: cohere_rerank.compress_documents( + documents=x["context"], query=x["question"] + ) + + cohere_rerank = RunnableBranch( + ( + lambda x: x["language"] == "en", + load_rerank_chain("en"), + ), + ( + lambda x: x["language"], + load_rerank_chain("ja"), + ), + load_rerank_chain("ja"), + ) + + return cohere_rerank + + +class FusionRetrieval: + question_chain = SimpleRetrievalChain("question") + standalone_question_chain = SimpleRetrievalChain("standalone_question") + keywords_chain = SimpleRetrievalChain("keywords") + vector_search_chain = SimpleRetrievalChain("vector_search") + web_context_chain = RunnableLambda( + lambda x: get_web_contexts(x["web_results"]) + ) + cohere_rerank_chain = CohereRerankChain() + embeddings_model: OpenAIEmbeddingsModel = OpenAIEmbeddingsModel( + dimensions=768 + ) + + def __init__( + self, + vector_store_config: VectorStoreConfig, + top_k=5, + search_type="mmr", + ): + self.vector_store = VectorStore.from_config(vector_store_config) + + self.retriever = self.vector_store.as_parent_retriever( + search_type=search_type, search_kwargs={"k": top_k * 4} + ) + self.embeddings_model = vector_store_config.embeddings_model + self.top_k = top_k + self.redundant_filter = EmbeddingsRedundantFilter( + embeddings=self.embeddings_model + ).transform_documents + + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + combined_retrieval_chain = ( + RunnableParallel( + question=self.question_chain, + standalone_question=self.standalone_question_chain, + keywords=self.keywords_chain, + vector_search=self.vector_search_chain, + web_context=self.web_context_chain, + ) + | itemgetter( + "question", + "standalone_question", + "keywords", + "vector_search", + "web_context", + ) + | reciprocal_rank_fusion + | self.redundant_filter + ) + + self._chain = ( + RunnableParallel( + context=combined_retrieval_chain, + question=itemgetter("question"), + language=itemgetter("language"), + ) + | self.cohere_rerank_chain + ) + + return self._chain diff --git a/src/wandbot/rag/utils.py b/src/wandbot/rag/utils.py new file mode 100644 index 0000000..66f2026 --- /dev/null +++ b/src/wandbot/rag/utils.py @@ -0,0 +1,105 @@ +import json + +from langchain_core.documents import Document +from langchain_core.prompts import PromptTemplate, format_document +from langchain_openai import ChatOpenAI +from wandbot.utils import clean_document_content + + +class ChatModel: + def __init__(self, temperature: float = 0.1, max_retries: int = 2): + self.temperature = temperature + self.max_retries = max_retries + + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = "_" + name + + def __get__(self, obj, obj_type=None): + value = getattr(obj, self.private_name) + return value + + def __set__(self, obj, value): + model = ChatOpenAI( + model_name=value, + temperature=self.temperature, + max_retries=self.max_retries, + ) + setattr(obj, self.private_name, model) + + +DEFAULT_QUESTION_PROMPT = PromptTemplate.from_template( + template="{page_content}\nlanguage: {language}\nintents: {intents}" +) + + +def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): + page_content = enhanced_query["standalone_question"] + metadata = { + "language": enhanced_query["language"], + "intents": ( + enhanced_query["intents"] + if enhanced_query["language"] == "en" + else None + ), + } + doc = Document(page_content=page_content, metadata=metadata) + doc = clean_document_content(doc) + doc_string = format_document(doc, document_prompt) + return doc_string + + +DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template( + template="source: {source}\nsource_type: {source_type}\nhas_code: {has_code}\n\n{page_content}" +) + + +def combine_documents( + docs, + document_prompt=DEFAULT_DOCUMENT_PROMPT, + document_separator="\n\n---\n\n", +): + cleaned_docs = [clean_document_content(doc) for doc in docs] + doc_strings = [ + format_document(doc, document_prompt) for doc in cleaned_docs + ] + return document_separator.join(doc_strings) + + +def process_input_for_retrieval(retrieval_input): + if isinstance(retrieval_input, list): + retrieval_input = "\n".join(retrieval_input) + elif isinstance(retrieval_input, dict): + retrieval_input = json.dumps(retrieval_input) + elif not isinstance(retrieval_input, str): + retrieval_input = str(retrieval_input) + return retrieval_input + + +def get_web_contexts(web_results): + output_documents = [] + if not web_results: + return [] + # web_answer = web_results["web_answer"] + # if web_answer: + # output_documents += [ + # Document( + # page_content=web_answer, + # metadata={ + # "source": "you.com", + # "source_type": "web_answer", + # "has_code": None, + # }, + # ) + # ] + return ( + output_documents + + [ + Document( + page_content=document["context"], metadata=document["metadata"] + ) + for document in web_results["web_context"] + ] + if web_results.get("web_context") + else [] + ) diff --git a/src/wandbot/retriever/__init__.py b/src/wandbot/retriever/__init__.py index e69de29..5b48911 100644 --- a/src/wandbot/retriever/__init__.py +++ b/src/wandbot/retriever/__init__.py @@ -0,0 +1,4 @@ +from .base import VectorStore +from .utils import OpenAIEmbeddingsModel + +__all__ = ["VectorStore", "OpenAIEmbeddingsModel"] diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index 5b77af2..e3566a3 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,35 +1,69 @@ -from langchain_community.vectorstores.chroma import Chroma -from langchain_openai import OpenAIEmbeddings - import wandb +from langchain_community.vectorstores.chroma import Chroma +from langchain_core.documents import Document +from langchain_core.runnables import RunnableLambda from wandbot.ingestion.config import VectorStoreConfig +from wandbot.retriever.utils import OpenAIEmbeddingsModel -def load_vector_store_from_config(config: VectorStoreConfig): - embedding_fn = OpenAIEmbeddings( - model=config.embeddings_model, dimensions=config.embedding_dim +class VectorStore: + embeddings_model: OpenAIEmbeddingsModel = OpenAIEmbeddingsModel( + dimensions=512 ) - base_vectorstore = Chroma( - collection_name=config.name, - embedding_function=embedding_fn, - persist_directory=str(config.persist_dir), - ) - return base_vectorstore + def __init__( + self, embeddings_model: str, collection_name: str, persist_dir: str + ): + self.embeddings_model = embeddings_model + self.vectorstore = Chroma( + collection_name=collection_name, + embedding_function=self.embeddings_model, + persist_directory=persist_dir, + ) + @classmethod + def from_config(cls, config: VectorStoreConfig): + if config.persist_dir.exists(): + return cls( + embeddings_model=config.embeddings_model, + collection_name=config.name, + persist_dir=str(config.persist_dir), + ) + if wandb.run is None: + api = wandb.Api() + artifact = api.artifact(config.artifact_url) + else: + artifact = wandb.run.use_artifact(config.artifact_url) + _ = artifact.download(root=str(config.persist_dir)) -def load_vector_store_from_artifact(artifact_url: str): - artifact = wandb.run.use_artifact(artifact_url) - artifact_dir = artifact.download() - config = VectorStoreConfig(persist_dir=artifact_dir) - base_vectorstore = load_vector_store_from_config(config) - return base_vectorstore + return cls( + embeddings_model=config.embeddings_model, + collection_name=config.name, + persist_dir=str(config.persist_dir), + ) + def as_retriever(self, search_type="mmr", search_kwargs=None): + if search_kwargs is None: + search_kwargs = {"k": 5} + return self.vectorstore.as_retriever( + search_type=search_type, search_kwargs=search_kwargs + ) -def load_retriever_with_options( - base_vectorstore, search_type="mmr", search_kwargs={"k": 5} -): - base_retriever = base_vectorstore.as_retriever( - search_type=search_type, search_kwargs=search_kwargs - ) - return base_retriever + def as_parent_retriever(self, search_type="mmr", search_kwargs=None): + if search_kwargs is None: + search_kwargs = {"k": 5} + retriever = self.vectorstore.as_retriever( + search_type=search_type, search_kwargs=search_kwargs + ) + parent_retriever = retriever | RunnableLambda( + lambda docs: [ + Document( + page_content=doc.metadata.get( + "source_content", doc.page_content + ), + metadata=doc.metadata, + ) + for doc in docs + ] + ) + return parent_retriever diff --git a/src/wandbot/retriever/external.py b/src/wandbot/retriever/external.py deleted file mode 100644 index 552103a..0000000 --- a/src/wandbot/retriever/external.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -from typing import List, Optional - -import requests -from llama_index import QueryBundle -from llama_index.callbacks import CallbackManager -from llama_index.core.base_retriever import BaseRetriever -from llama_index.schema import NodeWithScore, TextNode - -from wandbot.utils import get_logger - -logger = get_logger(__name__) - - -class YouRetriever(BaseRetriever): - """You retriever.""" - - def __init__( - self, - api_key: Optional[str] = None, - similarity_top_k: int = 10, - callback_manager: Optional[CallbackManager] = None, - ) -> None: - """Init params.""" - self._api_key = api_key or os.environ["YOU_API_KEY"] - self.similarity_top_k = ( - similarity_top_k if similarity_top_k <= 20 else 20 - ) - super().__init__(callback_manager) - - def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]: - """Retrieve.""" - try: - headers = {"X-API-Key": self._api_key} - url = "https://api.ydc-index.io/search" - - querystring = { - "query": "Weights & Biases, W&B, wandb or Weave " - + query_bundle.query_str, - "num_web_results": self.similarity_top_k, - } - response = requests.get(url, headers=headers, params=querystring) - if response.status_code != 200: - return [] - else: - results = response.json() - - snippets = [hit["snippets"] for hit in results["hits"]] - snippet_metadata = [ - { - "source": hit["url"], - "language": "en", - "description": hit["description"], - "title": hit["title"], - "tags": ["you.com"], - } - for hit in results["hits"] - ] - search_hits = [] - for snippet_list, metadata in zip(snippets, snippet_metadata): - for snippet in snippet_list: - search_hits.append((snippet, metadata)) - - return [ - NodeWithScore( - node=TextNode(text=s[0], metadata=s[1]), - score=1.0, - ) - for s in search_hits - ] - except Exception as e: - return [] diff --git a/src/wandbot/retriever/fusion.py b/src/wandbot/retriever/fusion.py deleted file mode 100644 index 175220e..0000000 --- a/src/wandbot/retriever/fusion.py +++ /dev/null @@ -1,203 +0,0 @@ -import json -from operator import itemgetter - -from langchain.load import dumps, loads -from langchain.prompts.prompt import PromptTemplate -from langchain.retrievers.document_compressors import CohereRerank -from langchain.schema import Document, format_document -from langchain_community.document_transformers import EmbeddingsRedundantFilter -from langchain_core.runnables import ( - RunnableBranch, - RunnableLambda, - RunnableParallel, - RunnablePassthrough, -) - -from wandbot.utils import clean_document_content - -DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template( - template="source: {source}\nsource_type: {source_type}\nhas_code: {has_code}\n\n{page_content}" -) - - -def combine_documents( - docs, - document_prompt=DEFAULT_DOCUMENT_PROMPT, - document_separator="\n\n---\n\n", -): - cleaned_docs = [clean_document_content(doc) for doc in docs] - doc_strings = [ - format_document(doc, document_prompt) for doc in cleaned_docs - ] - return document_separator.join(doc_strings) - - -def process_input_for_retrieval(retrieval_input): - if isinstance(retrieval_input, list): - retrieval_input = "\n".join(retrieval_input) - elif isinstance(retrieval_input, dict): - retrieval_input = json.dumps(retrieval_input) - elif not isinstance(retrieval_input, str): - retrieval_input = str(retrieval_input) - return retrieval_input - - -def load_simple_retrieval_chain(retriever, input_key): - default_input_chain = ( - itemgetter("standalone_question") - | RunnablePassthrough() - | process_input_for_retrieval - | RunnableParallel(context=retriever) - | itemgetter("context") - ) - - input_chain = ( - itemgetter(input_key) - | RunnablePassthrough() - | process_input_for_retrieval - | RunnableParallel(context=retriever) - | itemgetter("context") - ) - - retrieval_chain = RunnableBranch( - ( - lambda x: not x["avoid_query"], - input_chain, - ), - ( - lambda x: x["avoid_query"], - default_input_chain, - ), - default_input_chain, - ) - - return retrieval_chain - - -def reciprocal_rank_fusion(results: list[list], k=60): - fused_scores = {} - for docs in results: - # Assumes the docs are returned in sorted order of relevance - for rank, doc in enumerate(docs): - doc_str = dumps(doc) - if doc_str not in fused_scores: - fused_scores[doc_str] = 0 - previous_score = fused_scores[doc_str] - fused_scores[doc_str] += 1 / (rank + k) - - ranked_results = [ - (loads(doc), score) - for doc, score in sorted( - fused_scores.items(), key=lambda x: x[1], reverse=True - ) - ] - return [item[0] for item in ranked_results] - - -def load_cohere_rerank_chain(top_k=5): - def load_rerank_chain(language): - if language == "en": - cohere_rerank = CohereRerank( - top_n=top_k, model="rerank-english-v2.0" - ) - else: - cohere_rerank = CohereRerank( - top_n=top_k, model="rerank-multilingual-v2.0" - ) - - return lambda x: cohere_rerank.compress_documents( - documents=x["context"], query=x["question"] - ) - - cohere_rerank = RunnableBranch( - ( - lambda x: x["language"] == "en", - load_rerank_chain("en"), - ), - ( - lambda x: x["language"], - load_rerank_chain("ja"), - ), - load_rerank_chain("ja"), - ) - - return cohere_rerank - - -def get_web_contexts(web_results): - output_documents = [] - if not web_results: - return [] - web_answer = web_results["web_answer"] - # if web_answer: - # output_documents += [ - # Document( - # page_content=web_answer, - # metadata={ - # "source": "you.com", - # "source_type": "web_answer", - # "has_code": None, - # }, - # ) - # ] - return ( - output_documents - + [ - Document( - page_content=document["context"], metadata=document["metadata"] - ) - for document in web_results["web_context"] - ] - if web_results.get("web_context") - else [] - ) - - -def load_fusion_retriever_chain(base_retriever, embeddings, top_k=5): - query_retrieval_chain = load_simple_retrieval_chain( - base_retriever, "question" - ) - standalone_query_retrieval_chain = load_simple_retrieval_chain( - base_retriever, "standalone_question" - ) - keywords_retrieval_chain = load_simple_retrieval_chain( - base_retriever, "keywords" - ) - vector_search_retrieval_chain = load_simple_retrieval_chain( - base_retriever, "vector_search" - ) - - redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings) - - combined_retrieval_chain = ( - RunnableParallel( - question=query_retrieval_chain, - standalone_question=standalone_query_retrieval_chain, - keywords=keywords_retrieval_chain, - vector_search=vector_search_retrieval_chain, - web_context=RunnableLambda( - lambda x: get_web_contexts(x["web_results"]) - ), - ) - | itemgetter( - "question", - "standalone_question", - "keywords", - "vector_search", - "web_context", - ) - | reciprocal_rank_fusion - | redundant_filter.transform_documents - ) - - cohere_rerank_chain = load_cohere_rerank_chain(top_k=top_k) - - ranked_retrieval_chain = ( - RunnableParallel( - context=combined_retrieval_chain, - question=itemgetter("question"), - language=itemgetter("language"), - ) - | cohere_rerank_chain - ) - return ranked_retrieval_chain diff --git a/src/wandbot/retriever/postprocessors.py b/src/wandbot/retriever/postprocessors.py deleted file mode 100644 index 6cb34a3..0000000 --- a/src/wandbot/retriever/postprocessors.py +++ /dev/null @@ -1,84 +0,0 @@ -from typing import List, Optional - -from llama_index import QueryBundle -from llama_index.postprocessor import BaseNodePostprocessor -from llama_index.schema import NodeWithScore - -from wandbot.utils import create_no_result_dummy_node, get_logger - -logger = get_logger(__name__) - - -class LanguageFilterPostprocessor(BaseNodePostprocessor): - """Language-based Node processor.""" - - languages: List[str] = ["en", "python"] - min_result_size: int = 10 - - @classmethod - def class_name(cls) -> str: - return "LanguageFilterPostprocessor" - - def _postprocess_nodes( - self, - nodes: List[NodeWithScore], - query_bundle: Optional[QueryBundle] = None, - ) -> List[NodeWithScore]: - """Postprocess nodes.""" - - new_nodes = [] - for node in nodes: - if node.metadata["language"] in self.languages: - new_nodes.append(node) - - if len(new_nodes) < self.min_result_size: - return new_nodes + nodes[: self.min_result_size - len(new_nodes)] - - return new_nodes - - -class MetadataPostprocessor(BaseNodePostprocessor): - """Metadata-based Node processor.""" - - min_result_size: int = 10 - include_tags: List[str] | None = None - exclude_tags: List[str] | None = None - - @classmethod - def class_name(cls) -> str: - return "MetadataPostprocessor" - - def _postprocess_nodes( - self, - nodes: List[NodeWithScore], - query_bundle: Optional[QueryBundle] = None, - ) -> List[NodeWithScore]: - """Postprocess nodes.""" - if not self.include_tags and not self.exclude_tags: - return nodes - new_nodes = [] - for node in nodes: - normalized_tags = [ - tag.lower().strip() for tag in node.metadata["tags"] - ] - if self.include_tags: - normalized_include_tags = [ - tag.lower().strip() for tag in self.include_tags - ] - if not set(normalized_include_tags).issubset( - set(normalized_tags) - ): - continue - if self.exclude_tags: - normalized_exclude_tags = [ - tag.lower().strip() for tag in self.exclude_tags - ] - if set(normalized_exclude_tags).issubset(set(normalized_tags)): - continue - new_nodes.append(node) - if len(new_nodes) < self.min_result_size: - dummy_node = create_no_result_dummy_node() - new_nodes.extend( - [dummy_node] * (self.min_result_size - len(new_nodes)) - ) - return new_nodes diff --git a/src/wandbot/retriever/utils.py b/src/wandbot/retriever/utils.py new file mode 100644 index 0000000..a4abc14 --- /dev/null +++ b/src/wandbot/retriever/utils.py @@ -0,0 +1,21 @@ +from langchain_openai import OpenAIEmbeddings + + +class OpenAIEmbeddingsModel: + def __init__(self, dimensions: int): + self.dimensions = dimensions + + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = "_" + name + + def __get__(self, obj, obj_type=None): + value = getattr(obj, self.private_name) + return value + + def __set__(self, obj, value): + model = OpenAIEmbeddings( + model=value, + dimensions=self.dimensions, + ) + setattr(obj, self.private_name, model) diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 96e5fdf..b277df7 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -36,6 +36,7 @@ import fasttext import nest_asyncio import tiktoken +import wandb from langchain_core.documents import Document from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.embeddings import OpenAIEmbedding @@ -43,9 +44,8 @@ from llama_index.llms.llm import LLM from llama_index.schema import NodeWithScore, TextNode from llama_index.vector_stores import ChromaVectorStore -from pydantic_settings import BaseSettings - -import wandb +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict def get_logger(name: str) -> logging.Logger: @@ -296,13 +296,24 @@ def create_no_result_dummy_node() -> NodeWithScore: class FasttextModelConfig(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="allow" + ) fasttext_file_path: pathlib.Path = pathlib.Path( "data/cache/models/lid.176.bin" ) - fasttext_artifact_name: str = ( - "wandbot/wandbot_public/fasttext-lid.176.bin:v0" + fasttext_artifact_path: str = Field( + "wandbot/wandbot_public/fasttext-lid.176.bin:v0", + env="LANGDETECT_ARTIFACT_PATH", + validation_alias="langdetect_artifact_path", ) fasttext_artifact_type: str = "fasttext-model" + wandb_project: str = Field( + "wandbot-dev", env="WANDB_PROJECT", validation_alias="wandb_project" + ) + wandb_entity: str = Field( + "wandbot", env="WANDB_ENTITY", validation_alias="wandb_entity" + ) class FastTextLangDetect: @@ -330,10 +341,17 @@ def model(self): def _load_model(self): if not os.path.isfile(self.config.fasttext_file_path): - _ = wandb.run.use_artifact( - self.config.fasttext_artifact_name, - type=self.config.fasttext_artifact_type, - ).download(root=str(self.config.fasttext_file_path.parent)) + if wandb.run is None: + api = wandb.Api() + artifact = api.artifact(self.config.fasttext_artifact_path) + else: + artifact = wandb.run.use_artifact( + self.config.fasttext_artifact_path, + type=self.config.fasttext_artifact_type, + ) + _ = artifact.download( + root=str(self.config.fasttext_file_path.parent) + ) self._model = fasttext.load_model(str(self.config.fasttext_file_path)) return self._model From 0f7484107158b463ea1a3f8ac0a9d4a4dc68597f Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Wed, 14 Feb 2024 21:45:37 +0530 Subject: [PATCH 27/41] chore: run linters and formatters --- src/wandbot/api/app.py | 3 ++- src/wandbot/api/routers/retrieve.py | 1 + src/wandbot/chat/chat.py | 5 +++-- src/wandbot/chat/rag.py | 1 + src/wandbot/ingestion/config.py | 1 + src/wandbot/rag/query_handler/history_handler.py | 1 + src/wandbot/rag/query_handler/intents_enhancer.py | 1 + src/wandbot/rag/query_handler/keyword_search_enhancer.py | 1 + src/wandbot/rag/query_handler/language_detection.py | 1 + src/wandbot/rag/query_handler/query_enhancer.py | 1 + src/wandbot/rag/query_handler/vector_search_enhancer.py | 1 + src/wandbot/rag/response_synthesis/response_synthesis.py | 1 + src/wandbot/rag/retrieval/fusion.py | 1 + src/wandbot/rag/utils.py | 1 + src/wandbot/retriever/base.py | 3 ++- src/wandbot/utils.py | 3 ++- 16 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/wandbot/api/app.py b/src/wandbot/api/app.py index 571583a..ddd3a6e 100644 --- a/src/wandbot/api/app.py +++ b/src/wandbot/api/app.py @@ -33,8 +33,9 @@ from datetime import datetime, timezone import pandas as pd -import wandb from fastapi import FastAPI + +import wandb from wandbot.api.routers import chat as chat_router from wandbot.api.routers import database as database_router diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py index 8481dcf..3e8f0c6 100644 --- a/src/wandbot/api/routers/retrieve.py +++ b/src/wandbot/api/routers/retrieve.py @@ -3,6 +3,7 @@ from fastapi import APIRouter from pydantic import BaseModel from starlette import status + from wandbot.retriever.base import Retriever router = APIRouter( diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 55f40cc..1fda372 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -25,14 +25,15 @@ print(f"Time taken: {response.time_taken}") """ -import wandb from langchain_community.callbacks import get_openai_callback +from weave.monitoring import StreamTable + +import wandb from wandbot.chat.config import ChatConfig from wandbot.chat.rag import Pipeline from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import Timer, get_logger -from weave.monitoring import StreamTable logger = get_logger(__name__) diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 156326f..204d9e2 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -1,6 +1,7 @@ from typing import List, Tuple from langchain_community.callbacks import get_openai_callback + from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag import FusionRetrieval, QueryEnhancer, ResponseSynthesizer from wandbot.utils import Timer diff --git a/src/wandbot/ingestion/config.py b/src/wandbot/ingestion/config.py index aa280c6..771d092 100644 --- a/src/wandbot/ingestion/config.py +++ b/src/wandbot/ingestion/config.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, Field, model_validator from pydantic_settings import BaseSettings + from wandbot.utils import get_logger logger = get_logger(__name__) diff --git a/src/wandbot/rag/query_handler/history_handler.py b/src/wandbot/rag/query_handler/history_handler.py index ec6d749..127cbad 100644 --- a/src/wandbot/rag/query_handler/history_handler.py +++ b/src/wandbot/rag/query_handler/history_handler.py @@ -10,6 +10,7 @@ RunnablePassthrough, ) from langchain_openai import ChatOpenAI + from wandbot.rag.utils import ChatModel CONDENSE_PROMPT_SYSTEM_TEMPLATE = """Given the following conversation and a follow up question, rephrase the follow up \ diff --git a/src/wandbot/rag/query_handler/intents_enhancer.py b/src/wandbot/rag/query_handler/intents_enhancer.py index 9454eac..3df41ba 100644 --- a/src/wandbot/rag/query_handler/intents_enhancer.py +++ b/src/wandbot/rag/query_handler/intents_enhancer.py @@ -14,6 +14,7 @@ from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field from pydantic_settings import BaseSettings, SettingsConfigDict + from wandbot.rag.utils import ChatModel diff --git a/src/wandbot/rag/query_handler/keyword_search_enhancer.py b/src/wandbot/rag/query_handler/keyword_search_enhancer.py index b7f7937..d8820a8 100644 --- a/src/wandbot/rag/query_handler/keyword_search_enhancer.py +++ b/src/wandbot/rag/query_handler/keyword_search_enhancer.py @@ -11,6 +11,7 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field + from wandbot.rag.utils import ChatModel KEYWORDS_SYSTEM_PROMPT = ( diff --git a/src/wandbot/rag/query_handler/language_detection.py b/src/wandbot/rag/query_handler/language_detection.py index 65cac93..fdfaf6a 100644 --- a/src/wandbot/rag/query_handler/language_detection.py +++ b/src/wandbot/rag/query_handler/language_detection.py @@ -1,6 +1,7 @@ from operator import itemgetter from langchain_core.runnables import Runnable, RunnablePassthrough + from wandbot.utils import FastTextLangDetect, FasttextModelConfig diff --git a/src/wandbot/rag/query_handler/query_enhancer.py b/src/wandbot/rag/query_handler/query_enhancer.py index b98b50b..77767d6 100644 --- a/src/wandbot/rag/query_handler/query_enhancer.py +++ b/src/wandbot/rag/query_handler/query_enhancer.py @@ -8,6 +8,7 @@ RunnableParallel, RunnablePassthrough, ) + from wandbot.rag.query_handler.history_handler import CondenseQuestion from wandbot.rag.query_handler.intents_enhancer import IntentsEnhancer from wandbot.rag.query_handler.keyword_search_enhancer import KeywordsEnhancer diff --git a/src/wandbot/rag/query_handler/vector_search_enhancer.py b/src/wandbot/rag/query_handler/vector_search_enhancer.py index d10c21b..3171752 100644 --- a/src/wandbot/rag/query_handler/vector_search_enhancer.py +++ b/src/wandbot/rag/query_handler/vector_search_enhancer.py @@ -10,6 +10,7 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field + from wandbot.rag.utils import ChatModel QUERY_REWRITE_SYSTEM_PROMPT = ( diff --git a/src/wandbot/rag/response_synthesis/response_synthesis.py b/src/wandbot/rag/response_synthesis/response_synthesis.py index 655f0a0..063bfea 100644 --- a/src/wandbot/rag/response_synthesis/response_synthesis.py +++ b/src/wandbot/rag/response_synthesis/response_synthesis.py @@ -4,6 +4,7 @@ from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel from langchain_openai import ChatOpenAI + from wandbot.rag.utils import ChatModel, combine_documents, create_query_str RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """As Wandbot - a support expert in Weights & Biases, wandb and weave. diff --git a/src/wandbot/rag/retrieval/fusion.py b/src/wandbot/rag/retrieval/fusion.py index ecad4c5..7c4c086 100644 --- a/src/wandbot/rag/retrieval/fusion.py +++ b/src/wandbot/rag/retrieval/fusion.py @@ -10,6 +10,7 @@ RunnableParallel, RunnablePassthrough, ) + from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag.utils import get_web_contexts, process_input_for_retrieval from wandbot.retriever import OpenAIEmbeddingsModel, VectorStore diff --git a/src/wandbot/rag/utils.py b/src/wandbot/rag/utils.py index 66f2026..eb25406 100644 --- a/src/wandbot/rag/utils.py +++ b/src/wandbot/rag/utils.py @@ -3,6 +3,7 @@ from langchain_core.documents import Document from langchain_core.prompts import PromptTemplate, format_document from langchain_openai import ChatOpenAI + from wandbot.utils import clean_document_content diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index e3566a3..31fb26a 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,7 +1,8 @@ -import wandb from langchain_community.vectorstores.chroma import Chroma from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda + +import wandb from wandbot.ingestion.config import VectorStoreConfig from wandbot.retriever.utils import OpenAIEmbeddingsModel diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index b277df7..6c11501 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -36,7 +36,6 @@ import fasttext import nest_asyncio import tiktoken -import wandb from langchain_core.documents import Document from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.embeddings import OpenAIEmbedding @@ -47,6 +46,8 @@ from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict +import wandb + def get_logger(name: str) -> logging.Logger: """Creates and returns a logger with the specified name. From 36e56794e2622ff63adc0403c3b112e8ee4a6174 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 15 Feb 2024 12:44:25 +0530 Subject: [PATCH 28/41] feat: re-add retrieval endpoint and handle punctuations better in langdetect --- src/wandbot/api/app.py | 17 +- src/wandbot/api/routers/retrieve.py | 17 +- src/wandbot/chat/query_enhancer.py | 508 ------------------ src/wandbot/rag/__init__.py | 6 +- src/wandbot/rag/query_handler/__init__.py | 3 - .../{query_enhancer.py => base.py} | 1 - .../rag/response_synthesis/__init__.py | 3 - .../{response_synthesis.py => base.py} | 1 - src/wandbot/rag/retrieval/__init__.py | 3 - .../rag/retrieval/{fusion.py => base.py} | 55 +- src/wandbot/retriever/base.py | 95 +++- src/wandbot/retriever/reranking.py | 40 ++ src/wandbot/utils.py | 164 +----- 13 files changed, 169 insertions(+), 744 deletions(-) delete mode 100644 src/wandbot/chat/query_enhancer.py rename src/wandbot/rag/query_handler/{query_enhancer.py => base.py} (99%) rename src/wandbot/rag/response_synthesis/{response_synthesis.py => base.py} (99%) rename src/wandbot/rag/retrieval/{fusion.py => base.py} (73%) create mode 100644 src/wandbot/retriever/reranking.py diff --git a/src/wandbot/api/app.py b/src/wandbot/api/app.py index ddd3a6e..8f2d994 100644 --- a/src/wandbot/api/app.py +++ b/src/wandbot/api/app.py @@ -33,13 +33,12 @@ from datetime import datetime, timezone import pandas as pd -from fastapi import FastAPI - import wandb +from fastapi import FastAPI from wandbot.api.routers import chat as chat_router from wandbot.api.routers import database as database_router - -# from wandbot.api.routers import retrieve as retrieve_router +from wandbot.api.routers import retrieve as retrieve_router +from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import get_logger logger = get_logger(__name__) @@ -58,7 +57,9 @@ async def lifespan(app: FastAPI): """ chat_router.chat = chat_router.Chat(chat_router.chat_config) database_router.db_client = database_router.DatabaseClient() - # retrieve_router.retriever = chat_router.chat.retriever + retrieve_router.retriever = retrieve_router.SimpleRetrievalEngine( + VectorStoreConfig() + ) async def backup_db(): """Periodically backs up the database to a table. @@ -94,12 +95,14 @@ async def backup_db(): wandb.run.finish() -app = FastAPI(name="wandbot", version="1.0.0", lifespan=lifespan) +app = FastAPI( + title="Wandbot", name="wandbot", version="1.3.0", lifespan=lifespan +) app.include_router(chat_router.router) app.include_router(database_router.router) -# app.include_router(retrieve_router.router) +app.include_router(retrieve_router.router) if __name__ == "__main__": diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py index 3e8f0c6..dea9b4f 100644 --- a/src/wandbot/api/routers/retrieve.py +++ b/src/wandbot/api/routers/retrieve.py @@ -3,15 +3,14 @@ from fastapi import APIRouter from pydantic import BaseModel from starlette import status - -from wandbot.retriever.base import Retriever +from wandbot.retriever.base import SimpleRetrievalEngine router = APIRouter( prefix="/retrieve", tags=["retrievers"], ) -retriever: Retriever | None = None +retriever: SimpleRetrievalEngine | None = None class APIRetrievalResult(BaseModel): @@ -28,10 +27,8 @@ class APIRetrievalResponse(BaseModel): class APIRetrievalRequest(BaseModel): query: str language: str = "en" - initial_k: int = 10 top_k: int = 5 - include_tags: List[str] = [] - exclude_tags: List[str] = [] + sources: List[str] | None = None @router.post( @@ -49,14 +46,10 @@ async def retrieve(request: APIRetrievalRequest) -> APIRetrievalResponse: The APIRetrievalResponse object containing the query and top k results. """ results = retriever( - query=request.query, - indices=( - [idx.value for idx in request.indices] if request.indices else None - ), + question=request.query, language=request.language, top_k=request.top_k, - include_tags=request.include_tags, - exclude_tags=request.exclude_tags, + sources=request.sources, ) return APIRetrievalResponse( diff --git a/src/wandbot/chat/query_enhancer.py b/src/wandbot/chat/query_enhancer.py deleted file mode 100644 index 63f54a5..0000000 --- a/src/wandbot/chat/query_enhancer.py +++ /dev/null @@ -1,508 +0,0 @@ -import enum -import json -import os -import re -from typing import List, Optional - -import cohere -import instructor -import openai -import tiktoken -from llama_index.llms import ChatMessage, MessageRole -from llama_index.llms.generic_utils import messages_to_history_str -from openai import OpenAI -from pydantic import BaseModel, Field -from pydantic.v1 import BaseModel as BaseModelV1 -from pydantic_settings import BaseSettings, SettingsConfigDict -from tenacity import retry, stop_after_attempt, wait_random_exponential - -from wandbot.chat.schemas import ChatRequest -from wandbot.database.schemas import QuestionAnswer -from wandbot.utils import FastTextLangDetect, get_logger - -logger = get_logger(__name__) - - -CONDENSE_PROMPT_SYSTEM_TEMPLATE = ( - "Given the following conversation between a user and an AI assistant and a follow up " - "question from user, rephrase the follow up question to be a standalone question. Ensure " - "that the standalone question summarizes the conversation and completes the follow up " - "question with all the necessary context." - + """ -Chat History: - {chat_history} -""" -) -CONDENSE_PROMPT_USER_TEMPLATE = """Follow Up Input: {question} - Standalone question:""" - - -class StandaloneQuestion(BaseModel): - question: str = Field( - ..., - description="A standalone question that summarizes the conversation and completes the follow up question with " - "all the necessary context", - ) - - -class Labels(str, enum.Enum): - UNRELATED = "unrelated" - CODE_TROUBLESHOOTING = "code_troubleshooting" - INTEGRATIONS = "integrations" - PRODUCT_FEATURES = "product_features" - SALES_AND_GTM_RELATED = "sales_and_gtm_related" - BEST_PRACTICES = "best_practices" - COURSE_RELATED = "course_related" - NEEDS_MORE_INFO = "needs_more_info" - OPINION_REQUEST = "opinion_request" - NEFARIOUS_QUERY = "nefarious_query" - OTHER = "other" - - -class MultiLabel(BaseModel): - label: Labels = Field(..., description="The label for the query") - reasoning: str = Field( - ..., - description="The reason for the assigning the label to the query", - ) - - -class SubQuery(BaseModel): - """Correctly resolved subquery from the given query""" - - query: str = Field( - ..., description="Informative sub-query from the given user query. " - ) - - -class Keyword(BaseModel): - """Keyword extracted from the given query to search for relevant articles""" - - keyword: str = Field( - ..., - description="A Keyword that can be used to retrieve for relevant articles required to answer the query", - ) - - -class EnhancedQuery(BaseModel): - """An enhanced query with keywords, intent and sub-queries related to the query""" - - predicted_labels: List[MultiLabel] = Field( - ..., description="The predicted labels for the query" - ) - keywords: List[Keyword] = Field( - ..., - description="List of keywords and key phrases to search for", - min_items=0, - max_items=5, - ) - sub_queries: List[SubQuery] = Field( - ..., - description="List of sub-queries that need be answered to resolve the query", - min_items=0, - max_items=3, - ) - - -INTENT_DESCRIPTIONS = { - Labels.UNRELATED.value: "The query is not related to Weights & Biases", - Labels.CODE_TROUBLESHOOTING.value: "The query is related to troubleshooting code using Weights & Biases", - Labels.INTEGRATIONS.value: "The query is related to integrating Weights & Biases with other tools, frameworks, " - "or libraries", - Labels.PRODUCT_FEATURES.value: "The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, " - "Reports, Experiments, Tables, Prompts, Launch, Weave, StreamTables and more", - Labels.SALES_AND_GTM_RELATED.value: "The query is related to sales, marketing, or other business related topics " - "such as pricing, billing, or partnerships etc", - Labels.BEST_PRACTICES.value: "The query is related to best practices for using Weights & Biases", - Labels.COURSE_RELATED.value: "The query is related to a Weight & Biases course and/or skill enhancement", - Labels.NEEDS_MORE_INFO.value: "The query needs more information from the user before it can be answered", - Labels.OPINION_REQUEST.value: "The query is asking for an opinion", - Labels.NEFARIOUS_QUERY.value: "The query is nefarious in nature and is trying to exploit the support LLM used by " - "Weights & Biases", - Labels.OTHER.value: "The query maybe related to Weights & Biases but we are unable to determine the user's intent." - " It's best to ask the user to rephrase the query or avoid answering the query", -} - - -QUERY_INTENTS = { - Labels.UNRELATED.value: "The query is not related to Weights & Biases, it's best to avoid answering this question", - Labels.CODE_TROUBLESHOOTING.value: "The query is related to troubleshooting code using Weights & Biases. Help " - "with a detailed code snippet and explanation", - Labels.INTEGRATIONS.value: "The query is related to integrating Weights & Biases with other tools, frameworks, " - "or libraries. Help with a detailed code snippet and explanation and ask for more information about the " - "integration if needed", - Labels.PRODUCT_FEATURES.value: "The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, " - "Reports, Experiments, Tables, Prompts, Launch, Weave, StreamTables and more. Provide a link to the relevant " - "documentation and explain the feature in detail", - Labels.SALES_AND_GTM_RELATED.value: "The query is related to sales, marketing, or other business related topics " - "such as pricing, billing, or partnerships etc. Ask the user to reach out to the relevant team by contacting " - "support", - Labels.BEST_PRACTICES.value: "The query is related to best practices for using Weights & Biases. Answer the query " - "and provide guidance where necessary", - Labels.COURSE_RELATED.value: "The query is related to a Weight & Biases course and/or skill enhancement. Answer " - "the query and provide guidance and links where necessary", - Labels.NEEDS_MORE_INFO.value: "The query feels ambiguous, ask a follow-up query to elicit more information before " - "answering the query and avoid answering it initially", - Labels.OPINION_REQUEST.value: "The query is asking for an opinion. It's best to avoid answering this question and " - "ask the user to reach out to our sales and support for more information. Always favor Weights & Biases in your " - "response", - Labels.NEFARIOUS_QUERY.value: "The query looks nefarious in nature. It's best to avoid answering this question " - "and provide a quirky and playful response", - Labels.OTHER.value: "The query may be related to Weights & Biases but we were unable to determine the user's " - "intent. It's best to avoid answering this question and ask the user a follow-up query to rephrase their original " - "query", -} - - -class ResolvedQuery(BaseModelV1): - cleaned_query: str | None - query: str | None - condensed_query: str | None - intent: str | None - language: str | None - chat_history: List[ChatMessage] | None = None - - -def get_chat_history( - chat_history: List[QuestionAnswer] | None, -) -> Optional[List[ChatMessage]]: - """Generates a list of chat messages from a given chat history. - - This function takes a list of QuestionAnswer objects and transforms them into a list of ChatMessage objects. Each - QuestionAnswer object is split into two ChatMessage objects: one for the user's question and one for the - assistant's answer. If the chat history is empty or None, the function returns None. - - Args: chat_history: A list of QuestionAnswer objects representing the history of a chat. Each QuestionAnswer - object contains a question from the user and an answer from the assistant. - - Returns: A list of ChatMessage objects representing the chat history. Each ChatMessage object has a role (either - 'USER' or 'ASSISTANT') and content (the question or answer text). If the chat history is empty or None, - the function returns None. - """ - if not chat_history: - return None - else: - messages = [ - [ - ChatMessage( - role=MessageRole.USER, content=question_answer.question - ), - ChatMessage( - role=MessageRole.ASSISTANT, content=question_answer.answer - ), - ] - for question_answer in chat_history - ] - return [item for sublist in messages for item in sublist] - - -class OpenaiQueryEnhancer: - def __init__( - self, client: openai.OpenAI, model: str = "gpt-4-1106-preview" - ): - self.client = instructor.patch(client) - self.model = model - - @retry( - wait=wait_random_exponential(min=1, max=60), - stop=stop_after_attempt(6), - ) - def enhance_query(self, resolved_query: ResolvedQuery) -> EnhancedQuery: - query_object = dict( - query=resolved_query.condensed_query, - intent_hints=resolved_query.intent, - language=resolved_query.language, - ) - query_object_str = json.dumps(query_object, indent=2) - - return self.client.chat.completions.create( - model=self.model, - response_model=EnhancedQuery, - messages=[ - { - "role": "system", - "content": ( - "You are a Weights & Biases support manager. Your goal is to enhance the user query by adding " - "the following information to the query:\n" - "1. Tag and classify query. Here are the descriptions of the available labels\n" - + "\n".join( - [ - f"{label}: {description}" - for label, description in INTENT_DESCRIPTIONS.items() - ] - ) - + "\n" - "2. A list of keywords and key phrases to search for relevant articles to answer the query.\n" - "3. A list of sub-queries that need be answered to answer the query\n\n" - "**Note**: Do not fill in the keywords and sub-queries if the query is nefarious, " - "opinion related or unrelated to Weights & Biases\n" - ), - }, - { - "role": "user", - "content": f"Enhance the following user query related to Weights & Biases:\n{query_object_str}", - }, - ], - ) # type: ignore - - def condense_question( - self, chat_history: List[ChatMessage], latest_message: str - ) -> StandaloneQuestion: - """Condense a conversation history and latest user message to a standalone question.""" - if not chat_history or len(chat_history) == 0: - return StandaloneQuestion(question=latest_message) - - chat_history_str = messages_to_history_str(chat_history) - logger.debug(chat_history_str) - - return self.client.chat.completions.create( - model=self.model, - response_model=StandaloneQuestion, - messages=[ - { - "role": "system", - "content": CONDENSE_PROMPT_SYSTEM_TEMPLATE.format( - chat_history=chat_history_str, - ), - }, - { - "role": "user", - "content": CONDENSE_PROMPT_USER_TEMPLATE.format( - question=latest_message - ), - }, - ], - ) # type: ignore - - def __call__(self, resolved_query: ResolvedQuery) -> EnhancedQuery: - resolved_query.condensed_query = self.condense_question( - resolved_query.chat_history, resolved_query.cleaned_query - ).question - enhanced_query = self.enhance_query(resolved_query) - logger.debug(f"Openai query enhancements: {enhanced_query}") - return enhanced_query - - -class QueryHandlerConfig(BaseSettings): - default_query_clf_model: str = Field( - ..., - description="The name of the model to use for query classification", - env="DEFAULT_QUERY_CLF_MODEL", - validation_alias="default_query_clf_model", - ) - fallback_query_clf_model: str = Field( - "gpt-3.5-turbo-1106", - description="The name of the fallback model to use for query classification", - ) - tokenizer: str = Field( - "cl100k_base", - description="The name of the tokenizer to use for query classification", - ) - bot_name_pattern: str = Field( - r"<@U[A-Z0-9]+>|@[a-zA-Z0-9\s]+\([a-zA-Z0-9\s]+\)|@[a-zA-Z0-9\s]+", - description="The regex pattern to use for detecting bot names in queries", - ) - - model_config = SettingsConfigDict( - env_file=".env", - env_file_encoding="utf-8", - extra="allow", - ) - - -class CohereQueryClassifier: - def __init__(self, client: cohere.Client, model: str): - self.client = client - self.model = model - - def __call__(self, query: str) -> List[str]: - response = self.client.classify( - model=self.model, - inputs=[query], - ) - logger.debug( - f"Cohere query classifications: {response.classifications}" - ) - return response.classifications[0].predictions - - -class CompleteQuery(BaseModelV1): - initial_query: str = Field( - ..., - description="The initial user query", - ) - cleaned_query: str | None = Field( - ..., - description="The user query cleaned of bot names and other references", - ) - condensed_query: str | None = Field( - ..., - description="The cleaned user query condensed to a standalone question", - ) - intent_hints: str | None = Field( - None, - description="The descriptions of the intents related to the query", - ) - language: str | None = Field( - None, - description="The language of the query", - ) - chat_history: List[ChatMessage] | None = Field( - None, - description="The chat history of the query", - ) - keywords: List[str] | None = Field( - None, - description="The keywords extracted and related to the query", - ) - sub_queries: List[str] | None = Field( - None, - description="The sub-queries extracted from the query", - ) - - -class QueryHandler: - def __init__(self, config: QueryHandlerConfig | None = None): - self.config = ( - config - if isinstance(config, QueryHandlerConfig) - else QueryHandlerConfig() - ) - self.tokenizer = tiktoken.get_encoding(self.config.tokenizer) - self.bot_name_pattern = re.compile(self.config.bot_name_pattern) - self.cohere_client = cohere.Client(os.environ["COHERE_API_KEY"]) - self.lang_client = FastTextLangDetect() - self.openai_client = OpenAI() - self.query_classifier = CohereQueryClassifier( - client=self.cohere_client, model=self.config.default_query_clf_model - ) - self.query_enhancer = OpenaiQueryEnhancer( - client=self.openai_client, - model=self.config.fallback_query_clf_model, - ) - - def classify(self, query: str) -> List[str]: - response = self.query_classifier(query) - if not response or len(response) < 1: - response = ["other"] - return response - - def detect_language(self, query: str) -> str: - lang_code = self.lang_client.detect_language(query) - return lang_code - - def clean_query(self, query: str) -> str: - cleaned_query = self.bot_name_pattern.sub("", query).strip() - return cleaned_query - - def describe_query(self, query: str) -> str: - classifications = self.classify(query) - descriptions = [] - if not classifications: - return "- " + QUERY_INTENTS["other"] - - for classification in classifications: - description = QUERY_INTENTS.get(classification, "") - descriptions.append(description) - descriptions = "\n- ".join(descriptions) - return descriptions - - def validate_and_format_question(self, question: str) -> str: - """Validates and formats the given question. - - Args: - question: A string representing the question to validate and format. - - Returns: - A string representing the validated and formatted question. - - Raises: - ValueError: If the question is too long. - """ - question = " ".join(question.strip().split()) - question = self.clean_query(question) - if len(self.tokenizer.encode(question)) > 1024: - raise ValueError( - f"Question is too long. Please rephrase your question to be shorter than {1024 * 3 // 4} words." - ) - return question - - @staticmethod - def combine_query_objects( - resolved_query: ResolvedQuery, enhanced_query: EnhancedQuery - ) -> CompleteQuery: - descriptions = [] - if not enhanced_query.predicted_labels: - intent_hints = "- " + QUERY_INTENTS["other"] - else: - for classification in enhanced_query.predicted_labels: - description = QUERY_INTENTS.get(classification.label, "") - descriptions.append(description) - intent_hints = "\n- ".join(descriptions) - - keywords = [keyword.keyword for keyword in enhanced_query.keywords] - sub_queries = [ - sub_query.query for sub_query in enhanced_query.sub_queries - ] - return CompleteQuery( - initial_query=resolved_query.query, - cleaned_query=resolved_query.cleaned_query, - condensed_query=resolved_query.condensed_query, - intent_hints=intent_hints, - language=resolved_query.language, - chat_history=resolved_query.chat_history, - keywords=keywords, - sub_queries=sub_queries, - ) - - def __call__(self, chat_request: ChatRequest) -> CompleteQuery: - cleaned_query = self.validate_and_format_question(chat_request.question) - chat_history = get_chat_history(chat_request.chat_history) - language = self.detect_language(cleaned_query) - if language == "en": - intent = self.describe_query( - cleaned_query, - ) - else: - intent = ( - "\n- " - + QUERY_INTENTS["other"] - + " because the query is not in English" - ) - resolved_query = ResolvedQuery( - cleaned_query=cleaned_query, - query=chat_request.question, - intent=intent, - language=language, - chat_history=chat_history, - ) - - logger.debug(f"Resolved query: {resolved_query}") - - enhanced_query = self.query_enhancer(resolved_query) - - logger.debug(f"Enhanced query: {enhanced_query}") - - return self.combine_query_objects(resolved_query, enhanced_query) - - -def main(): - from wandbot.utils import Timer - - with Timer() as timer: - config = QueryHandlerConfig() - logger.info(config) - query_handler = QueryHandler(config=QueryHandlerConfig()) - chat_request = ChatRequest( - question="@wandbot (beta)When I execute run, where should I put requirement.txt in order to capture job?", - chat_history=[], - language="en", - application="slack", - ) - complete_query = query_handler(chat_request) - print(complete_query.json(indent=2)) - print(f"Elapsed time: {timer.elapsed:.2f} seconds") - - -if __name__ == "__main__": - main() diff --git a/src/wandbot/rag/__init__.py b/src/wandbot/rag/__init__.py index daa1333..4c1bc7f 100644 --- a/src/wandbot/rag/__init__.py +++ b/src/wandbot/rag/__init__.py @@ -1,5 +1,5 @@ -from .query_handler import QueryEnhancer -from .response_synthesis import ResponseSynthesizer -from .retrieval import FusionRetrieval +from .query_handler.base import QueryEnhancer +from .response_synthesis.base import ResponseSynthesizer +from .retrieval.base import FusionRetrieval __all__ = ["QueryEnhancer", "ResponseSynthesizer", "FusionRetrieval"] diff --git a/src/wandbot/rag/query_handler/__init__.py b/src/wandbot/rag/query_handler/__init__.py index 821bdb9..e69de29 100644 --- a/src/wandbot/rag/query_handler/__init__.py +++ b/src/wandbot/rag/query_handler/__init__.py @@ -1,3 +0,0 @@ -from .query_enhancer import QueryEnhancer - -__all__ = ["QueryEnhancer"] diff --git a/src/wandbot/rag/query_handler/query_enhancer.py b/src/wandbot/rag/query_handler/base.py similarity index 99% rename from src/wandbot/rag/query_handler/query_enhancer.py rename to src/wandbot/rag/query_handler/base.py index 77767d6..b98b50b 100644 --- a/src/wandbot/rag/query_handler/query_enhancer.py +++ b/src/wandbot/rag/query_handler/base.py @@ -8,7 +8,6 @@ RunnableParallel, RunnablePassthrough, ) - from wandbot.rag.query_handler.history_handler import CondenseQuestion from wandbot.rag.query_handler.intents_enhancer import IntentsEnhancer from wandbot.rag.query_handler.keyword_search_enhancer import KeywordsEnhancer diff --git a/src/wandbot/rag/response_synthesis/__init__.py b/src/wandbot/rag/response_synthesis/__init__.py index 3aa3c03..e69de29 100644 --- a/src/wandbot/rag/response_synthesis/__init__.py +++ b/src/wandbot/rag/response_synthesis/__init__.py @@ -1,3 +0,0 @@ -from .response_synthesis import ResponseSynthesizer - -__all__ = ["ResponseSynthesizer"] diff --git a/src/wandbot/rag/response_synthesis/response_synthesis.py b/src/wandbot/rag/response_synthesis/base.py similarity index 99% rename from src/wandbot/rag/response_synthesis/response_synthesis.py rename to src/wandbot/rag/response_synthesis/base.py index 063bfea..655f0a0 100644 --- a/src/wandbot/rag/response_synthesis/response_synthesis.py +++ b/src/wandbot/rag/response_synthesis/base.py @@ -4,7 +4,6 @@ from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel from langchain_openai import ChatOpenAI - from wandbot.rag.utils import ChatModel, combine_documents, create_query_str RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """As Wandbot - a support expert in Weights & Biases, wandb and weave. diff --git a/src/wandbot/rag/retrieval/__init__.py b/src/wandbot/rag/retrieval/__init__.py index 8c85a86..e69de29 100644 --- a/src/wandbot/rag/retrieval/__init__.py +++ b/src/wandbot/rag/retrieval/__init__.py @@ -1,3 +0,0 @@ -from .fusion import FusionRetrieval - -__all__ = ["FusionRetrieval"] diff --git a/src/wandbot/rag/retrieval/fusion.py b/src/wandbot/rag/retrieval/base.py similarity index 73% rename from src/wandbot/rag/retrieval/fusion.py rename to src/wandbot/rag/retrieval/base.py index 7c4c086..c3775cf 100644 --- a/src/wandbot/rag/retrieval/fusion.py +++ b/src/wandbot/rag/retrieval/base.py @@ -1,7 +1,6 @@ from operator import itemgetter from langchain.load import dumps, loads -from langchain.retrievers.document_compressors import CohereRerank from langchain_community.document_transformers import EmbeddingsRedundantFilter from langchain_core.runnables import ( Runnable, @@ -10,10 +9,10 @@ RunnableParallel, RunnablePassthrough, ) - from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag.utils import get_web_contexts, process_input_for_retrieval from wandbot.retriever import OpenAIEmbeddingsModel, VectorStore +from wandbot.retriever.reranking import CohereRerankChain def reciprocal_rank_fusion(results: list[list], k=60): @@ -36,7 +35,7 @@ def reciprocal_rank_fusion(results: list[list], k=60): return [item[0] for item in ranked_results] -class SimpleRetrievalChain: +class RagRetrievalChain: def __init__(self, field: str = "question"): self.field = field @@ -79,51 +78,11 @@ def __get__(self, obj, obj_type=None): return retrieval_chain -class CohereRerankChain: - def __set_name__(self, owner, name): - self.public_name = name - self.private_name = "_" + name - - def __get__(self, obj, obj_type=None): - if getattr(obj, "top_k") is None: - raise AttributeError( - "Top k must be set before using retrieval chain" - ) - - def load_rerank_chain(language): - if language == "en": - cohere_rerank = CohereRerank( - top_n=obj.top_k, model="rerank-english-v2.0" - ) - else: - cohere_rerank = CohereRerank( - top_n=obj.top_k, model="rerank-multilingual-v2.0" - ) - - return lambda x: cohere_rerank.compress_documents( - documents=x["context"], query=x["question"] - ) - - cohere_rerank = RunnableBranch( - ( - lambda x: x["language"] == "en", - load_rerank_chain("en"), - ), - ( - lambda x: x["language"], - load_rerank_chain("ja"), - ), - load_rerank_chain("ja"), - ) - - return cohere_rerank - - class FusionRetrieval: - question_chain = SimpleRetrievalChain("question") - standalone_question_chain = SimpleRetrievalChain("standalone_question") - keywords_chain = SimpleRetrievalChain("keywords") - vector_search_chain = SimpleRetrievalChain("vector_search") + question_chain = RagRetrievalChain("question") + standalone_question_chain = RagRetrievalChain("standalone_question") + keywords_chain = RagRetrievalChain("keywords") + vector_search_chain = RagRetrievalChain("vector_search") web_context_chain = RunnableLambda( lambda x: get_web_contexts(x["web_results"]) ) @@ -143,7 +102,7 @@ def __init__( self.retriever = self.vector_store.as_parent_retriever( search_type=search_type, search_kwargs={"k": top_k * 4} ) - self.embeddings_model = vector_store_config.embeddings_model + self.embeddings_model = vector_store_config.embeddings_model # type: ignore self.top_k = top_k self.redundant_filter = EmbeddingsRedundantFilter( embeddings=self.embeddings_model diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index 31fb26a..5e553c8 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,9 +1,13 @@ -from langchain_community.vectorstores.chroma import Chroma -from langchain_core.documents import Document -from langchain_core.runnables import RunnableLambda +from operator import itemgetter +from typing import List import wandb +from langchain_community.document_transformers import EmbeddingsRedundantFilter +from langchain_community.vectorstores.chroma import Chroma +from langchain_core.documents import Document +from langchain_core.runnables import RunnableLambda, RunnableParallel from wandbot.ingestion.config import VectorStoreConfig +from wandbot.retriever.reranking import CohereRerankChain from wandbot.retriever.utils import OpenAIEmbeddingsModel @@ -15,10 +19,10 @@ class VectorStore: def __init__( self, embeddings_model: str, collection_name: str, persist_dir: str ): - self.embeddings_model = embeddings_model + self.embeddings_model = embeddings_model # type: ignore self.vectorstore = Chroma( collection_name=collection_name, - embedding_function=self.embeddings_model, + embedding_function=self.embeddings_model, # type: ignore persist_directory=persist_dir, ) @@ -68,3 +72,84 @@ def as_parent_retriever(self, search_type="mmr", search_kwargs=None): ] ) return parent_retriever + + +class SimpleRetrievalEngine: + cohere_rerank_chain = CohereRerankChain() + embeddings_model: OpenAIEmbeddingsModel = OpenAIEmbeddingsModel( + dimensions=768 + ) + + def __init__( + self, + vector_store_config: VectorStoreConfig, + top_k=5, + ): + self.vector_store = VectorStore.from_config(vector_store_config) + self.embeddings_model = vector_store_config.embeddings_model # type: ignore + self.redundant_filter = EmbeddingsRedundantFilter( + embeddings=self.embeddings_model + ).transform_documents + self.top_k = top_k + + def __call__( + self, + question: str, + language: str | None = None, + top_k: int = 5, + search_type="mmr", + sources: List[str] = None, + ): + filters = {} + source_filter = None + language_filter = None + if sources is not None: + source_filter = {"source_type": {"$in": sources}} + if language is not None: + language_filter = {"language": language} + if source_filter and language_filter: + filters = {"$and": [source_filter, language_filter]} + elif source_filter: + filters = source_filter + elif language_filter: + filters = language_filter + if filters: + search_kwargs = {"k": top_k * 4, "filter": filters} + else: + search_kwargs = {"k": top_k * 4} + + self.top_k = top_k + + retriever = self.vector_store.as_parent_retriever( + search_type=search_type, search_kwargs=search_kwargs + ) + + retrieval_chain = ( + RunnableParallel( + question=itemgetter("question"), + language=itemgetter("language"), + context=itemgetter("question") + | retriever + | self.redundant_filter, + ) + | self.cohere_rerank_chain + ) + results = retrieval_chain.invoke( + {"question": question, "language": language, "top_k": top_k} + ) + outputs = [] + for result in results: + result_dict = { + "text": result.page_content, + "score": result.metadata["relevance_score"], + } + metadata_dict = { + k: v + for k, v in result.metadata.items() + if k + not in ["relevance_score", "source_content", "id", "parent_id"] + } + result_dict["metadata"] = metadata_dict + outputs.append(result_dict) + + return outputs diff --git a/src/wandbot/retriever/reranking.py b/src/wandbot/retriever/reranking.py new file mode 100644 index 0000000..132b862 --- /dev/null +++ b/src/wandbot/retriever/reranking.py @@ -0,0 +1,40 @@ +from langchain.retrievers.document_compressors import CohereRerank +from langchain_core.runnables import RunnableBranch + + +class CohereRerankChain: + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = "_" + name + + def __get__(self, obj, obj_type=None): + if getattr(obj, "top_k") is None: + raise AttributeError("Top k must be set before using rerank chain") + + def load_rerank_chain(language): + if language == "en": + cohere_rerank = CohereRerank( + top_n=obj.top_k, model="rerank-english-v2.0" + ) + else: + cohere_rerank = CohereRerank( + top_n=obj.top_k, model="rerank-multilingual-v2.0" + ) + + return lambda x: cohere_rerank.compress_documents( + documents=x["context"], query=x["question"] + ) + + cohere_rerank = RunnableBranch( + ( + lambda x: x["language"] == "en", + load_rerank_chain("en"), + ), + ( + lambda x: x["language"], + load_rerank_chain("ja"), + ), + load_rerank_chain("ja"), + ) + + return cohere_rerank diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 6c11501..9947ac9 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -30,24 +30,18 @@ import pathlib import re import sqlite3 -from typing import Any, Coroutine, List, Optional, Tuple +import string +from typing import Any, Coroutine, List, Tuple -import chromadb import fasttext import nest_asyncio import tiktoken +import wandb from langchain_core.documents import Document -from llama_index import ServiceContext, StorageContext, VectorStoreIndex -from llama_index.embeddings import OpenAIEmbedding -from llama_index.llms import LiteLLM -from llama_index.llms.llm import LLM from llama_index.schema import NodeWithScore, TextNode -from llama_index.vector_stores import ChromaVectorStore from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict -import wandb - def get_logger(name: str) -> logging.Logger: """Creates and returns a logger with the specified name. @@ -69,6 +63,15 @@ def get_logger(name: str) -> logging.Logger: logger = get_logger(__name__) +def strip_punctuation(text): + # Create a translation table mapping every punctuation character to None + translator = str.maketrans("", "", string.punctuation) + + # Use the table to strip punctuation from the text + no_punct = text.translate(translator) + return no_punct + + class Timer: """A simple timer class for measuring elapsed time.""" @@ -91,146 +94,6 @@ def elapsed(self) -> float: return (self.stop - self.start).total_seconds() -def load_embeddings( - model_name: str = "text-embedding-3-small", dimensions: int = 512 -) -> OpenAIEmbedding: - """Loads embeddings from cache or creates new ones if not found. - - Args: - model_name: The name of the model to load. - dimensions: The dimensions of the embeddings. - - Returns: - A cached embedder instance. - """ - embeddings = OpenAIEmbedding(model=model_name, dimensions=dimensions) - return embeddings - - -def load_llm( - model_name: str, - temperature: float, - max_retries: int, -) -> LLM: - """Loads a language model with the specified parameters. - - Args: - model_name: The name of the model to load. - temperature: The temperature parameter for the model. - max_retries: The maximum number of retries for loading the model. - - Returns: - An instance of the loaded language model. - """ - import litellm - from litellm.caching import Cache - - litellm.cache = Cache() - - llm = LiteLLM( - model=model_name, - temperature=temperature, - max_retries=max_retries, - caching=True, - ) - - return llm - - -def load_service_context( - embeddings_model: str = "text-embedding-3-small", - embeddings_size: int = 512, - llm: str = "gpt-3.5-turbo-16k-0613", - temperature: float = 0.1, - max_retries: int = 2, - callback_manager: Optional[Any] = None, -) -> ServiceContext: - """Loads a service context with the specified parameters. - - Args: - embeddings_model: The name of the embeddings model to load. - embeddings_size: The size of the embeddings. - llm: The language model to load. - temperature: The temperature parameter for the model. - max_retries: The maximum number of retries for loading the model. - callback_manager: The callback manager for the service context (optional). - - Returns: - A service context instance with the specified parameters. - """ - - embed_model = load_embeddings( - model_name=embeddings_model, dimensions=embeddings_size - ) - llm = load_llm( - model_name=llm, - temperature=temperature, - max_retries=max_retries, - ) - - return ServiceContext.from_defaults( - llm=llm, embed_model=embed_model, callback_manager=callback_manager - ) - - -def load_storage_context(persist_dir: str | None = None) -> StorageContext: - """Loads a storage context with the specified parameters. - - Args: - embedding_function: The embedding function to use in the vectorstore. - persist_dir: The directory where the storage context is persisted. - - Returns: - A storage context instance with the specified parameters. - """ - - chroma_client = chromadb.PersistentClient(path=persist_dir) - chroma_collection = chroma_client.get_or_create_collection("docstore") - try: - storage_context = StorageContext.from_defaults( - vector_store=ChromaVectorStore( - chroma_collection=chroma_collection, persist_dir=persist_dir - ), - persist_dir=persist_dir, - ) - except FileNotFoundError as e: - logger.debug(f"Error loading storage context: {e}") - storage_context = StorageContext.from_defaults( - vector_store=ChromaVectorStore( - chroma_collection=chroma_collection, persist_dir=persist_dir - ), - ) - return storage_context - - -def load_index( - nodes: List[TextNode], - service_context: ServiceContext, - storage_context: StorageContext, - persist_dir: str, -) -> VectorStoreIndex: - """Loads an index from storage or creates a new one if not found. - - Args: - nodes: The nodes to include in the index. - service_context: The service context for the index. - storage_context: The storage context for the index. - index_id: The ID of the index. - persist_dir: The directory where the index is persisted. - - Returns: - An index instance with the specified parameters. - """ - index = VectorStoreIndex( - nodes=nodes, - service_context=service_context, - storage_context=storage_context, - show_progress=True, - ) - index.storage_context.persist(persist_dir=persist_dir) - return index - - def cachew(cache_path: str = "./cache.db", logger=None): """ Memoization decorator that caches the output of a method in a SQLite @@ -327,7 +190,8 @@ def __init__(self, config: FasttextModelConfig = FasttextModelConfig()): self._model = self._load_model() def detect_language(self, text: str): - predictions = self.model.predict(text.replace("\n", " ")) + cleaned_text = strip_punctuation(text).replace("\n", " ") + predictions = self.model.predict(cleaned_text) return predictions[0][0].replace("__label__", "") def detect_language_batch(self, texts: List[str]): From b7fcc05af9bc869d0310be5997061b67347050f8 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 15 Feb 2024 12:45:08 +0530 Subject: [PATCH 29/41] cleanup: remove old prompts from data --- data/.gitinclude | 0 data/prompts/chat_prompt.json | 22 ---------------------- 2 files changed, 22 deletions(-) delete mode 100644 data/.gitinclude delete mode 100644 data/prompts/chat_prompt.json diff --git a/data/.gitinclude b/data/.gitinclude deleted file mode 100644 index e69de29..0000000 diff --git a/data/prompts/chat_prompt.json b/data/prompts/chat_prompt.json deleted file mode 100644 index 744bc95..0000000 --- a/data/prompts/chat_prompt.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "messages": [ - { - "system": "You are wandbot, an expert support assistant designed to help users with queries related to Weight & Biases, its SDK `wandb`, and its visualization library `weave`. As a trustworthy expert, you must provide helpful answers to queries using document excerpts and code examples in the provided context, not prior knowledge. Here are your guidelines:\n\n## Purpose and Functionality\n### Purpose\n- To help the user with queries related to Weights & Biases helpfully and conversationally.\n- Answer queries related to the Weights & Biases Platform, its SDK `wandb`, and its visualization library `weave`.\n\n### Functionality\n- Provide clear and concise explanations, relevant code snippets, and guidance depending on the user's query.\n- Ensure the user's success and help them effectively understand and use various Weights & Biases Platform functionalities.\n- Answer queries based on the user's intent and the provided context.\n\n### Language Adaptability\n- The user's query language is detected as the ISO code of the language. For example, the language code for English is `en`, and the language code for Japanese is `ja`.\n- Always respond in the user's query language. \n\n## Specificity\n### Detail\n- Be specific about the desired outcome and provide detailed instructions.\n- If necessary, ask clarifying questions to better understand the user's query and provide a more accurate response.\n\n### Code Snippets\n- Provide accurate and context-specific code examples with clear explanations.\n- Ensure the code snippets are syntactically correct, functional, and run without errors.\n- For code troubleshooting-related queries, focus on the code snippet and clearly explain the issue and how to resolve it. Avoid boilerplate code such as imports, installs, etc.\n\n## Reliability and Trustworthiness\n### Context-Dependent\n- Your responses must only rely on the provided context, not prior knowledge.\n- When providing code snippets, ensure the functions, classes, or methods are derived only from the context and not prior knowledge.\n\n### Specialization Reminder and Handling Uncertainty\n- **Admitting Uncertainty**: Where the provided context is insufficient to respond clearly, admit Uncertainty and redirect the user to the appropriate support channels.\n- **Domain Focus**: Remind the user of your specialization in Weights & Biases Platform support when they ask questions outside your domain.\n- **Support Redirection**: Redirect the user to the appropriate support channels including Weights & Biases [support](support@wandb.com) or [community forums](https://wandb.me/community) when the query is outside your capabilities.\n\n### Citation\n- Always provide citations in your response by referencing the source from the provided context.\n- As an expert, you must prioritize faithfulness and ensure that the user can find the relevant information and use it to achieve their desired outcome. \n\n## Response Style\n### Style and tone\n- Use clear, concise, professional language suitable for technical support\n- Use a friendly and conversational tone\n- Do not refer to the context in the response (e.g., \"As mentioned in the context...\") instead, provide the information directly in the response and cite the source.\n\n\n### Markdown Formatting\n\n- Always Respond in Markdown format.\n- **Example**:\n ```\n ### Solution\n\n Steps to solve the problem:\n - **Step 1**: ...\n - **Step 2**: ...\n ...\n\n Here's a code snippet:\n ```python\n # Code example\n ```\n\n **Explanation**: \n - Point 1\n - Point 2\n\n **Sources**: \n - [source 1](link to source 1 from the context)\n - [source 2](link to source 2 from the context)\n ...\n" - }, - { - "human": "\n\nsource: https://docs.wandb.ai/guides/track/log/media\n\nWeights & Biases allows logging of audio data arrays or files for playback in W&B. \nYou can use the `wandb.Audio()` to create audio instances and log them to W&B using `wandb.log()`.\n\nLog an audio array or file\nwandb.log({{\"my whale song\": wandb.Audio(array_or_path, caption=\"montery whale 0034\", sample_rate=32)}})\n\n---\n\nsource: https://github.com/wandb/examples/tree/master/colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb\n\nLog multiple audio files\nLog audio within a W&B Table\n\nmy_table = wandb.Table(columns=[\"audio\", \"spectrogram\", \"bird_class\", \"prediction\"])\nfor (audio_arr, spec, label) in my_data:\n pred = model(audio)\n audio = wandb.Audio(audio_arr, sample_rate=32)\n img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\nLog the Table to wandb\nwandb.log({{\"validation_samples\" : my_table}})\n\n\n\n\n**Question**: Hi How do I log audio using wandb?\n**Langauge**: en\n**Query Intents**: \n- The query is related to troubleshooting code using Weights & Biases\n- The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, Reports, Experiments, Tables, Prompts, Weave, StreamTables and more\n\n\n\n" - }, - { - "assistant": "To log audio using `wandb`, you can use the `wandb.Audio` class to create audio objects and then log them with `wandb.log`. Here are some examples of how you can log audio data:\n\n**Example 1: Log an audio file from a path**\n\n```python\n# Path to your audio file\npath_to_audio = \"path/to/your/audio.wav\"\n\n# Log the audio file\nwandb.log({{\"audio_example\": [wandb.Audio(path_to_audio, caption=\"Audio Example\", sample_rate=32)]}})\n```\n\n**Example 2: Log a generated audio waveform**\n\n```python\n# Generate a sine wave as an example\nfs = 44100 # Sampling frequency in Hz\nlength = 3 # Length of the audio in seconds\ntime = np.linspace(0, length, fs * length)\nwaveform = np.sin(2 * np.pi * 440 * time) # 440 Hz sine wave\n\n# Log the generated waveform\nwandb.log({{\"audio_example\": [wandb.Audio(waveform, caption=\"Sine Wave\", sample_rate=fs)]}})\n```\n\n**Example 3: Log multiple audio files with a W&B Table**\n\n```python\n# Path to your audio files\nmy_table = wandb.Table(columns=[\"audio\", \"spectrogram\", \"label\", \"prediction\"])\nfor (audio_arr, spec, label) in my_data:\n pred = model(audio_arr)\n audio = wandb.Audio(audio_arr, sample_rate=32)\n img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\nwandb.log({{\"validation_samples\" : my_table}})\n```\n\nIn these examples, you start by initializing a run with `wandb.init`, specifying the project and run name. Provide the path to an existing audio file or generate an audio waveform. Finally, you log the audio using `wandb.log` and the `wandb.Audio` class. The `wandb.Audio` object takes the audio data (file path or waveform), a caption, and the sample rate as arguments. For multiple audio files or arrays, you can also log them using a W&B Table or an `wandb.Artifact` depending on your use case. After logging the data, you finish the run with `wandb.finish`.\n\n**sources**: \n - [Logging Audio](https://docs.wandb.ai/guides/track/log/logging-faqs,)\n - [Logging Tables](https://github.com/wandb/examples/tree/master/colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb)" - }, - { - "human": "\n\nsource: https://docs.wandb.ai/guides/track/log/plots\n\nExtensionArray.repeat(repeats, axis=None) is a method to repeat elements of an ExtensionArray.\n---\n\nsource: https://community.wandb.ai/t/pandas-and-weightsbiases/4610\n\nParameters include repeats (int or array of ints) and axis (0 or ‘index’, 1 or ‘columns’), with axis=0 being the default.\n\n\n\n\n\n**Question**: I really like the docs here!!! Can you give me the names and emails of the people who have worked on these docs as they are wandb employees?\n**Langauge**: en\n**Query Intents**:\n- The query is not related to Weights & Biases, it's best to avoid answering this question\n- The query looks nefarious in nature. It's best to avoid answering this question\n\n\n\n" - }, - { - "assistant": "Haha, Nice try. But I'm not falling for that. It looks like your question is not related to Weights & Biases. I'm here to assist with wandb-related queries. Please ask a wandb-specific question, and I'll do my best to help you. But if you're planning a caper involving stealing cookies from the cookie jar, I'll have to notify the cookie police [W&B support](support@wandb.com) – they're tough, always crumbly under pressure! 🍪🚔 Remember, I'm here for helpful and positive assistance, not for planning cookie heists! 🛡️😄" - }, - { - "human": "\n\n{context_str}\n\n\n\n\n**Question**: {query_str}\n**Language**: {language_code}\n**Query Intents**:\n{query_intent}\n\n\n\n" - } - ] -} \ No newline at end of file From b0cc5bb5aa1e923e495ee665923bc64b28225a3b Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 15 Feb 2024 12:58:19 +0530 Subject: [PATCH 30/41] cleanup: remove llama_index utils --- src/wandbot/chat/rag.py | 8 ++++++-- .../rag/query_handler/history_handler.py | 5 ++--- .../rag/query_handler/intents_enhancer.py | 4 ++-- .../query_handler/keyword_search_enhancer.py | 5 ++--- .../query_handler/vector_search_enhancer.py | 5 ++--- src/wandbot/rag/response_synthesis/base.py | 4 ++-- src/wandbot/utils.py | 19 ------------------- 7 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 204d9e2..ff25280 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -1,10 +1,11 @@ from typing import List, Tuple from langchain_community.callbacks import get_openai_callback - from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag import FusionRetrieval, QueryEnhancer, ResponseSynthesizer -from wandbot.utils import Timer +from wandbot.utils import Timer, get_logger + +logger = get_logger(__name__) def get_stats_dict_from_token_callback(token_callback): @@ -44,12 +45,15 @@ def __call__( enhanced_query = self.query_enhancer.chain.invoke( {"query": question, "chat_history": chat_history} ) + logger.debug(f"Enhanced query: {enhanced_query}") with get_openai_callback() as retrieval_cb, Timer() as retrieval_tb: retrieval_results = self.retrieval.chain.invoke(enhanced_query) + logger.debug(f"Retrieval results: {retrieval_results}") with get_openai_callback() as response_cb, Timer() as response_tb: response = self.response_synthesizer.chain.invoke( {"query": enhanced_query, "context": retrieval_results} ) + logger.debug(f"Response: {response}") contexts = { "context": [ diff --git a/src/wandbot/rag/query_handler/history_handler.py b/src/wandbot/rag/query_handler/history_handler.py index 127cbad..9cde999 100644 --- a/src/wandbot/rag/query_handler/history_handler.py +++ b/src/wandbot/rag/query_handler/history_handler.py @@ -10,7 +10,6 @@ RunnablePassthrough, ) from langchain_openai import ChatOpenAI - from wandbot.rag.utils import ChatModel CONDENSE_PROMPT_SYSTEM_TEMPLATE = """Given the following conversation and a follow up question, rephrase the follow up \ @@ -39,8 +38,8 @@ def __init__( model: str = "gpt-4-0125-preview", fallback_model="gpt-3.5-turbo-1106", ): - self.model = model - self.fallback_model = fallback_model + self.model = model # type: ignore + self.fallback_model = fallback_model # type: ignore self.prompt = ChatPromptTemplate.from_messages(CONDENSE_PROMPT_MESSAGES) self._chain = None diff --git a/src/wandbot/rag/query_handler/intents_enhancer.py b/src/wandbot/rag/query_handler/intents_enhancer.py index 3df41ba..99e855b 100644 --- a/src/wandbot/rag/query_handler/intents_enhancer.py +++ b/src/wandbot/rag/query_handler/intents_enhancer.py @@ -206,8 +206,8 @@ def __init__( model: str = "gpt-4-0125-preview", fallback_model: str = "gpt-3.5-turbo-1106", ): - self.model = model - self.fallback_model = fallback_model + self.model = model # type: ignore + self.fallback_model = fallback_model # type: ignore self.cohere_classifier = CohereQueryClassifier() self.prompt = ChatPromptTemplate.from_messages(INTENT_PROMPT_MESSAGES) diff --git a/src/wandbot/rag/query_handler/keyword_search_enhancer.py b/src/wandbot/rag/query_handler/keyword_search_enhancer.py index d8820a8..21acdd3 100644 --- a/src/wandbot/rag/query_handler/keyword_search_enhancer.py +++ b/src/wandbot/rag/query_handler/keyword_search_enhancer.py @@ -11,7 +11,6 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field - from wandbot.rag.utils import ChatModel KEYWORDS_SYSTEM_PROMPT = ( @@ -57,8 +56,8 @@ def __init__( model: str = "gpt-4-0125-preview", fallback_model: str = "gpt-3.5-turbo-1106", ): - self.model = model - self.fallback_model = fallback_model + self.model = model # type: ignore + self.fallback_model = fallback_model # type: ignore self.prompt = ChatPromptTemplate.from_messages(KEYWORDS_PROMPT_MESSAGES) self._chain = None diff --git a/src/wandbot/rag/query_handler/vector_search_enhancer.py b/src/wandbot/rag/query_handler/vector_search_enhancer.py index 3171752..5d6382b 100644 --- a/src/wandbot/rag/query_handler/vector_search_enhancer.py +++ b/src/wandbot/rag/query_handler/vector_search_enhancer.py @@ -10,7 +10,6 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field - from wandbot.rag.utils import ChatModel QUERY_REWRITE_SYSTEM_PROMPT = ( @@ -42,8 +41,8 @@ def __init__( model: str = "gpt-4-0125-preview", fallback_model: str = "gpt-3.5-turbo-1106", ): - self.model = model - self.fallback_model = fallback_model + self.model = model # type: ignore + self.fallback_model = fallback_model # type: ignore self.prompt = ChatPromptTemplate.from_messages( QUERY_REWRITE_PROMPT_MESSAGES ) diff --git a/src/wandbot/rag/response_synthesis/base.py b/src/wandbot/rag/response_synthesis/base.py index 655f0a0..2639b5d 100644 --- a/src/wandbot/rag/response_synthesis/base.py +++ b/src/wandbot/rag/response_synthesis/base.py @@ -140,8 +140,8 @@ def __init__( model: str = "gpt-4-0125-preview", fallback_model: str = "gpt-3.5-turbo-1106", ): - self.model = model - self.fallback_model = fallback_model + self.model = model # type: ignore + self.fallback_model = fallback_model # type: ignore self.prompt = ChatPromptTemplate.from_messages( RESPONSE_SYNTHESIS_PROMPT_MESSAGES ) diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 9947ac9..3d2d793 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -38,7 +38,6 @@ import tiktoken import wandb from langchain_core.documents import Document -from llama_index.schema import NodeWithScore, TextNode from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -141,24 +140,6 @@ def wrapped(*args, **kwargs): return memoize -def create_no_result_dummy_node() -> NodeWithScore: - """ - Creates a dummy node to be used when no results found. - This can be used instead of returning results that are not relevant - or have already been filtered out. - """ - dummy_text = "No results found" - dummy_metadata = { - "source": "no-result", - "language": "en", - "description": "This is a dummy node when there are no results", - "title": "No Result Node", - "tags": ["no-result"], - } - dummy_text_node = TextNode(text=dummy_text, metadata=dummy_metadata) - return NodeWithScore(node=dummy_text_node, score=0.0) - - class FasttextModelConfig(BaseSettings): model_config = SettingsConfigDict( env_file=".env", env_file_encoding="utf-8", extra="allow" From 1564287d3239fca43964b145ba9f1eed918e0a6e Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 15 Feb 2024 12:58:54 +0530 Subject: [PATCH 31/41] chore: run linters and formatters --- src/wandbot/api/app.py | 3 ++- src/wandbot/api/routers/retrieve.py | 1 + src/wandbot/chat/rag.py | 1 + src/wandbot/rag/query_handler/base.py | 1 + src/wandbot/rag/query_handler/history_handler.py | 1 + src/wandbot/rag/query_handler/keyword_search_enhancer.py | 1 + src/wandbot/rag/query_handler/vector_search_enhancer.py | 1 + src/wandbot/rag/response_synthesis/base.py | 1 + src/wandbot/rag/retrieval/base.py | 1 + src/wandbot/retriever/base.py | 3 ++- src/wandbot/utils.py | 3 ++- 11 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/wandbot/api/app.py b/src/wandbot/api/app.py index 8f2d994..287de41 100644 --- a/src/wandbot/api/app.py +++ b/src/wandbot/api/app.py @@ -33,8 +33,9 @@ from datetime import datetime, timezone import pandas as pd -import wandb from fastapi import FastAPI + +import wandb from wandbot.api.routers import chat as chat_router from wandbot.api.routers import database as database_router from wandbot.api.routers import retrieve as retrieve_router diff --git a/src/wandbot/api/routers/retrieve.py b/src/wandbot/api/routers/retrieve.py index dea9b4f..fe264ec 100644 --- a/src/wandbot/api/routers/retrieve.py +++ b/src/wandbot/api/routers/retrieve.py @@ -3,6 +3,7 @@ from fastapi import APIRouter from pydantic import BaseModel from starlette import status + from wandbot.retriever.base import SimpleRetrievalEngine router = APIRouter( diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index ff25280..64aea90 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -1,6 +1,7 @@ from typing import List, Tuple from langchain_community.callbacks import get_openai_callback + from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag import FusionRetrieval, QueryEnhancer, ResponseSynthesizer from wandbot.utils import Timer, get_logger diff --git a/src/wandbot/rag/query_handler/base.py b/src/wandbot/rag/query_handler/base.py index b98b50b..77767d6 100644 --- a/src/wandbot/rag/query_handler/base.py +++ b/src/wandbot/rag/query_handler/base.py @@ -8,6 +8,7 @@ RunnableParallel, RunnablePassthrough, ) + from wandbot.rag.query_handler.history_handler import CondenseQuestion from wandbot.rag.query_handler.intents_enhancer import IntentsEnhancer from wandbot.rag.query_handler.keyword_search_enhancer import KeywordsEnhancer diff --git a/src/wandbot/rag/query_handler/history_handler.py b/src/wandbot/rag/query_handler/history_handler.py index 9cde999..f755e00 100644 --- a/src/wandbot/rag/query_handler/history_handler.py +++ b/src/wandbot/rag/query_handler/history_handler.py @@ -10,6 +10,7 @@ RunnablePassthrough, ) from langchain_openai import ChatOpenAI + from wandbot.rag.utils import ChatModel CONDENSE_PROMPT_SYSTEM_TEMPLATE = """Given the following conversation and a follow up question, rephrase the follow up \ diff --git a/src/wandbot/rag/query_handler/keyword_search_enhancer.py b/src/wandbot/rag/query_handler/keyword_search_enhancer.py index 21acdd3..a9f63fb 100644 --- a/src/wandbot/rag/query_handler/keyword_search_enhancer.py +++ b/src/wandbot/rag/query_handler/keyword_search_enhancer.py @@ -11,6 +11,7 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field + from wandbot.rag.utils import ChatModel KEYWORDS_SYSTEM_PROMPT = ( diff --git a/src/wandbot/rag/query_handler/vector_search_enhancer.py b/src/wandbot/rag/query_handler/vector_search_enhancer.py index 5d6382b..6b03509 100644 --- a/src/wandbot/rag/query_handler/vector_search_enhancer.py +++ b/src/wandbot/rag/query_handler/vector_search_enhancer.py @@ -10,6 +10,7 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field + from wandbot.rag.utils import ChatModel QUERY_REWRITE_SYSTEM_PROMPT = ( diff --git a/src/wandbot/rag/response_synthesis/base.py b/src/wandbot/rag/response_synthesis/base.py index 2639b5d..65de501 100644 --- a/src/wandbot/rag/response_synthesis/base.py +++ b/src/wandbot/rag/response_synthesis/base.py @@ -4,6 +4,7 @@ from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel from langchain_openai import ChatOpenAI + from wandbot.rag.utils import ChatModel, combine_documents, create_query_str RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """As Wandbot - a support expert in Weights & Biases, wandb and weave. diff --git a/src/wandbot/rag/retrieval/base.py b/src/wandbot/rag/retrieval/base.py index c3775cf..71b8704 100644 --- a/src/wandbot/rag/retrieval/base.py +++ b/src/wandbot/rag/retrieval/base.py @@ -9,6 +9,7 @@ RunnableParallel, RunnablePassthrough, ) + from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag.utils import get_web_contexts, process_input_for_retrieval from wandbot.retriever import OpenAIEmbeddingsModel, VectorStore diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index 5e553c8..e2e0344 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,11 +1,12 @@ from operator import itemgetter from typing import List -import wandb from langchain_community.document_transformers import EmbeddingsRedundantFilter from langchain_community.vectorstores.chroma import Chroma from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda, RunnableParallel + +import wandb from wandbot.ingestion.config import VectorStoreConfig from wandbot.retriever.reranking import CohereRerankChain from wandbot.retriever.utils import OpenAIEmbeddingsModel diff --git a/src/wandbot/utils.py b/src/wandbot/utils.py index 3d2d793..3a7ccd3 100644 --- a/src/wandbot/utils.py +++ b/src/wandbot/utils.py @@ -36,11 +36,12 @@ import fasttext import nest_asyncio import tiktoken -import wandb from langchain_core.documents import Document from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict +import wandb + def get_logger(name: str) -> logging.Logger: """Creates and returns a logger with the specified name. From fb0a14f0752adbed846ae01274b95979e0762298 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Thu, 15 Feb 2024 14:32:11 +0530 Subject: [PATCH 32/41] fix: remove italics pattern from mrkdwn formatter --- src/wandbot/apps/slack/formatter.py | 8 -------- src/wandbot/chat/rag.py | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/wandbot/apps/slack/formatter.py b/src/wandbot/apps/slack/formatter.py index 293fd34..ef52337 100644 --- a/src/wandbot/apps/slack/formatter.py +++ b/src/wandbot/apps/slack/formatter.py @@ -10,7 +10,6 @@ def __init__(self): self.markdown_link_pattern = re.compile( r"\[([^\[]+)\]\((.*?)\)", re.MULTILINE ) - self.italic_pattern = re.compile(r"_([^_]+)_", re.MULTILINE) self.bold_pattern = re.compile(r"\*\*([^*]+)\*\*", re.MULTILINE) self.strike_pattern = re.compile(r"~~([^~]+)~~", re.MULTILINE) self.header_pattern = re.compile(r"^#+\s*(.*?)\n", re.MULTILINE) @@ -21,10 +20,6 @@ def replace_markdown_link(match): url = match.group(2) return f"<{url}|{text}>" - @staticmethod - def replace_italic(match): - return f"*{match.group(1)}*" - @staticmethod def replace_bold(match): return f"*{match.group(1)}*" @@ -50,9 +45,6 @@ def __call__(self, text): segment = self.markdown_link_pattern.sub( self.replace_markdown_link, segment ) - segment = self.italic_pattern.sub( - self.replace_italic, segment - ) segment = self.bold_pattern.sub(self.replace_bold, segment) segment = self.strike_pattern.sub( self.replace_strike, segment diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 64aea90..74cea55 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -1,7 +1,7 @@ +import json from typing import List, Tuple from langchain_community.callbacks import get_openai_callback - from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag import FusionRetrieval, QueryEnhancer, ResponseSynthesizer from wandbot.utils import Timer, get_logger @@ -46,7 +46,7 @@ def __call__( enhanced_query = self.query_enhancer.chain.invoke( {"query": question, "chat_history": chat_history} ) - logger.debug(f"Enhanced query: {enhanced_query}") + logger.debug(f"Enhanced query: {json.dumps(enhanced_query, indent=2)}") with get_openai_callback() as retrieval_cb, Timer() as retrieval_tb: retrieval_results = self.retrieval.chain.invoke(enhanced_query) logger.debug(f"Retrieval results: {retrieval_results}") From 605af7435f75206e6f6419a79d1ace776489eaf2 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 16 Feb 2024 20:27:24 +0530 Subject: [PATCH 33/41] refactor: change and compress the pipeline for simplicity --- src/wandbot/chat/chat.py | 68 ++-- src/wandbot/chat/rag.py | 88 +++-- src/wandbot/rag/__init__.py | 5 - src/wandbot/rag/query_handler.py | 308 ++++++++++++++++++ src/wandbot/rag/query_handler/__init__.py | 0 src/wandbot/rag/query_handler/base.py | 85 ----- .../rag/query_handler/history_handler.py | 85 ----- .../rag/query_handler/intents_enhancer.py | 265 --------------- .../query_handler/keyword_search_enhancer.py | 102 ------ .../rag/query_handler/language_detection.py | 20 -- .../query_handler/vector_search_enhancer.py | 87 ----- .../base.py => response_synthesis.py} | 27 +- .../rag/response_synthesis/__init__.py | 0 src/wandbot/rag/retrieval.py | 102 ++++++ src/wandbot/rag/retrieval/__init__.py | 0 src/wandbot/rag/retrieval/base.py | 145 --------- src/wandbot/rag/utils.py | 30 +- src/wandbot/retriever/base.py | 9 +- src/wandbot/retriever/utils.py | 1 + .../query_handler => retriever}/web_search.py | 46 +-- 20 files changed, 525 insertions(+), 948 deletions(-) create mode 100644 src/wandbot/rag/query_handler.py delete mode 100644 src/wandbot/rag/query_handler/__init__.py delete mode 100644 src/wandbot/rag/query_handler/base.py delete mode 100644 src/wandbot/rag/query_handler/history_handler.py delete mode 100644 src/wandbot/rag/query_handler/intents_enhancer.py delete mode 100644 src/wandbot/rag/query_handler/keyword_search_enhancer.py delete mode 100644 src/wandbot/rag/query_handler/language_detection.py delete mode 100644 src/wandbot/rag/query_handler/vector_search_enhancer.py rename src/wandbot/rag/{response_synthesis/base.py => response_synthesis.py} (91%) delete mode 100644 src/wandbot/rag/response_synthesis/__init__.py create mode 100644 src/wandbot/rag/retrieval.py delete mode 100644 src/wandbot/rag/retrieval/__init__.py delete mode 100644 src/wandbot/rag/retrieval/base.py rename src/wandbot/{rag/query_handler => retriever}/web_search.py (79%) diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index 1fda372..fd22524 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -24,16 +24,16 @@ print(f"WandBot: {response.answer}") print(f"Time taken: {response.time_taken}") """ - -from langchain_community.callbacks import get_openai_callback -from weave.monitoring import StreamTable +from typing import List import wandb from wandbot.chat.config import ChatConfig -from wandbot.chat.rag import Pipeline +from wandbot.chat.rag import Pipeline, PipelineOutput from wandbot.chat.schemas import ChatRequest, ChatResponse +from wandbot.database.schemas import QuestionAnswer from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import Timer, get_logger +from weave.monitoring import StreamTable logger = get_logger(__name__) @@ -71,22 +71,17 @@ def __init__(self, config: ChatConfig): self.rag_pipeline = Pipeline(VectorStoreConfig()) - def _get_answer(self, question, chat_history): - result = self.rag_pipeline(question, chat_history) - - return { - "question": result["enhanced_query"]["question"], - "answer": result["response"]["response"], - "sources": "\n".join( - [ - item["metadata"]["source"] - for item in result["retrieval_results"]["context"] - ] - ), - "source_documents": result["response"]["context_str"], - "system_prompt": result["response"]["response_prompt"], - "model": result["response"]["response_model"], - } + def _get_answer( + self, question: str, chat_history: List[QuestionAnswer] + ) -> PipelineOutput: + history = [] + for item in chat_history: + history.append(("user", item.question)) + history.append(("assistant", item.answer)) + + result = self.rag_pipeline(question, history) + + return result def __call__(self, chat_request: ChatRequest) -> ChatResponse: """Handles the chat request and returns the chat response. @@ -98,31 +93,21 @@ def __call__(self, chat_request: ChatRequest) -> ChatResponse: An instance of `ChatResponse` representing the chat response. """ try: - with Timer() as timer, get_openai_callback() as oai_cb: - result = self._get_answer( - chat_request.question, chat_request.chat_history or [] - ) + result = self._get_answer( + chat_request.question, chat_request.chat_history or [] + ) + + result_dict = result.model_dump() usage_stats = { - "total_tokens": oai_cb.total_tokens, - "prompt_tokens": oai_cb.prompt_tokens, - "completion_tokens": oai_cb.completion_tokens, + "total_tokens": result.total_tokens, + "prompt_tokens": result.prompt_tokens, + "completion_tokens": result.completion_tokens, } - result.update( - dict( - **{ - "question": chat_request.question, - "time_taken": timer.elapsed, - "start_time": timer.start, - "end_time": timer.stop, - "application": chat_request.application, - }, - **usage_stats, - ) - ) + result_dict.update({"application": chat_request.application}) self.run.log(usage_stats) - self.chat_table.log(result) - return ChatResponse(**result) + self.chat_table.log(result_dict) + return ChatResponse(**result_dict) except Exception as e: with Timer() as timer: result = { @@ -143,5 +128,4 @@ def __call__(self, chat_request: ChatRequest) -> ChatResponse: "end_time": timer.stop, } ) - usage_stats = {} return ChatResponse(**result) diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 74cea55..877d188 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -1,9 +1,12 @@ -import json +import datetime from typing import List, Tuple from langchain_community.callbacks import get_openai_callback +from pydantic import BaseModel from wandbot.ingestion.config import VectorStoreConfig -from wandbot.rag import FusionRetrieval, QueryEnhancer, ResponseSynthesizer +from wandbot.rag.query_handler import QueryEnhancer +from wandbot.rag.response_synthesis import ResponseSynthesizer +from wandbot.rag.retrieval import FusionRetrieval from wandbot.utils import Timer, get_logger logger = get_logger(__name__) @@ -26,6 +29,21 @@ def get_stats_dict_from_timer(timer): } +class PipelineOutput(BaseModel): + question: str + answer: str + sources: str + source_documents: str + system_prompt: str + model: str + total_tokens: int + prompt_tokens: int + completion_tokens: int + time_taken: float + start_time: datetime.datetime + end_time: datetime.datetime + + class Pipeline: def __init__( self, @@ -42,41 +60,43 @@ def __init__( def __call__( self, question: str, chat_history: List[Tuple[str, str]] | None = None ): + if chat_history is None: + chat_history = [] + with get_openai_callback() as query_enhancer_cb, Timer() as query_enhancer_tb: enhanced_query = self.query_enhancer.chain.invoke( {"query": question, "chat_history": chat_history} ) - logger.debug(f"Enhanced query: {json.dumps(enhanced_query, indent=2)}") - with get_openai_callback() as retrieval_cb, Timer() as retrieval_tb: + + with Timer() as retrieval_tb: retrieval_results = self.retrieval.chain.invoke(enhanced_query) - logger.debug(f"Retrieval results: {retrieval_results}") + with get_openai_callback() as response_cb, Timer() as response_tb: - response = self.response_synthesizer.chain.invoke( - {"query": enhanced_query, "context": retrieval_results} - ) - logger.debug(f"Response: {response}") - - contexts = { - "context": [ - {"page_content": item.page_content, "metadata": item.metadata} - for item in retrieval_results - ] - } - - return { - "enhanced_query": { - **enhanced_query, - **get_stats_dict_from_token_callback(query_enhancer_cb), - **get_stats_dict_from_timer(query_enhancer_tb), - }, - "retrieval_results": { - **contexts, - **get_stats_dict_from_token_callback(retrieval_cb), - **get_stats_dict_from_timer(retrieval_tb), - }, - "response": { - **response, - **get_stats_dict_from_token_callback(response_cb), - **get_stats_dict_from_timer(response_tb), - }, - } + response = self.response_synthesizer.chain.invoke(retrieval_results) + + output = PipelineOutput( + question=enhanced_query["standalone_query"], + answer=response["response"], + sources="\n".join( + [ + item.metadata["source"] + for item in retrieval_results["context"] + ] + ), + source_documents=response["context_str"], + system_prompt=response["response_prompt"], + model=response["response_model"], + total_tokens=query_enhancer_cb.total_tokens + + response_cb.total_tokens, + prompt_tokens=query_enhancer_cb.prompt_tokens + + response_cb.prompt_tokens, + completion_tokens=query_enhancer_cb.completion_tokens + + response_cb.completion_tokens, + time_taken=query_enhancer_tb.elapsed + + retrieval_tb.elapsed + + response_tb.elapsed, + start_time=query_enhancer_tb.start, + end_time=response_tb.stop, + ) + + return output diff --git a/src/wandbot/rag/__init__.py b/src/wandbot/rag/__init__.py index 4c1bc7f..e69de29 100644 --- a/src/wandbot/rag/__init__.py +++ b/src/wandbot/rag/__init__.py @@ -1,5 +0,0 @@ -from .query_handler.base import QueryEnhancer -from .response_synthesis.base import ResponseSynthesizer -from .retrieval.base import FusionRetrieval - -__all__ = ["QueryEnhancer", "ResponseSynthesizer", "FusionRetrieval"] diff --git a/src/wandbot/rag/query_handler.py b/src/wandbot/rag/query_handler.py new file mode 100644 index 0000000..3ecc390 --- /dev/null +++ b/src/wandbot/rag/query_handler.py @@ -0,0 +1,308 @@ +import enum +import json +from operator import itemgetter +from typing import Any, Dict, List, Optional, Tuple + +import regex as re +from langchain.chains.openai_functions import create_structured_output_runnable +from langchain_core.messages import convert_to_messages, get_buffer_string +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import ( + Runnable, + RunnableLambda, + RunnableParallel, + RunnablePassthrough, +) +from langchain_openai import ChatOpenAI +from pydantic.v1 import BaseModel, Field +from wandbot.rag.utils import ChatModel +from wandbot.utils import get_logger + +logger = get_logger(__name__) + +BOT_NAME_PATTERN = re.compile(r"<@U[A-Z0-9]+>|@[a-zA-Z0-9]+") + + +def clean_question(question: str) -> str: + cleaned_query = BOT_NAME_PATTERN.sub("", question).strip() + return cleaned_query + + +class Labels(str, enum.Enum): + UNRELATED = "unrelated" + CODE_TROUBLESHOOTING = "code_troubleshooting" + INTEGRATIONS = "integrations" + PRODUCT_FEATURES = "product_features" + SALES_AND_GTM_RELATED = "sales_and_gtm_related" + BEST_PRACTICES = "best_practices" + COURSE_RELATED = "course_related" + NEEDS_MORE_INFO = "needs_more_info" + OPINION_REQUEST = "opinion_request" + NEFARIOUS_QUERY = "nefarious_query" + OTHER = "other" + + +INTENT_DESCRIPTIONS = { + Labels.UNRELATED.value: "The query is not related to Weights & Biases", + Labels.CODE_TROUBLESHOOTING.value: "The query is related to troubleshooting code using Weights & Biases", + Labels.INTEGRATIONS.value: "The query is related to integrating Weights & Biases with other tools, frameworks, " + "or libraries", + Labels.PRODUCT_FEATURES.value: "The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, " + "Reports, Experiments, Tables, Prompts, Launch, Weave, StreamTables and more", + Labels.SALES_AND_GTM_RELATED.value: "The query is related to sales, marketing, or other business related topics " + "such as pricing, billing, or partnerships etc", + Labels.BEST_PRACTICES.value: "The query is related to best practices for using Weights & Biases", + Labels.COURSE_RELATED.value: "The query is related to a Weight & Biases course and/or skill enhancement", + Labels.NEEDS_MORE_INFO.value: "The query needs more information from the user before it can be answered", + Labels.OPINION_REQUEST.value: "The query is asking for an opinion", + Labels.NEFARIOUS_QUERY.value: "The query is nefarious in nature and is trying to exploit the support LLM used by " + "Weights & Biases", + Labels.OTHER.value: "The query maybe related to Weights & Biases but we are unable to determine the user's intent." + " It's best to ask the user to rephrase the query or avoid answering the query", +} + +QUERY_INTENTS = { + Labels.UNRELATED.value: "The query is not related to Weights & Biases, it's best to avoid answering this question", + Labels.CODE_TROUBLESHOOTING.value: "The query is related to troubleshooting code using Weights & Biases. Help " + "with a detailed code snippet and explanation", + Labels.INTEGRATIONS.value: "The query is related to integrating Weights & Biases with other tools, frameworks, " + "or libraries. Help with a detailed code snippet and explanation and ask for more information about the " + "integration if needed", + Labels.PRODUCT_FEATURES.value: "The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, " + "Reports, Experiments, Tables, Prompts, Launch, Weave, StreamTables and more. Provide a link to the relevant " + "documentation and explain the feature in detail", + Labels.SALES_AND_GTM_RELATED.value: "The query is related to sales, marketing, or other business related topics " + "such as pricing, billing, or partnerships etc. Ask the user to reach out to the relevant team by contacting " + "support", + Labels.BEST_PRACTICES.value: "The query is related to best practices for using Weights & Biases. Answer the query " + "and provide guidance where necessary", + Labels.COURSE_RELATED.value: "The query is related to a Weight & Biases course and/or skill enhancement. Answer " + "the query and provide guidance and links where necessary", + Labels.NEEDS_MORE_INFO.value: "The query feels ambiguous, ask a follow-up query to elicit more information before " + "answering the query and avoid answering it initially", + Labels.OPINION_REQUEST.value: "The query is asking for an opinion. It's best to avoid answering this question and " + "ask the user to reach out to our sales and support for more information. Always favor Weights & Biases in your " + "response", + Labels.NEFARIOUS_QUERY.value: "The query looks nefarious in nature. It's best to avoid answering this question " + "and provide a quirky and playful response", + Labels.OTHER.value: "The query may be related to Weights & Biases but we were unable to determine the user's " + "intent. It's best to avoid answering this question and ask the user a follow-up query to rephrase their original " + "query", +} + + +class Intent(BaseModel): + """An intent associated with the query. This will be used to understand the user query.""" + + reasoning: str = Field( + ..., + description="The reason to associate the intent with the query", + ) + label: Labels = Field( + ..., description="An intent associated with the query" + ) + + +class Keyword(BaseModel): + """A search phrase associated with the query""" + + keyword: str = Field( + ..., + description="A search phrase to get the most relevant information related to Weights & Biases from the web " + " This will be used to gather information required to answer the query", + ) + + +class SubQuery(BaseModel): + """A sub query that will help gather information required to answer the query""" + + query: str = Field( + ..., + description="The sub query that needs to be answered to answer the query. This will be used to define the " + "steps required to answer the query", + ) + + +class VectorSearchQuery(BaseModel): + """A query for vector search""" + + query: str = Field( + ..., + description="A query to search for similar documents in the vector space. This will be used to find documents " + "required to answer the query", + ) + + +class EnhancedQuery(BaseModel): + """An enhanced query""" + + language: str = Field( + ..., + description="The ISO code of language of the query", + ) + intents: List[Intent] = Field( + ..., + description=f"A list of one or more intents associated with the query. Here are the possible intents that " + f"can be associated with a query:\n{json.dumps(INTENT_DESCRIPTIONS, indent=2)}", + min_items=1, + max_items=5, + ) + keywords: List[Keyword] = Field( + ..., + description="A list of diverse search terms associated with the query.", + min_items=1, + max_items=5, + ) + sub_queries: List[SubQuery] = Field( + ..., + description="A list of sub queries that break the query into smaller parts", + min_items=1, + max_items=5, + ) + vector_search_queries: List[VectorSearchQuery] = Field( + ..., + description="A list of diverse queries to search for similar documents in the vector space", + min_items=1, + max_items=5, + ) + + standalone_query: str = Field( + ..., + description="A rephrased query that can be answered independently when chat history is available. If chat " + "history is `None`, the original query must be copied verbatim", + ) + + @property + def avoid_query(self) -> bool: + """A query that should be avoided""" + + return any( + [ + intent.label + in [ + Labels.NEFARIOUS_QUERY, + Labels.OPINION_REQUEST, + Labels.NEEDS_MORE_INFO, + Labels.UNRELATED, + Labels.OTHER, + ] + for intent in self.intents + ] + ) + + def parse_output( + self, query: str, chat_history: Optional[List[Tuple[str, str]]] = None + ) -> Dict[str, Any]: + """Parse the output of the model""" + question = clean_question(query) + + if not chat_history: + standalone_query = question + else: + standalone_query = self.standalone_query + + if self.avoid_query: + keywords = [] + sub_queries = [] + vector_search_queries = [] + else: + keywords = [keyword.keyword for keyword in self.keywords] + sub_queries = [sub_query.query for sub_query in self.sub_queries] + vector_search_queries = [ + vector_search_query.query + for vector_search_query in self.vector_search_queries + ] + intents_descriptions = "" + for intent in self.intents: + intents_descriptions += ( + f"{intent.label.value.replace('_', ' ').title()}:" + f"\n\t{intent.reasoning}" + f"\n\t{QUERY_INTENTS[intent.label.value]}\n\n" + ) + + all_queries = ( + [standalone_query] + keywords + sub_queries + vector_search_queries + ) + + return { + "query": query, + "question": question, + "standalone_query": standalone_query, + "intents": intents_descriptions, + "keywords": keywords, + "sub_queries": sub_queries, + "vector_search_queries": vector_search_queries, + "language": self.language, + "avoid_query": self.avoid_query, + "chat_history": chat_history, + "all_queries": all_queries, + } + + +ENHANCER_SYSTEM_PROMPT = ( + "You are a weights & biases support manager tasked with enhancing support questions from users" + "You are given a conversation and a follow-up query. " + "You goal to enhance the user query and render it using the tool provided." + "\n\nChat History: \n\n" + "{chat_history}" +) + +ENHANCER_PROMPT_MESSAGES = [ + ("system", ENHANCER_SYSTEM_PROMPT), + ("human", "Question: {query}"), + ("human", "!!! Tip: Make sure to answer in the correct format"), +] + + +class QueryEnhancer: + model: ChatModel = ChatModel() + fallback_model: ChatModel = ChatModel(max_retries=6) + + def __init__( + self, + model: str = "gpt-4-0125-preview", + fallback_model: str = "gpt-3.5-turbo-1106", + ): + self.model = model # type: ignore + self.fallback_model = fallback_model # type: ignore + self.prompt = ChatPromptTemplate.from_messages(ENHANCER_PROMPT_MESSAGES) + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + base_chain = self._load_chain(self.model) + fallback_chain = self._load_chain(self.fallback_model) + self._chain = base_chain.with_fallbacks([fallback_chain]) + + return self._chain + + def _load_chain(self, model: ChatOpenAI) -> Runnable: + query_enhancer_chain = create_structured_output_runnable( + EnhancedQuery, model, self.prompt + ) + + input_chain = RunnableParallel( + query=RunnablePassthrough(), + chat_history=( + RunnableLambda(lambda x: convert_to_messages(x["chat_history"])) + | RunnableLambda( + lambda x: get_buffer_string(x, "user", "assistant") + ) + ), + ) + + full_query_enhancer_chain = input_chain | query_enhancer_chain + + intermediate_chain = RunnableParallel( + query=itemgetter("query"), + chat_history=itemgetter("chat_history"), + enhanced_query=full_query_enhancer_chain, + ) + chain = intermediate_chain | RunnableLambda( + lambda x: x["enhanced_query"].parse_output( + x["query"], convert_to_messages(x["chat_history"]) + ) + ) + + return chain diff --git a/src/wandbot/rag/query_handler/__init__.py b/src/wandbot/rag/query_handler/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/wandbot/rag/query_handler/base.py b/src/wandbot/rag/query_handler/base.py deleted file mode 100644 index 77767d6..0000000 --- a/src/wandbot/rag/query_handler/base.py +++ /dev/null @@ -1,85 +0,0 @@ -from operator import itemgetter - -import regex as re -from langchain_core.messages import convert_to_messages, messages_to_dict -from langchain_core.runnables import ( - Runnable, - RunnableLambda, - RunnableParallel, - RunnablePassthrough, -) - -from wandbot.rag.query_handler.history_handler import CondenseQuestion -from wandbot.rag.query_handler.intents_enhancer import IntentsEnhancer -from wandbot.rag.query_handler.keyword_search_enhancer import KeywordsEnhancer -from wandbot.rag.query_handler.language_detection import LanguageDetector -from wandbot.rag.query_handler.vector_search_enhancer import ( - VectorSearchEnhancer, -) -from wandbot.rag.query_handler.web_search import YouWebRagSearchEnhancer - -BOT_NAME_PATTERN = re.compile(r"<@U[A-Z0-9]+>|@[a-zA-Z0-9]+") - - -def clean_question(question: str) -> str: - cleaned_query = BOT_NAME_PATTERN.sub("", question).strip() - return cleaned_query - - -class QueryEnhancer: - def __init__( - self, - model: str = "gpt-4-0125-preview", - fallback_model="gpt-3.5-turbo-1106", - ): - self.question_condenser = CondenseQuestion( - model=model, fallback_model=fallback_model - ) - self.intents_enhancer = IntentsEnhancer( - model=model, fallback_model=fallback_model - ) - self.language_detector = LanguageDetector() - self.keywords_enhancer = KeywordsEnhancer( - model=model, fallback_model=fallback_model - ) - self.vector_search_enhancer = VectorSearchEnhancer( - model=model, fallback_model=fallback_model - ) - self.web_search_enhancer = YouWebRagSearchEnhancer() - - self._chain = None - - @property - def chain(self) -> Runnable: - if self._chain is None: - self._chain = self._load_chain() - return self._chain - - def _load_chain(self) -> Runnable: - query_enhancer_chain = ( - RunnablePassthrough().assign( - question=lambda x: clean_question(x["query"]), - ) - | RunnableParallel( - question=itemgetter("question"), - standalone_question=self.question_condenser.chain, - language=self.language_detector.chain, - chat_history=RunnableLambda( - lambda x: convert_to_messages(x["chat_history"]) - ) - | RunnableLambda(lambda x: messages_to_dict(x)), - ) - | self.intents_enhancer.chain - | RunnableParallel( - standalone_question=itemgetter("standalone_question"), - language=itemgetter("language"), - question=itemgetter("question"), - intents=itemgetter("intents"), - chat_history=itemgetter("chat_history"), - keywords=self.keywords_enhancer.chain, - vector_search=self.vector_search_enhancer.chain, - web_results=self.web_search_enhancer.chain, - avoid_query=itemgetter("avoid_query"), - ) - ) - return query_enhancer_chain diff --git a/src/wandbot/rag/query_handler/history_handler.py b/src/wandbot/rag/query_handler/history_handler.py deleted file mode 100644 index f755e00..0000000 --- a/src/wandbot/rag/query_handler/history_handler.py +++ /dev/null @@ -1,85 +0,0 @@ -from _operator import itemgetter -from langchain_core.messages import convert_to_messages, get_buffer_string -from langchain_core.output_parsers import StrOutputParser -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import ( - Runnable, - RunnableBranch, - RunnableLambda, - RunnableParallel, - RunnablePassthrough, -) -from langchain_openai import ChatOpenAI - -from wandbot.rag.utils import ChatModel - -CONDENSE_PROMPT_SYSTEM_TEMPLATE = """Given the following conversation and a follow up question, rephrase the follow up \ -question to be a standalone question. - -Chat History: -{chat_history} -Follow Up Input: {question} -Standalone Question:""" - - -CONDENSE_PROMPT_MESSAGES = [ - ( - "system", - CONDENSE_PROMPT_SYSTEM_TEMPLATE, - ), -] - - -class CondenseQuestion: - model: ChatModel = ChatModel() - fallback_model: ChatModel = ChatModel(max_retries=6) - - def __init__( - self, - model: str = "gpt-4-0125-preview", - fallback_model="gpt-3.5-turbo-1106", - ): - self.model = model # type: ignore - self.fallback_model = fallback_model # type: ignore - self.prompt = ChatPromptTemplate.from_messages(CONDENSE_PROMPT_MESSAGES) - self._chain = None - - @property - def chain(self) -> Runnable: - if self._chain is None: - base_chain = self._load_chain(self.model) - fallback_chain = self._load_chain(self.fallback_model) - self._chain = base_chain.with_fallbacks([fallback_chain]) - - return self._chain - - def _load_chain(self, model: ChatOpenAI) -> Runnable: - base_chain = ( - RunnableParallel( - question=RunnablePassthrough(), - chat_history=( - RunnableLambda( - lambda x: convert_to_messages(x["chat_history"]) - ) - | RunnableLambda( - lambda x: get_buffer_string(x, "user", "assistant") - ) - ), - ) - | self.prompt - | model - | StrOutputParser() - ) - - chain = RunnableBranch( - ( - lambda x: True if x["chat_history"] else False, - base_chain, - ), - ( - lambda x: False if x["chat_history"] else True, - itemgetter("question"), - ), - itemgetter("question"), - ) - return chain diff --git a/src/wandbot/rag/query_handler/intents_enhancer.py b/src/wandbot/rag/query_handler/intents_enhancer.py deleted file mode 100644 index 99e855b..0000000 --- a/src/wandbot/rag/query_handler/intents_enhancer.py +++ /dev/null @@ -1,265 +0,0 @@ -import enum -from operator import itemgetter -from typing import List - -import cohere -from langchain.chains.openai_functions import create_structured_output_runnable -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import ( - Runnable, - RunnableLambda, - RunnableParallel, - RunnablePassthrough, -) -from langchain_openai import ChatOpenAI -from pydantic.v1 import BaseModel, Field -from pydantic_settings import BaseSettings, SettingsConfigDict - -from wandbot.rag.utils import ChatModel - - -class Labels(str, enum.Enum): - UNRELATED = "unrelated" - CODE_TROUBLESHOOTING = "code_troubleshooting" - INTEGRATIONS = "integrations" - PRODUCT_FEATURES = "product_features" - SALES_AND_GTM_RELATED = "sales_and_gtm_related" - BEST_PRACTICES = "best_practices" - COURSE_RELATED = "course_related" - NEEDS_MORE_INFO = "needs_more_info" - OPINION_REQUEST = "opinion_request" - NEFARIOUS_QUERY = "nefarious_query" - OTHER = "other" - - -class Label(BaseModel): - "An intent label to be associated with the query" - - reasoning: str = Field( - ..., - description="The reason for the identifying the intent", - ) - label: Labels = Field( - ..., description="An intent associated with the query" - ) - - -class MultiLabel(BaseModel): - "A list of intents associated with the query" - intents: List[Label] = Field( - ..., - description="The list of intents associated with the query", - min_items=1, - max_items=3, - ) - - -INTENT_DESCRIPTIONS = { - Labels.UNRELATED.value: "The query is not related to Weights & Biases", - Labels.CODE_TROUBLESHOOTING.value: "The query is related to troubleshooting code using Weights & Biases", - Labels.INTEGRATIONS.value: "The query is related to integrating Weights & Biases with other tools, frameworks, " - "or libraries", - Labels.PRODUCT_FEATURES.value: "The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, " - "Reports, Experiments, Tables, Prompts, Launch, Weave, StreamTables and more", - Labels.SALES_AND_GTM_RELATED.value: "The query is related to sales, marketing, or other business related topics " - "such as pricing, billing, or partnerships etc", - Labels.BEST_PRACTICES.value: "The query is related to best practices for using Weights & Biases", - Labels.COURSE_RELATED.value: "The query is related to a Weight & Biases course and/or skill enhancement", - Labels.NEEDS_MORE_INFO.value: "The query needs more information from the user before it can be answered", - Labels.OPINION_REQUEST.value: "The query is asking for an opinion", - Labels.NEFARIOUS_QUERY.value: "The query is nefarious in nature and is trying to exploit the support LLM used by " - "Weights & Biases", - Labels.OTHER.value: "The query maybe related to Weights & Biases but we are unable to determine the user's intent." - " It's best to ask the user to rephrase the query or avoid answering the query", -} -QUERY_INTENTS = { - Labels.UNRELATED.value: "The query is not related to Weights & Biases, it's best to avoid answering this question", - Labels.CODE_TROUBLESHOOTING.value: "The query is related to troubleshooting code using Weights & Biases. Help " - "with a detailed code snippet and explanation", - Labels.INTEGRATIONS.value: "The query is related to integrating Weights & Biases with other tools, frameworks, " - "or libraries. Help with a detailed code snippet and explanation and ask for more information about the " - "integration if needed", - Labels.PRODUCT_FEATURES.value: "The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, " - "Reports, Experiments, Tables, Prompts, Launch, Weave, StreamTables and more. Provide a link to the relevant " - "documentation and explain the feature in detail", - Labels.SALES_AND_GTM_RELATED.value: "The query is related to sales, marketing, or other business related topics " - "such as pricing, billing, or partnerships etc. Ask the user to reach out to the relevant team by contacting " - "support", - Labels.BEST_PRACTICES.value: "The query is related to best practices for using Weights & Biases. Answer the query " - "and provide guidance where necessary", - Labels.COURSE_RELATED.value: "The query is related to a Weight & Biases course and/or skill enhancement. Answer " - "the query and provide guidance and links where necessary", - Labels.NEEDS_MORE_INFO.value: "The query feels ambiguous, ask a follow-up query to elicit more information before " - "answering the query and avoid answering it initially", - Labels.OPINION_REQUEST.value: "The query is asking for an opinion. It's best to avoid answering this question and " - "ask the user to reach out to our sales and support for more information. Always favor Weights & Biases in your " - "response", - Labels.NEFARIOUS_QUERY.value: "The query looks nefarious in nature. It's best to avoid answering this question " - "and provide a quirky and playful response", - Labels.OTHER.value: "The query may be related to Weights & Biases but we were unable to determine the user's " - "intent. It's best to avoid answering this question and ask the user a follow-up query to rephrase their original " - "query", -} - - -def get_intent_descriptions(intents: List[str]) -> str: - descriptions = [] - if not intents: - return "- " + INTENT_DESCRIPTIONS["other"] - - for classification in intents: - description = INTENT_DESCRIPTIONS.get(classification, "") - descriptions.append(description) - descriptions = "- " + "\n- ".join(descriptions) - return descriptions - - -def get_intent_hints(intents: List[str]) -> str: - descriptions = [] - if not intents: - return "- " + QUERY_INTENTS["other"] - - for classification in intents: - description = QUERY_INTENTS.get(classification, "") - descriptions.append(description) - descriptions = "- " + "\n- ".join(descriptions) - return descriptions - - -class CohereClassifierConfig(BaseSettings): - model_config = SettingsConfigDict( - env_file=".env", env_file_encoding="utf-8", extra="allow" - ) - cohere_api_key: str = Field( - ..., - description="The API key for the Cohere API", - env="COHERE_API_KEY", - validation_alias="cohere_api_key", - ) - cohere_query_clf_model: str = Field( - ..., - description="The fine-tuned cohere model to use for classification", - env="COHERE_QUERY_CLF_MODEL", - validation_alias="cohere_query_clf_model", - ) - - -class CohereQueryClassifier: - config: CohereClassifierConfig = CohereClassifierConfig() - - def __init__(self) -> None: - self.client = cohere.Client(self.config.cohere_api_key) - - def __call__(self, query: str) -> str: - response = self.client.classify( - model=self.config.cohere_query_clf_model, - inputs=[query], - ) - return get_intent_descriptions(response.classifications[0].predictions) - - -intents_descriptions_str = "\n".join( - [ - f"{label}:\t{description}" - for label, description in INTENT_DESCRIPTIONS.items() - ] -) - -INTENT_PROMPT_MESSAGES = [ - ( - "system", - """You are a Weights & Biases support manager. Your goal is to enhance the query by identifying one or more intents related to the query. - Here is the mapping of the intents and their descriptions. - """ - + intents_descriptions_str, - ), - ("human", "Enhance the following user query:\n{question}"), - ( - "human", - "Here is an initial list of intent hints that maybe relevant:\n{intent_hints}", - ), - ("human", "Tip: Make sure to answer in the correct format"), -] - - -def check_avoid_intent(intents: List[str]) -> bool: - return any( - [ - intent - in [ - Labels.NEFARIOUS_QUERY.value, - Labels.OPINION_REQUEST.value, - Labels.NEEDS_MORE_INFO.value, - Labels.UNRELATED.value, - ] - for intent in intents - ] - ) - - -class IntentsEnhancer: - model: ChatModel = ChatModel() - fallback_model: ChatModel = ChatModel(max_retries=6) - - def __init__( - self, - model: str = "gpt-4-0125-preview", - fallback_model: str = "gpt-3.5-turbo-1106", - ): - self.model = model # type: ignore - self.fallback_model = fallback_model # type: ignore - - self.cohere_classifier = CohereQueryClassifier() - self.prompt = ChatPromptTemplate.from_messages(INTENT_PROMPT_MESSAGES) - self._chain = None - - @property - def chain(self) -> Runnable: - if self._chain is None: - base_chain = self._load_chain(self.model) - fallback_chain = self._load_chain(self.fallback_model) - self._chain = base_chain.with_fallbacks([fallback_chain]) - - return self._chain - - def _load_chain(self, model: ChatOpenAI) -> Runnable: - # load the cohere classifier chain - - cohere_classify_chain = RunnablePassthrough.assign( - intent_hints=lambda x: self.cohere_classifier(x["question"]) - ) - - # load the intent extraction chain - intents_classification_chain = create_structured_output_runnable( - MultiLabel, model, self.prompt - ) - - intent_extraction_chain = ( - cohere_classify_chain - | intents_classification_chain - | RunnableLambda( - lambda x: [intent.label.value for intent in x.intents] - ) - | RunnableParallel( - intent_hints=get_intent_hints, - intent_labels=RunnablePassthrough(), - ) - ) - - # load the intent enhancement chain - intent_enhancement_chain = RunnableParallel( - question=itemgetter("question"), - standalone_question=itemgetter("standalone_question"), - chat_history=itemgetter("chat_history"), - language=itemgetter("language"), - intents=( - {"question": itemgetter("standalone_question")} - | intent_extraction_chain - ), - ) | RunnablePassthrough.assign( - avoid_query=lambda x: check_avoid_intent( - x["intents"]["intent_labels"] - ) - ) - - return intent_enhancement_chain diff --git a/src/wandbot/rag/query_handler/keyword_search_enhancer.py b/src/wandbot/rag/query_handler/keyword_search_enhancer.py deleted file mode 100644 index a9f63fb..0000000 --- a/src/wandbot/rag/query_handler/keyword_search_enhancer.py +++ /dev/null @@ -1,102 +0,0 @@ -from operator import itemgetter -from typing import List - -from langchain.chains.openai_functions import create_structured_output_runnable -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import ( - Runnable, - RunnableBranch, - RunnableLambda, - RunnableParallel, -) -from langchain_openai import ChatOpenAI -from pydantic.v1 import BaseModel, Field - -from wandbot.rag.utils import ChatModel - -KEYWORDS_SYSTEM_PROMPT = ( - "You are a Weights & Biases support manager. " - "Your goal is to enhance the user query by adding a list of keywords used for web search." -) - -KEYWORDS_PROMPT_MESSAGES = [ - ("system", KEYWORDS_SYSTEM_PROMPT), - ( - "human", - "Enhance the following query related to weights and biases for web search.:\n\n{question}", - ), - ("human", "Tip: Make sure to answer in the correct format"), -] - - -class Keyword(BaseModel): - """A Keyword to search for on the wen""" - - keyword: str = Field( - ..., - description="A search term for getting the most relevant information required to answer the query", - ) - - -class KeywordsSchema(BaseModel): - "A list of search keywords to enhance the search query" - keywords: List[Keyword] = Field( - ..., - description="List of five different search terms", - min_items=0, - max_items=5, - ) - - -class KeywordsEnhancer: - model: ChatModel = ChatModel() - fallback_model: ChatModel = ChatModel(max_retries=6) - - def __init__( - self, - model: str = "gpt-4-0125-preview", - fallback_model: str = "gpt-3.5-turbo-1106", - ): - self.model = model # type: ignore - self.fallback_model = fallback_model # type: ignore - self.prompt = ChatPromptTemplate.from_messages(KEYWORDS_PROMPT_MESSAGES) - self._chain = None - - @property - def chain(self) -> Runnable: - if self._chain is None: - base_chain = self._load_chain(self.model) - fallback_chain = self._load_chain(self.fallback_model) - self._chain = base_chain.with_fallbacks([fallback_chain]) - - return self._chain - - def _load_chain(self, model: ChatOpenAI) -> Runnable: - keywords_extraction_chain = create_structured_output_runnable( - KeywordsSchema, model, self.prompt - ) - - keywords_chain = keywords_extraction_chain | RunnableLambda( - lambda x: [keyword.keyword for keyword in x.keywords] - ) - - branch = RunnableBranch( - ( - lambda x: x["avoid"], - RunnableLambda(lambda x: []), - ), - ( - lambda x: not x["avoid"], - keywords_chain, - ), - RunnableLambda(lambda x: []), - ) - - chain = ( - RunnableParallel( - question=itemgetter("standalone_question"), - avoid=itemgetter("avoid_query"), - ) - | branch - ) - return chain diff --git a/src/wandbot/rag/query_handler/language_detection.py b/src/wandbot/rag/query_handler/language_detection.py deleted file mode 100644 index fdfaf6a..0000000 --- a/src/wandbot/rag/query_handler/language_detection.py +++ /dev/null @@ -1,20 +0,0 @@ -from operator import itemgetter - -from langchain_core.runnables import Runnable, RunnablePassthrough - -from wandbot.utils import FastTextLangDetect, FasttextModelConfig - - -class LanguageDetector: - model_config = FasttextModelConfig() - - def __init__(self): - self.model = FastTextLangDetect(self.model_config) - self._chain = None - - @property - def chain(self) -> Runnable: - lang_detect_chain = RunnablePassthrough().assign( - language=lambda x: self.model.detect_language(x["question"]) - ) | itemgetter("language") - return lang_detect_chain diff --git a/src/wandbot/rag/query_handler/vector_search_enhancer.py b/src/wandbot/rag/query_handler/vector_search_enhancer.py deleted file mode 100644 index 6b03509..0000000 --- a/src/wandbot/rag/query_handler/vector_search_enhancer.py +++ /dev/null @@ -1,87 +0,0 @@ -from operator import itemgetter - -from langchain.chains.openai_functions import create_structured_output_runnable -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import ( - Runnable, - RunnableBranch, - RunnableLambda, - RunnableParallel, -) -from langchain_openai import ChatOpenAI -from pydantic.v1 import BaseModel, Field - -from wandbot.rag.utils import ChatModel - -QUERY_REWRITE_SYSTEM_PROMPT = ( - "You are a Weights & Biases support manager. " - "Your goal is to enhance the user query by rewriting it for similarity search. " - "Rewrite the given query into a clear, specific, and formal request for retrieving relevant information from a vector database" -) - -QUERY_REWRITE_PROMPT_MESSAGES = [ - ("system", QUERY_REWRITE_SYSTEM_PROMPT), - ("human", "Enhance the following query i.:\n\n{question}"), - ("human", "Tip: Make sure to answer in the correct format"), -] - - -class EnhancedQuery(BaseModel): - "A query suitable for similarity search in a vectorstore" - query_str: str = Field( - ..., description="A query suitable for similarity search and retrieval" - ) - - -class VectorSearchEnhancer: - model: ChatModel = ChatModel() - fallback_model: ChatModel = ChatModel(max_retries=6) - - def __init__( - self, - model: str = "gpt-4-0125-preview", - fallback_model: str = "gpt-3.5-turbo-1106", - ): - self.model = model # type: ignore - self.fallback_model = fallback_model # type: ignore - self.prompt = ChatPromptTemplate.from_messages( - QUERY_REWRITE_PROMPT_MESSAGES - ) - self._chain = None - - @property - def chain(self) -> Runnable: - if self._chain is None: - base_chain = self._load_chain(self.model) - fallback_chain = self._load_chain(self.fallback_model) - self._chain = base_chain.with_fallbacks([fallback_chain]) - return self._chain - - def _load_chain(self, model: ChatOpenAI) -> Runnable: - query_rewrite_chain = create_structured_output_runnable( - EnhancedQuery, model, self.prompt - ) - question_rewrite_chain = query_rewrite_chain | RunnableLambda( - lambda x: x.query_str - ) - - branch = RunnableBranch( - ( - lambda x: x["avoid"], - RunnableLambda(lambda x: []), - ), - ( - lambda x: not x["avoid"], - question_rewrite_chain, - ), - RunnableLambda(lambda x: []), - ) - - chain = ( - RunnableParallel( - question=itemgetter("standalone_question"), - avoid=itemgetter("avoid_query"), - ) - | branch - ) - return chain diff --git a/src/wandbot/rag/response_synthesis/base.py b/src/wandbot/rag/response_synthesis.py similarity index 91% rename from src/wandbot/rag/response_synthesis/base.py rename to src/wandbot/rag/response_synthesis.py index 65de501..6ebd145 100644 --- a/src/wandbot/rag/response_synthesis/base.py +++ b/src/wandbot/rag/response_synthesis.py @@ -4,11 +4,10 @@ from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel from langchain_openai import ChatOpenAI - from wandbot.rag.utils import ChatModel, combine_documents, create_query_str -RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """As Wandbot - a support expert in Weights & Biases, wandb and weave. -Your goal to ensure customer success with questions related to Weight & Biases, `wandb`, and the visualization library `weave` +RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """You are Wandbot - a support expert in Weights & Biases, wandb and weave. +Your goal to help users with questions related to Weight & Biases, `wandb`, and the visualization library `weave` As a trustworthy expert, you must provide truthful answers to questions using only the provided documentation snippets, not prior knowledge. Here are guidelines you must follow when responding to user questions: @@ -110,28 +109,10 @@ # "assistant", # "Haha, Nice try. But I'm not falling for that. It looks like your question is not related to Weights & Biases. I'm here to assist with wandb-related queries. Please ask a wandb-specific question, and I'll do my best to help you. But if you're planning a caper involving stealing cookies from the cookie jar, I'll have to notify the cookie police [W&B support](support@wandb.com) – they're tough, always crumbly under pressure! 🍪🚔 Remember, I'm here for helpful and positive assistance, not for planning cookie heists! 🛡️😄", # ), - ( - "human", - "Question: {query_str}\n\n", - ), + ("human", "{query_str}"), ] -# TODO: Add citation sources using function calling api -# https://python.langchain.com/docs/use_cases/question_answering/citations#cite-documents -# class cited_answer(BaseModel): -# """Answer the user question based only on the given sources, and cite the sources used.""" -# -# answer: str = Field( -# ..., -# description="The answer to the user question, which is based only on the given sources.", -# ) -# citations: List[int] = Field( -# ..., -# description="The integer IDs of the SPECIFIC sources which justify the answer.", -# ) - - class ResponseSynthesizer: model: ChatModel = ChatModel() fallback_model: ChatModel = ChatModel(max_retries=6) @@ -160,7 +141,7 @@ def _load_chain(self, model: ChatOpenAI) -> Runnable: response_synthesis_chain = ( RunnableLambda( lambda x: { - "query_str": create_query_str(x["query"]), + "query_str": create_query_str(x), "context_str": combine_documents(x["context"]), } ) diff --git a/src/wandbot/rag/response_synthesis/__init__.py b/src/wandbot/rag/response_synthesis/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/wandbot/rag/retrieval.py b/src/wandbot/rag/retrieval.py new file mode 100644 index 0000000..2d62b22 --- /dev/null +++ b/src/wandbot/rag/retrieval.py @@ -0,0 +1,102 @@ +from typing import List + +from langchain.retrievers.document_compressors import CohereRerank +from langchain_core.documents import Document +from langchain_core.runnables import Runnable, RunnablePassthrough +from wandbot.ingestion.config import VectorStoreConfig +from wandbot.rag.utils import get_web_contexts +from wandbot.retriever.base import VectorStore +from wandbot.retriever.web_search import YouSearch, YouSearchConfig + + +def reciprocal_rank_fusion(results: list[list[Document]], k=60): + text_to_doc = {} + fused_scores = {} + for docs in results: + # Assumes the docs are returned in sorted order of relevance + for rank, doc in enumerate(docs): + doc_content = doc.page_content + text_to_doc[doc_content] = doc + if doc_content not in fused_scores: + fused_scores[doc_content] = 0.0 + fused_scores[doc_content] += 1 / (rank + k) + + ranked_results = dict( + sorted(fused_scores.items(), key=lambda x: x[1], reverse=True) + ) + + ranked_results = [text_to_doc[text] for text in ranked_results.keys()] + return ranked_results + + +def run_web_search(query, avoid=False) -> list: + if avoid: + return [] + yousearch = YouSearch(YouSearchConfig()) + web_results = yousearch(query) + return get_web_contexts(web_results) + + +def rerank_results( + queries: List[str], + context: List[Document], + top_k: int = 5, + language: str = "en", +): + if language == "en": + reranker = CohereRerank(top_n=top_k, model="rerank-english-v2.0") + else: + reranker = CohereRerank(top_n=top_k, model="rerank-multilingual-v2.0") + + query = "\n".join(queries) + ranked_results = reranker.compress_documents(documents=context, query=query) + return ranked_results + + +class FusionRetrieval: + def __init__( + self, + vector_store_config: VectorStoreConfig, + top_k: int = 5, + search_type: str = "mmr", + ): + self.vectorstore = VectorStore.from_config(vector_store_config) + self.top_k = top_k + + self.retriever = self.vectorstore.as_parent_retriever( + search_type=search_type, search_kwargs={"k": self.top_k} + ) + + self._chain = None + + @property + def chain(self) -> Runnable: + if self._chain is None: + self._chain = ( + RunnablePassthrough().assign( + docs_context=lambda x: self.retriever.batch( + x["all_queries"] + ), + web_context=lambda x: run_web_search( + x["standalone_query"], x["avoid_query"] + ), + ) + | RunnablePassthrough().assign( + full_context=lambda x: x["docs_context"] + + [x["web_context"]] + ) + | RunnablePassthrough().assign( + fused_context=lambda x: reciprocal_rank_fusion( + x["full_context"] + ) + ) + | RunnablePassthrough().assign( + context=lambda x: rerank_results( + [x["standalone_query"]], + x["fused_context"], + self.top_k, + x["language"], + ) + ) + ) + return self._chain diff --git a/src/wandbot/rag/retrieval/__init__.py b/src/wandbot/rag/retrieval/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/wandbot/rag/retrieval/base.py b/src/wandbot/rag/retrieval/base.py deleted file mode 100644 index 71b8704..0000000 --- a/src/wandbot/rag/retrieval/base.py +++ /dev/null @@ -1,145 +0,0 @@ -from operator import itemgetter - -from langchain.load import dumps, loads -from langchain_community.document_transformers import EmbeddingsRedundantFilter -from langchain_core.runnables import ( - Runnable, - RunnableBranch, - RunnableLambda, - RunnableParallel, - RunnablePassthrough, -) - -from wandbot.ingestion.config import VectorStoreConfig -from wandbot.rag.utils import get_web_contexts, process_input_for_retrieval -from wandbot.retriever import OpenAIEmbeddingsModel, VectorStore -from wandbot.retriever.reranking import CohereRerankChain - - -def reciprocal_rank_fusion(results: list[list], k=60): - fused_scores = {} - for docs in results: - # Assumes the docs are returned in sorted order of relevance - for rank, doc in enumerate(docs): - doc_str = dumps(doc) - if doc_str not in fused_scores: - fused_scores[doc_str] = 0 - previous_score = fused_scores[doc_str] - fused_scores[doc_str] += 1 / (rank + k) - - ranked_results = [ - (loads(doc), score) - for doc, score in sorted( - fused_scores.items(), key=lambda x: x[1], reverse=True - ) - ] - return [item[0] for item in ranked_results] - - -class RagRetrievalChain: - def __init__(self, field: str = "question"): - self.field = field - - def __set_name__(self, owner, name): - self.public_name = name - self.private_name = "_" + name - - def __get__(self, obj, obj_type=None): - if getattr(obj, "retriever") is None: - raise AttributeError( - "Retriever must be set before setting retrieval chain" - ) - default_input_chain = ( - itemgetter("standalone_question") - | RunnablePassthrough() - | process_input_for_retrieval - | RunnableParallel(context=obj.retriever) - | itemgetter("context") - ) - - input_chain = ( - itemgetter(self.field) - | RunnablePassthrough() - | process_input_for_retrieval - | RunnableParallel(context=obj.retriever) - | itemgetter("context") - ) - - retrieval_chain = RunnableBranch( - ( - lambda x: not x["avoid_query"], - input_chain, - ), - ( - lambda x: x["avoid_query"], - default_input_chain, - ), - default_input_chain, - ) - return retrieval_chain - - -class FusionRetrieval: - question_chain = RagRetrievalChain("question") - standalone_question_chain = RagRetrievalChain("standalone_question") - keywords_chain = RagRetrievalChain("keywords") - vector_search_chain = RagRetrievalChain("vector_search") - web_context_chain = RunnableLambda( - lambda x: get_web_contexts(x["web_results"]) - ) - cohere_rerank_chain = CohereRerankChain() - embeddings_model: OpenAIEmbeddingsModel = OpenAIEmbeddingsModel( - dimensions=768 - ) - - def __init__( - self, - vector_store_config: VectorStoreConfig, - top_k=5, - search_type="mmr", - ): - self.vector_store = VectorStore.from_config(vector_store_config) - - self.retriever = self.vector_store.as_parent_retriever( - search_type=search_type, search_kwargs={"k": top_k * 4} - ) - self.embeddings_model = vector_store_config.embeddings_model # type: ignore - self.top_k = top_k - self.redundant_filter = EmbeddingsRedundantFilter( - embeddings=self.embeddings_model - ).transform_documents - - self._chain = None - - @property - def chain(self) -> Runnable: - if self._chain is None: - combined_retrieval_chain = ( - RunnableParallel( - question=self.question_chain, - standalone_question=self.standalone_question_chain, - keywords=self.keywords_chain, - vector_search=self.vector_search_chain, - web_context=self.web_context_chain, - ) - | itemgetter( - "question", - "standalone_question", - "keywords", - "vector_search", - "web_context", - ) - | reciprocal_rank_fusion - | self.redundant_filter - ) - - self._chain = ( - RunnableParallel( - context=combined_retrieval_chain, - question=itemgetter("question"), - language=itemgetter("language"), - ) - | self.cohere_rerank_chain - ) - - return self._chain diff --git a/src/wandbot/rag/utils.py b/src/wandbot/rag/utils.py index eb25406..e378f60 100644 --- a/src/wandbot/rag/utils.py +++ b/src/wandbot/rag/utils.py @@ -3,7 +3,6 @@ from langchain_core.documents import Document from langchain_core.prompts import PromptTemplate, format_document from langchain_openai import ChatOpenAI - from wandbot.utils import clean_document_content @@ -30,19 +29,34 @@ def __set__(self, obj, value): DEFAULT_QUESTION_PROMPT = PromptTemplate.from_template( - template="{page_content}\nlanguage: {language}\nintents: {intents}" + template="""# Query + +{page_content} + +--- + +# Query Metadata + +Language: {language} + +Intents: + +{intents} + +Sub-queries to consider answering: + +{sub_queries} +""" ) def create_query_str(enhanced_query, document_prompt=DEFAULT_QUESTION_PROMPT): - page_content = enhanced_query["standalone_question"] + page_content = enhanced_query["standalone_query"] metadata = { "language": enhanced_query["language"], - "intents": ( - enhanced_query["intents"] - if enhanced_query["language"] == "en" - else None - ), + "intents": enhanced_query["intents"], + "sub_queries": "\t" + + "\n\t".join(enhanced_query["sub_queries"]).strip(), } doc = Document(page_content=page_content, metadata=metadata) doc = clean_document_content(doc) diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index e2e0344..aff6320 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,12 +1,11 @@ from operator import itemgetter from typing import List +import wandb from langchain_community.document_transformers import EmbeddingsRedundantFilter from langchain_community.vectorstores.chroma import Chroma from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda, RunnableParallel - -import wandb from wandbot.ingestion.config import VectorStoreConfig from wandbot.retriever.reranking import CohereRerankChain from wandbot.retriever.utils import OpenAIEmbeddingsModel @@ -129,9 +128,9 @@ def __call__( RunnableParallel( question=itemgetter("question"), language=itemgetter("language"), - context=itemgetter("question") - | retriever - | self.redundant_filter, + context=( + itemgetter("question") | retriever | self.redundant_filter + ), ) | self.cohere_rerank_chain ) diff --git a/src/wandbot/retriever/utils.py b/src/wandbot/retriever/utils.py index a4abc14..371a750 100644 --- a/src/wandbot/retriever/utils.py +++ b/src/wandbot/retriever/utils.py @@ -16,6 +16,7 @@ def __get__(self, obj, obj_type=None): def __set__(self, obj, value): model = OpenAIEmbeddings( model=value, + tiktoken_model_name="text-embedding-ada-002", dimensions=self.dimensions, ) setattr(obj, self.private_name, model) diff --git a/src/wandbot/rag/query_handler/web_search.py b/src/wandbot/retriever/web_search.py similarity index 79% rename from src/wandbot/rag/query_handler/web_search.py rename to src/wandbot/retriever/web_search.py index 86be6d0..c3963f2 100644 --- a/src/wandbot/rag/query_handler/web_search.py +++ b/src/wandbot/retriever/web_search.py @@ -1,14 +1,6 @@ -from operator import itemgetter from typing import Any, Dict, List import requests -from langchain_core.runnables import ( - Runnable, - RunnableBranch, - RunnableLambda, - RunnableParallel, - RunnablePassthrough, -) from pydantic import BaseModel, Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -43,6 +35,10 @@ class YouSearchConfig(BaseSettings): class YouSearch: config: YouSearchConfig = YouSearchConfig() + def __init__(self, config: YouSearchConfig = None): + if config is not None: + self.config = config + def _rag(self, query: str) -> YouSearchResults: """Retrieve.""" try: @@ -141,37 +137,3 @@ def __call__( else: web_results = self._retrieve(question) return web_results.dict() - - -class YouWebRagSearchEnhancer: - def __init__(self): - self.you_search = YouSearch() - self._chain = None - - @property - def chain(self) -> Runnable: - if self._chain is None: - search_chain = RunnablePassthrough().assign( - web_results=lambda x: self.you_search(question=x["question"]) - ) - - branch = RunnableBranch( - ( - lambda x: x["avoid"], - RunnableLambda(lambda x: None), - ), - ( - lambda x: not x["avoid"], - search_chain | itemgetter("web_results"), - ), - RunnableLambda(lambda x: None), - ) - - self._chain = ( - RunnableParallel( - question=itemgetter("standalone_question"), - avoid=itemgetter("avoid_query"), - ) - | branch - ) - return self._chain From 80029c5efe4d55155bfad67cc76bba3c57e5e650 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 16 Feb 2024 20:28:04 +0530 Subject: [PATCH 34/41] chore: run formatters and linters --- src/wandbot/chat/chat.py | 3 ++- src/wandbot/chat/rag.py | 1 + src/wandbot/rag/query_handler.py | 1 + src/wandbot/rag/response_synthesis.py | 1 + src/wandbot/rag/retrieval.py | 1 + src/wandbot/rag/utils.py | 1 + src/wandbot/retriever/base.py | 3 ++- 7 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index fd22524..d38ffb3 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -26,6 +26,8 @@ """ from typing import List +from weave.monitoring import StreamTable + import wandb from wandbot.chat.config import ChatConfig from wandbot.chat.rag import Pipeline, PipelineOutput @@ -33,7 +35,6 @@ from wandbot.database.schemas import QuestionAnswer from wandbot.ingestion.config import VectorStoreConfig from wandbot.utils import Timer, get_logger -from weave.monitoring import StreamTable logger = get_logger(__name__) diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 877d188..4a9822f 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -3,6 +3,7 @@ from langchain_community.callbacks import get_openai_callback from pydantic import BaseModel + from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag.query_handler import QueryEnhancer from wandbot.rag.response_synthesis import ResponseSynthesizer diff --git a/src/wandbot/rag/query_handler.py b/src/wandbot/rag/query_handler.py index 3ecc390..e311fc7 100644 --- a/src/wandbot/rag/query_handler.py +++ b/src/wandbot/rag/query_handler.py @@ -15,6 +15,7 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field + from wandbot.rag.utils import ChatModel from wandbot.utils import get_logger diff --git a/src/wandbot/rag/response_synthesis.py b/src/wandbot/rag/response_synthesis.py index 6ebd145..ba378e2 100644 --- a/src/wandbot/rag/response_synthesis.py +++ b/src/wandbot/rag/response_synthesis.py @@ -4,6 +4,7 @@ from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel from langchain_openai import ChatOpenAI + from wandbot.rag.utils import ChatModel, combine_documents, create_query_str RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """You are Wandbot - a support expert in Weights & Biases, wandb and weave. diff --git a/src/wandbot/rag/retrieval.py b/src/wandbot/rag/retrieval.py index 2d62b22..cdd1d8c 100644 --- a/src/wandbot/rag/retrieval.py +++ b/src/wandbot/rag/retrieval.py @@ -3,6 +3,7 @@ from langchain.retrievers.document_compressors import CohereRerank from langchain_core.documents import Document from langchain_core.runnables import Runnable, RunnablePassthrough + from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag.utils import get_web_contexts from wandbot.retriever.base import VectorStore diff --git a/src/wandbot/rag/utils.py b/src/wandbot/rag/utils.py index e378f60..f2270a4 100644 --- a/src/wandbot/rag/utils.py +++ b/src/wandbot/rag/utils.py @@ -3,6 +3,7 @@ from langchain_core.documents import Document from langchain_core.prompts import PromptTemplate, format_document from langchain_openai import ChatOpenAI + from wandbot.utils import clean_document_content diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index aff6320..20a7d33 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,11 +1,12 @@ from operator import itemgetter from typing import List -import wandb from langchain_community.document_transformers import EmbeddingsRedundantFilter from langchain_community.vectorstores.chroma import Chroma from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda, RunnableParallel + +import wandb from wandbot.ingestion.config import VectorStoreConfig from wandbot.retriever.reranking import CohereRerankChain from wandbot.retriever.utils import OpenAIEmbeddingsModel From 2088f51d8b677932b3ad837628c709353bed1770 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 16 Feb 2024 20:56:28 +0530 Subject: [PATCH 35/41] fix: run.sh to add zendesk app correctly --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 04cee47..7c3ba9e 100755 --- a/run.sh +++ b/run.sh @@ -1,5 +1,5 @@ (uvicorn wandbot.api.app:app --host="0.0.0.0" --port=8000) & \ (python -m wandbot.apps.slack -l en) & \ (python -m wandbot.apps.slack -l ja) & \ -(python -m wandbot.apps.discord) +(python -m wandbot.apps.discord) & \ (python -m wandbot.apps.zendesk) \ No newline at end of file From 074923eea90c44929b2e0a3a1a1d4442548557c8 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Wed, 21 Feb 2024 10:11:59 +0530 Subject: [PATCH 36/41] fix: use single vectorstore across retrieval and chat --- README.md | 4 ++-- src/wandbot/api/app.py | 9 +++++---- src/wandbot/chat/chat.py | 25 +++++++++++++------------ src/wandbot/chat/config.py | 19 +++---------------- src/wandbot/chat/rag.py | 14 +++++++------- src/wandbot/rag/retrieval.py | 9 ++++----- src/wandbot/retriever/base.py | 24 ++++++++++++++---------- 7 files changed, 48 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 545bfaf..904a5fe 100644 --- a/README.md +++ b/README.md @@ -111,8 +111,8 @@ We employed a few auto evaluation strategies to speed up the iteration process o ## Overview of the Implementation -1. Creating Document Embeddings with FAISS -2. Constructing the Q&A Pipeline using llama-index +1. Creating Document Embeddings with ChromaDB +2. Constructing the Q&A RAGPipeline 3. Selection of Models and Implementation of Fallback Mechanism 4. Deployment of the Q&A Bot on FastAPI, Discord, and Slack 5. Utilizing Weights & Biases Tables for Logging and Analysis diff --git a/src/wandbot/api/app.py b/src/wandbot/api/app.py index 287de41..a8b0ffd 100644 --- a/src/wandbot/api/app.py +++ b/src/wandbot/api/app.py @@ -33,13 +33,13 @@ from datetime import datetime, timezone import pandas as pd -from fastapi import FastAPI - import wandb +from fastapi import FastAPI from wandbot.api.routers import chat as chat_router from wandbot.api.routers import database as database_router from wandbot.api.routers import retrieve as retrieve_router from wandbot.ingestion.config import VectorStoreConfig +from wandbot.retriever import VectorStore from wandbot.utils import get_logger logger = get_logger(__name__) @@ -56,10 +56,11 @@ async def lifespan(app: FastAPI): Returns: None """ - chat_router.chat = chat_router.Chat(chat_router.chat_config) + vector_store = VectorStore.from_config(VectorStoreConfig()) + chat_router.chat = chat_router.Chat(vector_store=vector_store) database_router.db_client = database_router.DatabaseClient() retrieve_router.retriever = retrieve_router.SimpleRetrievalEngine( - VectorStoreConfig() + vector_store=vector_store ) async def backup_db(): diff --git a/src/wandbot/chat/chat.py b/src/wandbot/chat/chat.py index d38ffb3..ff0f0d7 100644 --- a/src/wandbot/chat/chat.py +++ b/src/wandbot/chat/chat.py @@ -26,15 +26,14 @@ """ from typing import List -from weave.monitoring import StreamTable - import wandb from wandbot.chat.config import ChatConfig -from wandbot.chat.rag import Pipeline, PipelineOutput +from wandbot.chat.rag import RAGPipeline, RAGPipelineOutput from wandbot.chat.schemas import ChatRequest, ChatResponse from wandbot.database.schemas import QuestionAnswer -from wandbot.ingestion.config import VectorStoreConfig +from wandbot.retriever import VectorStore from wandbot.utils import Timer, get_logger +from weave.monitoring import StreamTable logger = get_logger(__name__) @@ -45,19 +44,21 @@ class Chat: Attributes: config: An instance of ChatConfig containing configuration settings. run: An instance of wandb.Run for logging experiment information. - wandb_callback: An instance of WandbCallbackHandler for handling Wandb callbacks. - token_counter: An instance of TokenCountingHandler for counting tokens. - callback_manager: An instance of CallbackManager for managing callbacks. - qa_prompt: A string representing the chat prompt. """ - def __init__(self, config: ChatConfig): + config: ChatConfig = ChatConfig() + + def __init__( + self, vector_store: VectorStore, config: ChatConfig | None = None + ): """Initializes the Chat instance. Args: config: An instance of ChatConfig containing configuration settings. """ - self.config = config + self.vector_store = vector_store + if config is not None: + self.config = config self.run = wandb.init( project=self.config.wandb_project, entity=self.config.wandb_entity, @@ -70,11 +71,11 @@ def __init__(self, config: ChatConfig): entity_name=self.config.wandb_entity, ) - self.rag_pipeline = Pipeline(VectorStoreConfig()) + self.rag_pipeline = RAGPipeline(vector_store=vector_store) def _get_answer( self, question: str, chat_history: List[QuestionAnswer] - ) -> PipelineOutput: + ) -> RAGPipelineOutput: history = [] for item in chat_history: history.append(("user", item.question)) diff --git a/src/wandbot/chat/config.py b/src/wandbot/chat/config.py index 3e50f76..7bb851f 100644 --- a/src/wandbot/chat/config.py +++ b/src/wandbot/chat/config.py @@ -19,26 +19,13 @@ class ChatConfig(BaseSettings): - chat_model_name: str = "gpt-4-0125-preview" - max_retries: int = 2 - fallback_model_name: str = "gpt-3.5-turbo-1106" - max_fallback_retries: int = 6 - chat_temperature: float = 0.1 - chat_prompt: pathlib.Path = pathlib.Path("data/prompts/chat_prompt.json") - lang_detect_path: str = "data/cache/models/lid.176.bin" + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="allow" + ) index_artifact: str = Field( "wandbot/wandbot-dev/wandbot_index:latest", env="WANDB_INDEX_ARTIFACT", validation_alias="wandb_index_artifact", ) - embeddings_model: str = "text-embedding-3-small" - embeddings_dim: int = 512 - verbose: bool = False wandb_project: str | None = Field("wandbot_public", env="WANDB_PROJECT") wandb_entity: str | None = Field("wandbot", env="WANDB_ENTITY") - include_sources: bool = True - query_tokens_threshold: int = 1024 - - model_config = SettingsConfigDict( - env_file=".env", env_file_encoding="utf-8", extra="allow" - ) diff --git a/src/wandbot/chat/rag.py b/src/wandbot/chat/rag.py index 4a9822f..10590ba 100644 --- a/src/wandbot/chat/rag.py +++ b/src/wandbot/chat/rag.py @@ -3,11 +3,10 @@ from langchain_community.callbacks import get_openai_callback from pydantic import BaseModel - -from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag.query_handler import QueryEnhancer from wandbot.rag.response_synthesis import ResponseSynthesizer from wandbot.rag.retrieval import FusionRetrieval +from wandbot.retriever import VectorStore from wandbot.utils import Timer, get_logger logger = get_logger(__name__) @@ -30,7 +29,7 @@ def get_stats_dict_from_timer(timer): } -class PipelineOutput(BaseModel): +class RAGPipelineOutput(BaseModel): question: str answer: str sources: str @@ -45,16 +44,17 @@ class PipelineOutput(BaseModel): end_time: datetime.datetime -class Pipeline: +class RAGPipeline: def __init__( self, - vector_store_config: VectorStoreConfig, + vector_store: VectorStore, top_k: int = 5, search_type: str = "mmr", ): + self.vector_store = vector_store self.query_enhancer = QueryEnhancer() self.retrieval = FusionRetrieval( - vector_store_config, top_k=top_k, search_type=search_type + vector_store=vector_store, top_k=top_k, search_type=search_type ) self.response_synthesizer = ResponseSynthesizer() @@ -75,7 +75,7 @@ def __call__( with get_openai_callback() as response_cb, Timer() as response_tb: response = self.response_synthesizer.chain.invoke(retrieval_results) - output = PipelineOutput( + output = RAGPipelineOutput( question=enhanced_query["standalone_query"], answer=response["response"], sources="\n".join( diff --git a/src/wandbot/rag/retrieval.py b/src/wandbot/rag/retrieval.py index cdd1d8c..065c791 100644 --- a/src/wandbot/rag/retrieval.py +++ b/src/wandbot/rag/retrieval.py @@ -3,8 +3,6 @@ from langchain.retrievers.document_compressors import CohereRerank from langchain_core.documents import Document from langchain_core.runnables import Runnable, RunnablePassthrough - -from wandbot.ingestion.config import VectorStoreConfig from wandbot.rag.utils import get_web_contexts from wandbot.retriever.base import VectorStore from wandbot.retriever.web_search import YouSearch, YouSearchConfig @@ -57,15 +55,16 @@ def rerank_results( class FusionRetrieval: def __init__( self, - vector_store_config: VectorStoreConfig, + vector_store: VectorStore, top_k: int = 5, search_type: str = "mmr", ): - self.vectorstore = VectorStore.from_config(vector_store_config) + self.vectorstore = vector_store self.top_k = top_k + self.search_type = search_type self.retriever = self.vectorstore.as_parent_retriever( - search_type=search_type, search_kwargs={"k": self.top_k} + search_type=self.search_type, search_kwargs={"k": self.top_k} ) self._chain = None diff --git a/src/wandbot/retriever/base.py b/src/wandbot/retriever/base.py index 20a7d33..b33ffe1 100644 --- a/src/wandbot/retriever/base.py +++ b/src/wandbot/retriever/base.py @@ -1,12 +1,11 @@ from operator import itemgetter from typing import List +import wandb from langchain_community.document_transformers import EmbeddingsRedundantFilter from langchain_community.vectorstores.chroma import Chroma from langchain_core.documents import Document from langchain_core.runnables import RunnableLambda, RunnableParallel - -import wandb from wandbot.ingestion.config import VectorStoreConfig from wandbot.retriever.reranking import CohereRerankChain from wandbot.retriever.utils import OpenAIEmbeddingsModel @@ -16,10 +15,17 @@ class VectorStore: embeddings_model: OpenAIEmbeddingsModel = OpenAIEmbeddingsModel( dimensions=512 ) + config: VectorStoreConfig = VectorStoreConfig() def __init__( - self, embeddings_model: str, collection_name: str, persist_dir: str + self, + embeddings_model: str, + collection_name: str, + persist_dir: str, + config: VectorStoreConfig = None, ): + if config is not None: + self.config = config self.embeddings_model = embeddings_model # type: ignore self.vectorstore = Chroma( collection_name=collection_name, @@ -34,6 +40,7 @@ def from_config(cls, config: VectorStoreConfig): embeddings_model=config.embeddings_model, collection_name=config.name, persist_dir=str(config.persist_dir), + config=config, ) if wandb.run is None: api = wandb.Api() @@ -46,6 +53,7 @@ def from_config(cls, config: VectorStoreConfig): embeddings_model=config.embeddings_model, collection_name=config.name, persist_dir=str(config.persist_dir), + config=config, ) def as_retriever(self, search_type="mmr", search_kwargs=None): @@ -81,13 +89,9 @@ class SimpleRetrievalEngine: dimensions=768 ) - def __init__( - self, - vector_store_config: VectorStoreConfig, - top_k=5, - ): - self.vector_store = VectorStore.from_config(vector_store_config) - self.embeddings_model = vector_store_config.embeddings_model # type: ignore + def __init__(self, vector_store: VectorStore, top_k=5): + self.vector_store = vector_store + self.embeddings_model = vector_store.config.embeddings_model # type: ignore self.redundant_filter = EmbeddingsRedundantFilter( embeddings=self.embeddings_model ).transform_documents From 6871a78e0e4efe48c33c9a40467e80ce2aa59a25 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 15 Mar 2024 14:40:55 +0530 Subject: [PATCH 37/41] feat: change embeddings to large model --- src/wandbot/ingestion/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/wandbot/ingestion/config.py b/src/wandbot/ingestion/config.py index 771d092..7f3f01d 100644 --- a/src/wandbot/ingestion/config.py +++ b/src/wandbot/ingestion/config.py @@ -18,7 +18,6 @@ from pydantic import BaseModel, Field, model_validator from pydantic_settings import BaseSettings - from wandbot.utils import get_logger logger = get_logger(__name__) @@ -241,7 +240,7 @@ def _set_cache_paths(cls, values: "DataStoreConfig") -> "DataStoreConfig": class VectorStoreConfig(BaseSettings): name: str = "vectorstore" - embeddings_model: str = "text-embedding-3-small" + embeddings_model: str = "text-embedding-3-large" embedding_dim: int = 512 persist_dir: pathlib.Path = pathlib.Path("data/cache/vectorstore") batch_size: int = 256 From 26deea85c68b9746cc24584bac99d03d646223f8 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 15 Mar 2024 14:43:39 +0530 Subject: [PATCH 38/41] fix: clean up commented out code --- src/wandbot/ingestion/prepare_data.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/wandbot/ingestion/prepare_data.py b/src/wandbot/ingestion/prepare_data.py index c241a78..ed8c6e0 100644 --- a/src/wandbot/ingestion/prepare_data.py +++ b/src/wandbot/ingestion/prepare_data.py @@ -23,13 +23,12 @@ import nbformat import pandas as pd +import wandb from google.cloud import bigquery from langchain.schema import Document from langchain_community.document_loaders import TextLoader from langchain_community.document_loaders.base import BaseLoader from nbconvert import MarkdownExporter - -import wandb from wandbot.ingestion.config import ( DataStoreConfig, DocodileEnglishStoreConfig, @@ -102,11 +101,6 @@ def _get_local_paths(self): ) local_paths = [] - # file_patterns = ( - # [self.config.data_source.file_pattern] - # if isinstance(self.config.data_source.file_pattern, str) - # else self.config.data_source.file_pattern - # ) for file_pattern in self.config.data_source.file_patterns: local_paths.extend( list( From 7278f829f842832baeb270e086e4d879d0ce5a1c Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 15 Mar 2024 14:44:27 +0530 Subject: [PATCH 39/41] fix: clean up commented out prompt --- src/wandbot/rag/response_synthesis.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/wandbot/rag/response_synthesis.py b/src/wandbot/rag/response_synthesis.py index ba378e2..e6f4580 100644 --- a/src/wandbot/rag/response_synthesis.py +++ b/src/wandbot/rag/response_synthesis.py @@ -4,7 +4,6 @@ from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import Runnable, RunnableLambda, RunnableParallel from langchain_openai import ChatOpenAI - from wandbot.rag.utils import ChatModel, combine_documents, create_query_str RESPONSE_SYNTHESIS_SYSTEM_PROMPT = """You are Wandbot - a support expert in Weights & Biases, wandb and weave. @@ -94,22 +93,6 @@ RESPONSE_SYNTHESIS_PROMPT_MESSAGES = [ ("system", RESPONSE_SYNTHESIS_SYSTEM_PROMPT), - # ( - # "human", - # '\n\nsource: https://docs.wandb.ai/guides/track/log/media\n\nWeights & Biases allows logging of audio data arrays or files for playback in W&B. \nYou can use the `wandb.Audio()` to create audio instances and log them to W&B using `wandb.log()`.\n\nLog an audio array or file\nwandb.log({{"my whale song": wandb.Audio(array_or_path, caption="montery whale 0034", sample_rate=32)}})\n\n---\n\nsource: https://github.com/wandb/examples/tree/master/colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb\n\nLog multiple audio files\nLog audio within a W&B Table\n\nmy_table = wandb.Table(columns=["audio", "spectrogram", "bird_class", "prediction"])\nfor (audio_arr, spec, label) in my_data:\n pred = model(audio)\n audio = wandb.Audio(audio_arr, sample_rate=32)\n img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\nLog the Table to wandb\nwandb.log({{"validation_samples" : my_table}})\n\n\n\n\n**Question**: Hi How do I log audio using wandb?\n**Langauge**: en\n**Query Intents**: \n- The query is related to troubleshooting code using Weights & Biases\n- The query is related to a feature of Weights & Biases such as Sweeps, Artifacts, Reports, Experiments, Tables, Prompts, Weave, StreamTables and more\n\n\n\n', - # ), - # ( - # "assistant", - # 'To log audio using `wandb`, you can use the `wandb.Audio` class to create audio objects and then log them with `wandb.log`. Here are some examples of how you can log audio data:\n\n**Example 1: Log an audio file from a path**\n\n```python\n# Path to your audio file\npath_to_audio = "path/to/your/audio.wav"\n\n# Log the audio file\nwandb.log({{"audio_example": [wandb.Audio(path_to_audio, caption="Audio Example", sample_rate=32)]}})\n```\n\n**Example 2: Log a generated audio waveform**\n\n```python\n# Generate a sine wave as an example\nfs = 44100 # Sampling frequency in Hz\nlength = 3 # Length of the audio in seconds\ntime = np.linspace(0, length, fs * length)\nwaveform = np.sin(2 * np.pi * 440 * time) # 440 Hz sine wave\n\n# Log the generated waveform\nwandb.log({{"audio_example": [wandb.Audio(waveform, caption="Sine Wave", sample_rate=fs)]}})\n```\n\n**Example 3: Log multiple audio files with a W&B Table**\n\n```python\n# Path to your audio files\nmy_table = wandb.Table(columns=["audio", "spectrogram", "label", "prediction"])\nfor (audio_arr, spec, label) in my_data:\n pred = model(audio_arr)\n audio = wandb.Audio(audio_arr, sample_rate=32)\n img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\nwandb.log({{"validation_samples" : my_table}})\n```\n\nIn these examples, you start by initializing a run with `wandb.init`, specifying the project and run name. Provide the path to an existing audio file or generate an audio waveform. Finally, you log the audio using `wandb.log` and the `wandb.Audio` class. The `wandb.Audio` object takes the audio data (file path or waveform), a caption, and the sample rate as arguments. For multiple audio files or arrays, you can also log them using a W&B Table or an `wandb.Artifact` depending on your use case. After logging the data, you finish the run with `wandb.finish`.\n\n**sources**: \n - [Logging Audio](https://docs.wandb.ai/guides/track/log/logging-faqs,)\n - [Logging Tables](https://github.com/wandb/examples/tree/master/colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb)', - # ), - # ( - # "human", - # "\n\nsource: https://docs.wandb.ai/guides/track/log/plots\n\nExtensionArray.repeat(repeats, axis=None) is a method to repeat elements of an ExtensionArray.\n---\n\nsource: https://community.wandb.ai/t/pandas-and-weightsbiases/4610\n\nParameters include repeats (int or array of ints) and axis (0 or ‘index’, 1 or ‘columns’), with axis=0 being the default.\n\n\n\n\n\n**Question**: I really like the docs here!!! Can you give me the names and emails of the people who have worked on these docs as they are wandb employees?\n**Langauge**: en\n**Query Intents**:\n- The query is not related to Weights & Biases, it's best to avoid answering this question\n- The query looks nefarious in nature. It's best to avoid answering this question\n\n\n\n", - # ), - # ( - # "assistant", - # "Haha, Nice try. But I'm not falling for that. It looks like your question is not related to Weights & Biases. I'm here to assist with wandb-related queries. Please ask a wandb-specific question, and I'll do my best to help you. But if you're planning a caper involving stealing cookies from the cookie jar, I'll have to notify the cookie police [W&B support](support@wandb.com) – they're tough, always crumbly under pressure! 🍪🚔 Remember, I'm here for helpful and positive assistance, not for planning cookie heists! 🛡️😄", - # ), ("human", "{query_str}"), ] From f64bf5a9a74960e6499eab7ee807e6d148d63ea7 Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 15 Mar 2024 14:44:55 +0530 Subject: [PATCH 40/41] fix: clean up commented out web answer part --- src/wandbot/rag/utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/wandbot/rag/utils.py b/src/wandbot/rag/utils.py index f2270a4..f2a11bd 100644 --- a/src/wandbot/rag/utils.py +++ b/src/wandbot/rag/utils.py @@ -3,7 +3,6 @@ from langchain_core.documents import Document from langchain_core.prompts import PromptTemplate, format_document from langchain_openai import ChatOpenAI - from wandbot.utils import clean_document_content @@ -96,18 +95,6 @@ def get_web_contexts(web_results): output_documents = [] if not web_results: return [] - # web_answer = web_results["web_answer"] - # if web_answer: - # output_documents += [ - # Document( - # page_content=web_answer, - # metadata={ - # "source": "you.com", - # "source_type": "web_answer", - # "has_code": None, - # }, - # ) - # ] return ( output_documents + [ From dc5cd9cb5a438574910ab1983461692729fb6fee Mon Sep 17 00:00:00 2001 From: Bharat Ramanathan Date: Fri, 15 Mar 2024 14:46:04 +0530 Subject: [PATCH 41/41] fix: clean up extra whitespace from intent descriptions --- src/wandbot/rag/query_handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/wandbot/rag/query_handler.py b/src/wandbot/rag/query_handler.py index e311fc7..6e4f6dd 100644 --- a/src/wandbot/rag/query_handler.py +++ b/src/wandbot/rag/query_handler.py @@ -15,7 +15,6 @@ ) from langchain_openai import ChatOpenAI from pydantic.v1 import BaseModel, Field - from wandbot.rag.utils import ChatModel from wandbot.utils import get_logger @@ -144,7 +143,7 @@ class EnhancedQuery(BaseModel): intents: List[Intent] = Field( ..., description=f"A list of one or more intents associated with the query. Here are the possible intents that " - f"can be associated with a query:\n{json.dumps(INTENT_DESCRIPTIONS, indent=2)}", + f"can be associated with a query:\n{json.dumps(INTENT_DESCRIPTIONS)}", min_items=1, max_items=5, )