diff --git a/_includes/code/howto/indexes/indexes-dynamic-v2.ts b/_includes/code/howto/indexes/indexes-dynamic-v2.ts new file mode 100644 index 0000000000..d0420283f3 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-dynamic-v2.ts @@ -0,0 +1,94 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { WeaviateClient } from 'weaviate-ts-client'; + +// Create client connection +function getClient(){ + const client: WeaviateClient = weaviate.client({ + scheme: 'http', + host: 'localhost:8080', +}); + + return client; +} + +// Delete pre-existing collections +async function deleteClass(client: WeaviateClient, className: string){ + if (await client.schema.exists(className)) { + await client.schema.classDeleter().withClassName(className).do(); + } +} + +//////////////////// +// ENABLE DYNAMIC // +//////////////////// + +// START EnableDynamic +async function createDynamicCollection(client: WeaviateClient, className: string){ + const setIndexType = { + class: className, + // Add property definitions + vectorizer: 'text2vec-openai', + vectorIndexType: 'dynamic', + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END EnableDynamic + +//////////////////// +// CONFIGURE DYNAMIC // +//////////////////// + +// START ConfigDynamic +async function configureDynamicCollection(client: WeaviateClient, className: string){ + const setIndexType = { + class: className, + // Add property definitions + vectorizer: 'text2vec-openai', + vectorIndexType: 'dynamic', + vectorIndexConfig: { + distance: 'cosine', + vector_cache_max_objects: 100000, + bq: { enabled: true, }, + }, + vectorIndexConfigDynamic: { + distance: 'cosine', + ef_construction: '256', // Dynamic list size during construction + max_connections: '128', // Maximum number of connections per node + ef: '-1', // Dynamic list size during search; -1 enables dynamic Ef + dynamic_ef_factor: '15', // Multiplier for dynamic Ef + dynamic_ef_min: '200', // Minimum threshold for dynamic Ef + dynamic_ef_max: '1000', // Maximum threshold for dynamic Ef + quantizer: 'Configure.VectorIndex.Quantizer.pq()', // Quantizer configuration + }, + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END ConfigDynamic + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const className = "ConfigCollection"; + + const client = await getClient(); + + // Run enable dynamic collection code + await deleteClass(client, className) + createDynamicCollection(client, className); + + // // Run configure dynamic collection code + // await deleteClass(client, className) + // configureDynamicCollection(client, className); +} + +main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.py b/_includes/code/howto/indexes/indexes-dynamic-v3.py new file mode 100644 index 0000000000..62bf0abcde --- /dev/null +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.py @@ -0,0 +1,101 @@ +# TODO: Configure as part of the test harness + +class_name = "ConfigCollection" + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +client = weaviate.Client( + url="http://localhost:8080", + additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")}, +) + +###################### +### ENABLE DYNAMIC ### +###################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableDynamic +class_obj = { + "class": class_name, + "vectorIndexType": "dynamic", +} + +client.schema.create_class(class_obj) +# END EnableDynamic + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "dynamic" +): + correct_index = True +assert correct_index, "Wrong index type" + +######################### +### CONFIGURE DYNAMIC ### +######################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START ConfigDynamic +class_obj = { + "description": "Dynamic configuration example", + "class": class_name, + "vectorIndexType": "dynamic", + "vectorIndexConfig": { + "hnsw": { + "cleanupIntervalSeconds": 300, + "distanceMetric": "cosine", + "dynamicEfMin": 100, + "dynamicEfMax": 600, + "dynamicEfFactor": 8, + "ef": -1, + "efConstruction": 128, + "flatSearchCutoff": 40000, + "maxConnections": 32, + "skip": False, + "vectorCacheMaxObjects": 1000000000000, + "pq": { "enabled": True, }, + }, + "flat": { + "distanceMetric": "cosine", + "vector_cache_max_objects": 100000, + "bq": { "enabled": True, }, + }, + } +} + +client.schema.create_class(class_obj) +# END ConfigDynamic + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "dynamic" +): + correct_index = True +assert correct_index, "Wrong index type" diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.ts b/_includes/code/howto/indexes/indexes-dynamic-v3.ts new file mode 100644 index 0000000000..828388925d --- /dev/null +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.ts @@ -0,0 +1,94 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { WeaviateClient,vectorizer, configure } from 'weaviate-client'; + +// Delete pre-existing collections +async function deleteCollection(client: WeaviateClient, collectionName: string){ + if(await client.collections.exists(collectionName)){ + await client.collections.delete(collectionName) + } +} + +// Create client connection +async function getClient(){ + const client: WeaviateClient = await weaviate.connectToLocal(); + return client; +} + +//////////////////// +// ENABLE DYNAMIC // +//////////////////// + +// START EnableDynamic +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function createDynamicCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.dynamic(), + }), + }) +} +// END EnableDynamic + + +/////////////////////// +// CONFIGURE DYNAMIC // +/////////////////////// + +// START ConfigDynamic +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function configureDynamicCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.dynamic({ + distanceMetric: 'cosine', + hnsw: { + distanceMetric: 'cosine', + efConstruction: 256, // Dynamic list size during construction + maxConnections: 128, // Maximum number of connections per node + ef: -1, // Dynamic list size during search; -1 enables dynamic Ef + dynamicEfFactor: 15, // Multiplier for dynamic Ef + dynamicEfMin: 200, // Minimum threshold for dynamic Ef + dynamicEfMax: 1000, // Maximum threshold for dynamic Ef + quantizer: configure.vectorIndex.quantizer.pq() // Compression + }, + flat: { + distanceMetric: 'cosine', + vectorCacheMaxObjects: 1000000, + quantizer: configure.vectorIndex.quantizer.bq(), + }, + }), + }), + }) +} +// END ConfigDynamic + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const collectionName = "ConfigCollection"; + + const client = await getClient(); + + // // Run enable dynamic collection code + // await deleteCollection(client, collectionName) + // await createDynamicCollection(client, collectionName); + + // Run configure dynamic collection code + await deleteCollection(client, collectionName) + await configureDynamicCollection(client, collectionName); + +} + +main() diff --git a/_includes/code/howto/indexes/indexes-dynamic-v4.py b/_includes/code/howto/indexes/indexes-dynamic-v4.py new file mode 100644 index 0000000000..ccc2184ab4 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-dynamic-v4.py @@ -0,0 +1,92 @@ +# TODO: Configure as part of the test harness + +collection_name = "ConfigCollection" + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +cohere_api_key = os.environ["COHERE_API_KEY"] +client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) + +###################### +### ENABLE DYNAMIC ### +###################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START EnableDynamic +from weaviate.classes.config import Configure, Property, DataType, VectorDistances + +client.collections.create( + name=collection_name, + description="Configuration example", + vector_index_config=Configure.VectorIndex.dynamic(), + # Configure vectorizer, properties + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="body", data_type=DataType.TEXT), + ] +) +# END EnableDynamic + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" +), "Wrong index type" + +###################### +### ENABLE DYNAMIC ### +###################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START ConfigDynamic +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.dynamic( + distance_metric=VectorDistances.COSINE, + threshold=20000, + hnsw=Configure.VectorIndex.hnsw( + # Any hnsw configuration parameters + dynamic_ef_factor=15, # Multiplier for dynamic Ef + dynamic_ef_min=200, # Minimum threshold for dynamic Ef + dynamic_ef_max=1000, # Maximum threshold for dynamic Ef + ), + flat=Configure.VectorIndex.flat( + # Any flat index configuration parameters + vector_cache_max_objects=100000, + quantizer=Configure.VectorIndex.Quantizer.bq() + ), + ), + # Configure vectorizer, properties +) +# END ConfigDynamic + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" +), "Wrong index type" + +################ +### CLEAN UP ### +################ + +client.close() diff --git a/_includes/code/howto/indexes/indexes-inverted-v2.ts b/_includes/code/howto/indexes/indexes-inverted-v2.ts new file mode 100644 index 0000000000..54b9140704 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-inverted-v2.ts @@ -0,0 +1,183 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { WeaviateClient } from 'weaviate-ts-client'; + +// Create client connection +function getClient(){ + const client: WeaviateClient = weaviate.client({ + scheme: 'http', + host: 'localhost:8080', + }); + + return client; +} + +// Delete pre-existing collections +async function deleteClass(client: WeaviateClient, className: string){ + if (await client.schema.exists(className)) { + await client.schema.classDeleter().withClassName(className).do(); + } +} + +//////////// +// SEARCH // +/////////// + +// START SearchIndex +async function searchInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + properties: [ + { + name: 'textProperty', + dataType: ['text'], + indexSearchable: true, + }, + ], +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END SearchIndex + +//////////// +// FILTER // +//////////// + +// START FilterIndex +async function filterInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + properties: [ + { + name: 'textProperty', + dataType: ['text'], + indexFilterable: true, + }, + ], +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END FilterIndex + +/////////// +// RANGE // +/////////// + +// START RangeIndex +async function rangeInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + properties: [ + { + name: 'numericProperty', + dataType: ['int'], + indexRangeFilters: true, + }, + ], +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END RangeIndex + +////////// +// BM25 // +////////// + +// START BM25Index +async function bm25InvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + invertedIndexConfig: { + bm25: { + b: 0.7, + k1: 1.25 + }, + } +} + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END BM25Index + +////////////////////// +// COLLECTION LEVEL // +////////////////////// + +// START CollLevIndex +async function collLevInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + invertedIndexConfig: { + indexTimestamps: true, + indexNullState: true, + indexPropertyLength: true + } +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END CollLevIndex + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const className = "ConfigCollection"; + + const client = await getClient(); + + // // Run search code + // await deleteClass(client, className) + // searchInvertedCollection(client, className); + + // // Run filter code + // await deleteClass(client, className) + // filterInvertedCollection(client, className); + + // // Run range code + // await deleteClass(client, className) + // rangeInvertedCollection(client, className); + + // Run bm25 code + await deleteClass(client, className) + bm25InvertedCollection(client, className); + + // // Run collection level code + // await deleteClass(client, className) + // collLevInvertedCollection(client, className); + +} + +main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.py b/_includes/code/howto/indexes/indexes-inverted-v3.py new file mode 100644 index 0000000000..b1ff3b485c --- /dev/null +++ b/_includes/code/howto/indexes/indexes-inverted-v3.py @@ -0,0 +1,172 @@ +# TODO: Configure as part of the test harness + +class_name = "ConfigCollection" + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +client = weaviate.Client( + url="http://localhost:8080", + additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")}, +) + +############## +### SEARCH ### +############## + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START SearchIndex +class_obj = { + "class": class_name, + "vectorizer": "text2vec-huggingface", + "properties": [ + { + "name": "title", + "dataType": ["text"], + "indexSearchable": False, + }, + ] +} + +client.schema.create_class(class_obj) +# END SearchIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + + +############## +### FILTER ### +############## + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START FilterIndex +class_obj = { + "class": class_name, + "vectorizer": "text2vec-huggingface", + "properties": [ + { + "name": "title", + "dataType": ["text"], + "indexFilterable": False, + }, + ] +} + +client.schema.create_class(class_obj) +# END FilterIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + + +############# +### RANGE ### +############# + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START RangeIndex +class_obj = { + "class": class_name, + "vectorizer": "text2vec-huggingface", + "properties": [ + { + "name": "scores", + "dataType": ["int"], + "indexRangeFilters": True, + }, + ], +} + +client.schema.create_class(class_obj) +# END RangeIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +############ +### BM25 ### +############ + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START BM25Index +class_obj = { + "class": class_name, + "invertedIndexConfig": { + "bm25": { + "b": 0.7, + "k1": 1.25 + }, + } +} + +client.schema.create_class(class_obj) +# END BM25Index + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +######################## +### COLLECTION LEVEL ### +######################## + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START CollLevIndex +class_obj = { + "class": class_name, + "invertedIndexConfig": { + "indexTimestamps": True, + "indexNullState": True, + "indexPropertyLength": True + } +} + +client.schema.create_class(class_obj) +# END CollLevIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.ts b/_includes/code/howto/indexes/indexes-inverted-v3.ts new file mode 100644 index 0000000000..1c8bc6d866 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-inverted-v3.ts @@ -0,0 +1,161 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; + +// Delete pre-existing collections +async function deleteCollection(client: WeaviateClient, collectionName: string){ + if(await client.collections.exists(collectionName)){ + await client.collections.delete(collectionName) + } +} + +// Create client connection +async function getClient(){ + const client: WeaviateClient = weaviate.connectToLocal(); + + return client; +} + +//////////// +// SEARCH // +//////////// + +// START SearchIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function searchInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + properties: [ + { + name: 'TextProperty', + dataType: dataType.TEXT, + indexSearchable: true, + }, + ], + }) +} +// END SearchIndex + +//////////// +// FILTER // +//////////// + +// START FilterIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function filterInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + properties: [ + { + name: 'TextProperty', + dataType: dataType.TEXT, + indexFilterable: true, + }, + ], + }) +} +// END FilterIndex + +/////////// +// RANGE // +/////////// + +// START RangeIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function rangeInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + properties: [ + { + name: 'NumericProperty', + dataType: dataType.INT, + indexRangeFilters: true, + }, + ], + }) +} +// END RangeIndex + +////////// +// BM25 // +////////// + +// START BM25Index +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function bm25InvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + invertedIndex: { + bm25: { + b: 0.7, + k1: 1.25 + }, + } + }) +} +// END BM25Index + +////////////////////// +// COLLECTION LEVEL // +////////////////////// + +// START CollLevIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function collLevInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + invertedIndex: { + indexNullState: true, + indexPropertyLength: true, + indexTimestamps: true + } + }) +} +// END CollLevIndex + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const collectionName = "ConfigCollection"; + + const client = await getClient(); + + // Run search code + await deleteCollection(client, collectionName) + await searchInvertedCollection(client, collectionName); + + // // Run filter code + // await deleteCollection(client, collectionName) + // await filterInvertedCollection(client, collectionName); + + // // Run range code + // await deleteCollection(client, collectionName) + // await rangeInvertedCollection(client, collectionName); + + // // Run bm25 code + // await deleteCollection(client, collectionName) + // await bm25InvertedCollection(client, collectionName); + + // // Run collection level code + // await deleteCollection(client, collectionName) + // await collLevInvertedCollection(client, collectionName); + + +} + +main() diff --git a/_includes/code/howto/indexes/indexes-inverted-v4.py b/_includes/code/howto/indexes/indexes-inverted-v4.py new file mode 100644 index 0000000000..6a92aa0d8a --- /dev/null +++ b/_includes/code/howto/indexes/indexes-inverted-v4.py @@ -0,0 +1,155 @@ +# TODO: Configure as part of the test harness + +collection_name = "ConfigCollection" + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +cohere_api_key = os.environ["COHERE_API_KEY"] + +client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) + +############## +### SEARCH ### +############## + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START SearchIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + properties=[ + Property( + name="TextProperty", + data_type=DataType.TEXT, + index_searchable=True, + ), + ], +) +# END SearchIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +############## +### FILTER ### +############## + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START FilterIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + properties=[ + Property( + name="TextProperty", + data_type=DataType.TEXT, + index_filterable=True, + ), + ], +) +# END FilterIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +############# +### RANGE ### +############# + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START RangeIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + properties=[ + Property( + name="NumericProperty", + data_type=DataType.INT, + index_range_filters=True, + ), + ], +) +# END RangeIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +############ +### BM25 ### +############ + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START BM25Index +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + inverted_index_config=Configure.inverted_index( + bm25_b=0.7, + bm25_k1=1.25, + ) +) +# END BM25Index + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +######################## +### COLLECTION LEVEL ### +######################## + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START CollLevIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + inverted_index_config=Configure.inverted_index( + index_null_state=True, + index_property_length=True, + index_timestamps=True + ) +) +# END CollLevIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +################ +### CLEAN UP ### +################ + +client.close() diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts new file mode 100644 index 0000000000..afba33e40b --- /dev/null +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -0,0 +1,225 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { WeaviateClient } from 'weaviate-ts-client'; + +// Create client connection +function getClient(){ + const client: WeaviateClient = weaviate.client({ + scheme: 'http', + host: 'localhost:8080', + }); + + return client; +} + +// Delete pre-existing collections +async function deleteClass(client: WeaviateClient, className: string){ + if (await client.schema.exists(className)) { + await client.schema.classDeleter().withClassName(className).do(); + } +} + +////////////////////////////// +// ENABLE HNSW - COLLECTION // +////////////////////////////// + +// START EnableHNSW +async function createHNSWCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + vectorIndexType: 'hnsw', + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END EnableHNSW + + +//////////////////// +// CONFIGURE HNSW // +//////////////////// + +// START ConfigHNSW +async function configHNSWCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + vectorIndexType: 'hnsw', + vectorIndexConfig: { + distance: 'cosine', + ef_construction: '256', // Dynamic list size during construction + max_connections: '128', // Maximum number of connections per node + ef: '-1', // Dynamic list size during search; -1 enables dynamic Ef + dynamic_ef_factor: '15', // Multiplier for dynamic Ef + dynamic_ef_min: '200', // Minimum threshold for dynamic Ef + dynamic_ef_max: '1000', // Maximum threshold for dynamic Ef + quantizer: 'Configure.VectorIndex.Quantizer.pq()', // Quantizer configuration + }, + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END ConfigHNSW + +/////////////////// +// COMPRESS HNSW // +/////////////////// + +// START CompressHNSW +async function compressHNSWCollection(client: WeaviateClient, className: string){ + const setIndexType = { + class: className, + vectorIndexConfig: { + quantizer: 'Configure.VectorIndex.Quantizer.pq()', + }, + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END CompressHNSW + +////////////////////////////// +/// ENABLE HNSW - MULTIPLE /// +////////////////////////////// + +// START EnableMulti +async function createMultiCollection(client: WeaviateClient, className: string){ + + const classWithNamedVectors = { + class: className, + vectorConfig: { + // Define a named vector + vectorForFieldOne: { + vectorizer: { + 'text2vec-cohere': { + properties: ['FieldOne'], + }, + }, + vectorIndexType: 'hnsw', + }, + // Define another named vector + vectorForFieldTwo: { + vectorizer: { + 'text2vec-openai': { + properties: ['FieldTwo'], + }, + }, + vectorIndexType: 'flat' + }, + }, + // Configure properties + } + + // Add the class to the schema + await client.schema.classCreator().withClass(classWithNamedVectors).do(); +} +// END EnableMulti + +///////////////// +// ENABLE FLAT // +///////////////// + +// START EnableFlat +async function createFlatCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + vectorIndexType: 'flat', + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END EnableFlat + +//////////////////// +// CONFIGURE FLAT // +//////////////////// + +// START ConfigFlat +async function configureFlatCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + vectorIndexType: 'flat', + vectorIndexConfig: { + distance: 'cosine', + vector_cache_max_objects: 100000, + bq: { enabled: true, }, + }, + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END ConfigFlat + +/////////////////// +// COMPRESS FLAT // +/////////////////// + +// START CompressFlat +async function compressFlatCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + vectorIndexType: 'flat', + vectorIndexConfig: { + bq: { enabled: true, }, + }, + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END CompressFlat + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const className = "ConfigCollection"; + + const client = await getClient(); + + // Run enable HNSW collection code + await deleteClass(client, className) + createHNSWCollection(client, className); + + // // Run configure HNSW collection code + // await deleteClass(client, className) + // configHNSWCollection(client, className); + + // // Run multiple named vector collection code + // await deleteClass(client, className) + // createMultiCollection(client, className); + + // // Run enable flat collection code + // await deleteClass(client, className) + // createFlatCollection(client, className); + + // // Run configure flat collection code + // await deleteClass(client, className) + // configureFlatCollection(client, className); + + // Run compress flat collection code + await deleteClass(client, className) + compressFlatCollection(client, className); + +} + +main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py new file mode 100644 index 0000000000..414e3dbead --- /dev/null +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -0,0 +1,246 @@ +# TODO: Configure as part of the test harness + +class_name = "ConfigCollection" + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +client = weaviate.Client( + url="http://localhost:8080", + additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")}, +) + +################################ +### ENABLE HNSW - COLLECTION ### +################################ + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableHNSW +class_obj = { + "class": class_name, + "vectorIndexType": "hnsw", +} + +client.schema.create_class(class_obj) +# END EnableHNSW + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "hnsw" +): + correct_index = True +assert correct_index, "Wrong index type" + + +################### +### CONFIG HNSW ### +################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START ConfigHNSW +class_obj = { + "class": class_name, + "vectorIndexType": "hnsw", + "vectorIndexConfig": { + "distance_metric": "cosine", + "ef_construction": 256, # Dynamic list size during construction + "max_connections": 128, # Maximum number of connections per node + "ef": -1, # Dynamic list size during search; -1 enables dynamic Ef + "dynamic_ef_factor": 15, # Multiplier for dynamic Ef + "dynamic_ef_min": 200, # Minimum threshold for dynamic Ef + "dynamic_ef_max": 1000, # Maximum threshold for dynamic Ef + "pq": { + "enabled": True, + }, # Enable compression + }, +} + +client.schema.create_class(class_obj) +# END ConfigHNSW + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "hnsw" +): + correct_index = True +assert correct_index, "Wrong index type" + +##################### +### COMPRESS HNSW ### +##################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START CompressHNSW +class_obj = { + "class": class_name, + "vectorIndexType": "hnsw", + "vectorIndexConfig": { + "distance_metric": "cosine", + "pq": { + "enabled": True, + }, + }, +} + +client.schema.create_class(class_obj) +# END CompressHNSW + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "hnsw" +): + correct_index = True +assert correct_index, "Wrong index type" + +# ############################## +# ### ENABLE HNSW - MULTIPLE ### +# ############################## + +# START EnableMulti +# The Python client v3 doesn't support multiple named vectors. + +# To use multiple named vectors, upgrade to the Python client v4. +# END EnableMulti + +# ################### +# ### ENABLE FLAT ### +# ################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableFlat +class_obj = { + "class": class_name, + "vectorIndexType": "flat", +} + +client.schema.create_class(class_obj) +# END EnableFlat + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "flat" +): + correct_index = True +assert correct_index, "Wrong index type" + +###################### +### CONFIGURE FLAT ### +###################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START ConfigFlat +class_obj = { + "class": class_name, + "vectorIndexType": "flat", + "vectorIndexConfig": { + "distance_metric": "cosine", + "vector_cache_max_objects": 100000, + "bq": {"enabled": True} + }, +} + +client.schema.create_class(class_obj) +# END ConfigFlat + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "flat" +): + correct_index = True + +assert correct_index, "Wrong index type" + +##################### +### COMPRESS FLAT ### +##################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START CompressFlat +class_obj = { + "class": class_name, + "vectorIndexType": "flat", + "vectorIndexConfig": { + "bq": {"enabled": True} + }, +} + +client.schema.create_class(class_obj) +# END CompressFlat + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "flat" +): + correct_index = True + +assert correct_index, "Wrong index type" diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts new file mode 100644 index 0000000000..c6284d4b8e --- /dev/null +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -0,0 +1,228 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; + +// Delete pre-existing collections +async function deleteCollection(client: WeaviateClient, collectionName: string){ + if(await client.collections.exists(collectionName)){ + await client.collections.delete(collectionName) + } +} + +// Create client connection +async function getClient(){ + const client: WeaviateClient = weaviate.connectToLocal(); + + return client; +} + +////////////////////////////// +// ENABLE HNSW - COLLECTION // +////////////////////////////// + +// START EnableHNSW +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function createHNSWCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.hnsw(), + }), + }) +} +// END EnableHNSW + +//////////////////// +// CONFIGURE HNSW // +//////////////////// + +// START ConfigHNSW +// Add this import line: +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function configureHNSWCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.hnsw({ + distanceMetric: 'cosine', + efConstruction: 256, // Dynamic list size during construction + maxConnections: 128, // Maximum number of connections per node + ef: -1, // Dynamic list size during search; -1 enables dynamic Ef + dynamicEfFactor: 15, // Multiplier for dynamic Ef + dynamicEfMin: 200, // Minimum threshold for dynamic Ef + dynamicEfMax: 1000, // Maximum threshold for dynamic Ef + quantizer: configure.vectorIndex.quantizer.pq() // Compression + }), + }), + }) +} +// END ConfigHNSW + +//////////////////// +// COMPRESS HNSW // +//////////////////// + +// START CompressHNSW +// Add this import line: +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function compressHNSWCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.hnsw({ + quantizer: configure.vectorIndex.quantizer.pq() // Compression + }), + }), + }) +} +// END CompressHNSW + +////////////////////////////// +/// ENABLE HNSW - MULTIPLE /// +////////////////////////////// + +// START EnableMulti +// Add this import line +// import { vectorizer, configure } from 'weaviate-client'; + +async function createMultiCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: [ + // Define a named vector + vectorizer.text2VecCohere({ + name: "vectorForFieldOne", + sourceProperties: ['FieldOne'], + vectorIndexConfig: configure.vectorIndex.hnsw() + }), + // Define another named vector + vectorizer.text2VecOpenAI({ + name: "vectorForFieldTwo", + sourceProperties: ['FieldTwo'], + vectorIndexConfig: configure.vectorIndex.flat() + }) + ], + // Configure properties + }) +} +// END EnableMulti + +///////////////// +// ENABLE FLAT // +///////////////// + +// START EnableFlat +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function createFlatCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.flat(), + }), + // Configure properties + }) +} +// END EnableFlat + +//////////////////// +// CONFIGURE FLAT // +//////////////////// + +// START ConfigFlat +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function configureFlatCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.flat({ + distanceMetric: 'cosine', + vectorCacheMaxObjects: 1000000, + quantizer: configure.vectorIndex.quantizer.bq(), + }), + }) + // Configure properties + }) +} +// END ConfigFlat + +//////////////////// +// COMPRESS FLAT // +/////////////////// + +// START CompressFlat +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function compressFlatCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.flat({ + quantizer: configure.vectorIndex.quantizer.bq({ + rescoreLimit: 200, + cache: true + }), + }), + }) + // Configure properties + }) +} +// END CompressFlat + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const collectionName = "ConfigCollection"; + + const client = await getClient(); + + // // Run enable HNSW collection code + // await deleteCollection(client, collectionName) + // await createHNSWCollection(client, collectionName); + + + // // Run configure HNSW collection code + // await deleteCollection(client, collectionName) + // await configureHNSWCollection(client, collectionName); + + + // // Run compress HNSW collection code + // await deleteCollection(client, collectionName) + // await compressHNSWCollection(client, collectionName); + + + // // Run multiple named vector collection code + // await deleteCollection(client, collectionName) + // await createMultiCollection(client, collectionName); + + + // // Run enable Flat collection code + // await deleteCollection(client, collectionName) + // await createFlatCollection(client, collectionName); + + + // // Run configure Flat collection code + // await deleteCollection(client, collectionName) + // await configureFlatCollection(client, collectionName); + + + // Run compress Flat collection code + await deleteCollection(client, collectionName) + await compressFlatCollection(client, collectionName); + +} + +main() diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py new file mode 100644 index 0000000000..4ca32974d2 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -0,0 +1,251 @@ +# TODO: Configure as part of the test harness + +collection_name = "ConfigCollection" + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +cohere_api_key = os.environ["COHERE_API_KEY"] + +client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) + +################################ +### ENABLE HNSW - COLLECTION ### +################################ + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START EnableHNSW +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.hnsw(), + # Configure properties, vectorizer +) +# END EnableHNSW + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.HNSW" +), "Wrong index type" + +################################ +### CONFIG HNSW - COLLECTION ### +################################ + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START ConfigHNSW +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.hnsw( + distance_metric=VectorDistances.COSINE, + ef_construction=256, # Dynamic list size during construction + max_connections=128, # Maximum number of connections per node + quantizer=Configure.VectorIndex.Quantizer.pq(), # Quantizer configuration + ef=-1, # Dynamic list size during search; -1 enables dynamic Ef + dynamic_ef_factor=15, # Multiplier for dynamic Ef + dynamic_ef_min=200, # Minimum threshold for dynamic Ef + dynamic_ef_max=1000, # Maximum threshold for dynamic Ef + ), + # Configure properties, vectorizer +) +# END ConfigHNSW + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.HNSW" +), "Wrong index type" + +##################### +### COMPRESS HNSW ### +##################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START CompressHNSW +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.hnsw( + quantizer=Configure.VectorIndex.Quantizer.pq(), # Quantizer configuration + ), + # Configure properties, vectorizer +) +# END CompressHNSW + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.HNSW" +), "Wrong index type" + +############################## +### ENABLE HNSW - MULTIPLE ### +############################## + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START EnableMulti + +from weaviate.classes.config import Configure, Property, DataType, Tokenization + +client.collections.create( + name=collection_name, + vectorizer_config=[ + # Define a named vector + Configure.NamedVectors.text2vec_cohere( + name="vectorForFieldOne", + source_properties=["FieldOne"], + vector_index_config=Configure.VectorIndex.hnsw( + max_connections=128, + ), + ), + # Define another named vector + Configure.NamedVectors.text2vec_openai( + name="vectorForFieldTwo", + source_properties=["FieldTwo"], + vector_index_config=Configure.VectorIndex.flat(), + ), + ], + # Configure properties +) +# END EnableMulti + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + collection.config.get() + .vector_config["vectorForFieldOne"] + .vector_index_config.vector_index_type() + == "hnsw" +), "Wrong index type" +assert ( + collection.config.get() + .vector_config["vectorForFieldTwo"] + .vector_index_config.vector_index_type() + == "flat" +), "Wrong index type" + +################### +### ENABLE FLAT ### +################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START EnableFlat +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.flat(), + # Configure properties, vectorizer +) +# END EnableFlat + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.FLAT" +), "Wrong index type" + +################### +### CONFIG FLAT ### +################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START ConfigFlat +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.flat( + distance_metric=VectorDistances.COSINE, + vector_cache_max_objects=100000, + quantizer=Configure.VectorIndex.Quantizer.bq() + ), + # Configure properties, vectorizer +) +# END ConfigFlat + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.FLAT" +), "Wrong index type" + +################### +### COMPRESS FLAT ### +################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START CompressFlat +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.flat( + quantizer=Configure.VectorIndex.Quantizer.bq() + ), + # Configure properties, vectorizer +) +# END CompressFlat + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.FLAT" +), "Wrong index type" + +################ +### CLEAN UP ### +################ + +client.close() diff --git a/_includes/configuration/compression-methods.mdx b/_includes/configuration/compression-methods.mdx new file mode 100644 index 0000000000..76ce28c530 --- /dev/null +++ b/_includes/configuration/compression-methods.mdx @@ -0,0 +1,3 @@ +- [Binary Quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) +- [Product Quantization (PQ)](/developers/weaviate/configuration/compression/pq-compression) +- [Scalar Quantization (SQ)](/developers/weaviate/configuration/compression/sq-compression) \ No newline at end of file diff --git a/_includes/dynamic-index-async-req.mdx b/_includes/dynamic-index-async-req.mdx index c0e1af1629..a6ba6feb4c 100644 --- a/_includes/dynamic-index-async-req.mdx +++ b/_includes/dynamic-index-async-req.mdx @@ -1,3 +1,3 @@ -:::info Dynamic index requires `ASYNC_INDEXING` -To use the `dynamic` vector index type, your Weaviate instance must have the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) enabled. This can be done by setting the `ASYNC_INDEXING` environment variable to `true`. For Weaviate Cloud users, this can be enabled through the Weaviate Cloud dashboard. +:::info Dynamic indexes require asynchronous indexing +The `dynamic` vector index type requires asynchronous indexing. To enable asynchronous indexing, set the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) to `true`. Weaviate Cloud users can enable asynchronous indexing in the Weaviate Cloud dashboard. ::: diff --git a/_includes/indexes/async-indexing.mdx b/_includes/indexes/async-indexing.mdx new file mode 100644 index 0000000000..5c74d88092 --- /dev/null +++ b/_includes/indexes/async-indexing.mdx @@ -0,0 +1,5 @@ +Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). + +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; + + \ No newline at end of file diff --git a/_includes/indexes/consider-compression.mdx b/_includes/indexes/consider-compression.mdx new file mode 100644 index 0000000000..a335d6b044 --- /dev/null +++ b/_includes/indexes/consider-compression.mdx @@ -0,0 +1 @@ +When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. \ No newline at end of file diff --git a/_includes/indexes/dynamic-intro.mdx b/_includes/indexes/dynamic-intro.mdx new file mode 100644 index 0000000000..a09dc0c162 --- /dev/null +++ b/_includes/indexes/dynamic-intro.mdx @@ -0,0 +1 @@ +[Dynamic indexes](/developers/weaviate/concepts/indexing/dynamic-indexes) offer a flexible approach to indexing. A dynamic index starts as a flat index and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file diff --git a/_includes/indexes/enable-async.mdx b/_includes/indexes/enable-async.mdx new file mode 100644 index 0000000000..a462dff681 --- /dev/null +++ b/_includes/indexes/enable-async.mdx @@ -0,0 +1,6 @@ +To enable asynchronous indexing, set the `ASYNC_INDEXING` parameter to `true` in your Weaviate configuration. + +- [Docker Compose](developers/weaviate/installation/docker-compose): Add `ASYNC_INDEXING` to `docker-compose.yml`. +- [Kubernetes](/developers/weaviate/installation/kubernetes): Add `ASYNC_INDEXING` to `values.yaml`. + +When enabled, asynchronous indexing is available for all of the collections in your Weaviate instance. \ No newline at end of file diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx new file mode 100644 index 0000000000..b023e5a8d0 --- /dev/null +++ b/_includes/indexes/flat-intro.mdx @@ -0,0 +1,3 @@ +[Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) are disk based. Compared to HNSW indexes, flat indexes have very small RAM requirements. + +Flat indexes do brute-force vector searches. The search latency increases linearly with the number of objects. For that reason, flat indexes work best with small collections, less than 10,000 objects. If you expect the object count to grow significantly, consider using a [dynamic index](#dynamic-indexes). \ No newline at end of file diff --git a/_includes/indexes/hnsw-how.mdx b/_includes/indexes/hnsw-how.mdx new file mode 100644 index 0000000000..46b45bb7b9 --- /dev/null +++ b/_includes/indexes/hnsw-how.mdx @@ -0,0 +1,3 @@ +HNSW indexes build a multi-layered object graph. The graph structure and HNSW algorithm result in efficient, approximate nearest neighbor [(ANN)](https://en.wikipedia.org/wiki/Nearest_neighbor_search) searches. + +The index and graph structure are stored in RAM memory. The vectors are also stored in RAM. This makes HNSW indexes fast, but RAM is an expensive resource. Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. \ No newline at end of file diff --git a/_includes/indexes/hnsw-intro.mdx b/_includes/indexes/hnsw-intro.mdx new file mode 100644 index 0000000000..95e20c7679 --- /dev/null +++ b/_includes/indexes/hnsw-intro.mdx @@ -0,0 +1 @@ +[Hierarchical Navigable Small World (HNSW) indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well. Search times grow logarithmically, so vector searches are fast, even for very large data sets. \ No newline at end of file diff --git a/_includes/indexes/index-compression.mdx b/_includes/indexes/index-compression.mdx new file mode 100644 index 0000000000..c6b21433c6 --- /dev/null +++ b/_includes/indexes/index-compression.mdx @@ -0,0 +1,12 @@ +Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. + +Weaviate provides these compression methods: + +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + + +See also: + +- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) +- [Enable compression](/developers/weaviate/configuration/compression) \ No newline at end of file diff --git a/_includes/indexes/multiple-named-vectors.mdx b/_includes/indexes/multiple-named-vectors.mdx new file mode 100644 index 0000000000..77512b5dad --- /dev/null +++ b/_includes/indexes/multiple-named-vectors.mdx @@ -0,0 +1,50 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + +Items in a collection can have multiple named vectors. Each named vectors has its own vector index. These vector indexes can be configured independently. + +Configure a collection to use named vectors: + + + + + + + + + + + + + + + + +See also: + +- [Multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector) \ No newline at end of file diff --git a/_includes/indexes/vector-intro.mdx b/_includes/indexes/vector-intro.mdx new file mode 100644 index 0000000000..f0d8afd9c2 --- /dev/null +++ b/_includes/indexes/vector-intro.mdx @@ -0,0 +1,5 @@ +Weaviate offers three types of vector indexes: + +- [Hierarchical Navigable Small World (HNSW) indexes](#hnsw-indexes) +- [Flat indexes](#flat-indexes) +- [Dynamic indexes](#dynamic-indexes) \ No newline at end of file diff --git a/_includes/pq-compression/overview-text.mdx b/_includes/pq-compression/overview-text.mdx index c82e0dc4f8..aba30f05b8 100644 --- a/_includes/pq-compression/overview-text.mdx +++ b/_includes/pq-compression/overview-text.mdx @@ -1 +1,3 @@ -[Product quantization (PQ)](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq) is a form of data compression for vectors. PQ reduces the memory footprint of a vector index, so enabling PQ for HNSW lets you work with larger datasets. For a discussion of how PQ saves memory, see [this page](/developers/weaviate/concepts/vector-quantization#product-quantization). +Product quantization (PQ) is a form of data compression for vectors. PQ reduces the memory requirements for vector indexes so you can use larger datasets. + +For a discussion of how PQ saves memory, see [Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization). diff --git a/blog/2022-07-04-weaviate-1-14-release/index.mdx b/blog/2022-07-04-weaviate-1-14-release/index.mdx index 63b3fbccc9..8349c88d51 100644 --- a/blog/2022-07-04-weaviate-1-14-release/index.mdx +++ b/blog/2022-07-04-weaviate-1-14-release/index.mdx @@ -193,7 +193,7 @@ The REST API CRUD operations now require you to use both an **object ID** and th This ensures that the operations are performed on the correct objects. ### Background -One of Weaviate's features is full CRUD support. CRUD operations enable the mutability of data objects and their vectors, which is a key difference between a vector database and an ANN library. In Weaviate, every data object has an ID (UUID). This ID is stored with the data object in a key-value store. IDs don't have to be globally unique, because in Weaviate [classes](/developers/weaviate/manage-data/collections) act as namespaces. While each class has a different [HNSW index](/developers/weaviate/concepts/vector-index#hnsw), including the store around it, which is isolated on disk. +One of Weaviate's features is full CRUD support. CRUD operations enable the mutability of data objects and their vectors, which is a key difference between a vector database and an ANN library. In Weaviate, every data object has an ID (UUID). This ID is stored with the data object in a key-value store. IDs don't have to be globally unique, because in Weaviate [classes](/developers/weaviate/manage-data/collections) act as namespaces. While each class has a different [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes), including the store around it, which is isolated on disk. There was however one point in the API where reusing IDs between classes was causing serious issues. Most noticeable this was for the [v1/objects/{id}](/developers/weaviate/api/rest#tag/objects) REST endpoints. If you wanted to retrieve, modify or delete a data object by its ID, you would just need to specify the ID, without specifying the class name. So if the same ID exists for objects in multiple classes (which is fine because of the namespaces per class), Weaviate would not know which object to address and would address all objects with that ID instead. I.e. if you tried to delete an object by ID, this would result in the deletion of all objects with that ID. diff --git a/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx b/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx index 91af619455..2bf03b1ba2 100644 --- a/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx +++ b/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx @@ -25,7 +25,7 @@ Vector databases like [Weaviate](/developers/weaviate/) use Bi-Encoder machine l The models that compute **dense vector embeddings for data** (which later can be used for **search queries**) are so-called *Bi-Encoder models*. Data vectors and query vectors can be compared by computing the similarity (for example cosine similarity) between vectors. All data items in the database can be represented by a vector embedding using a Bi-Encoder model. -> Note. Vector databases **compute** dense vector embeddings **during import**. Calculating vectors up-front coupled with [ANN algorithms](/developers/weaviate/concepts/vector-index) (for faster retrieval) makes working with Bi-Encoder highly efficient. +> Note. Vector databases **compute** dense vector embeddings **during import**. Calculating vectors up-front coupled with [ANN algorithms](/developers/weaviate/concepts/indexing/vector-indexes) (for faster retrieval) makes working with Bi-Encoder highly efficient. ![Vector Database](./img/vector-database.png) diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx b/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx index 1c338f38a7..e48e1d623f 100644 --- a/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx +++ b/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx @@ -98,7 +98,7 @@ Which algorithm works best depends on your project. Performance can be measured So, while ANN is not some magic method that will always find the true k nearest neighbors in a dataset, it can find a pretty good approximation of the true k neighbors. And it can do this in a fraction of the time! ## HNSW in Weaviate -[Weaviate](/developers/weaviate/) is a great example of a vector database that uses ANN algorithms to offer ultra-fast queries. The first ANN algorithm introduced to Weaviate is a custom implementation of [Hierarchical Navigable Small World graphs (HNSW)](/developers/weaviate/concepts/vector-index#hnsw). +[Weaviate](/developers/weaviate/) is a great example of a vector database that uses ANN algorithms to offer ultra-fast queries. The first ANN algorithm introduced to Weaviate is a custom implementation of [Hierarchical Navigable Small World graphs (HNSW)](/developers/weaviate/concepts/indexing/hnsw-indexes). :::tip Read more: -- [Concepts: Vector Indexing # Product Quantization](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq) +- [Concepts: Vector Indexing # Product Quantization](/developers/weaviate/concepts/indexing/hnsw-indexes) - [How-to configure: Indexes # Vector index (see `pq`)](/developers/weaviate/config-refs/schema/vector-index#weaviates-vector-index) ::: diff --git a/blog/2023-09-19-pq-rescoring/index.mdx b/blog/2023-09-19-pq-rescoring/index.mdx index 2ce62b0a20..e77ee057ad 100644 --- a/blog/2023-09-19-pq-rescoring/index.mdx +++ b/blog/2023-09-19-pq-rescoring/index.mdx @@ -35,7 +35,7 @@ If you’d like to dig deeper into the inner workings of PQ and how these compre In this blog we cover how we can take this one step further, with 1.21 we introduced significant improvements to make PQ even more viable by reducing the recall price you pay in order to save on memory requirements. If you want to learn about these improvements and why using PQ should be a no-brainer now for most users, keep reading! :::info Enabling PQ -To learn how to enable and configure Weaviate to use PQ along with HNSW refer to the docs [here](/developers/weaviate/config-refs/schema/vector-index#how-to-configure-hnsw). You can read more about HNSW+PQ in the documentation [here](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq). +To learn how to enable and configure Weaviate to use PQ along with HNSW refer to the docs [here](/developers/weaviate/config-refs/schema/vector-index#how-to-configure-hnsw). You can read more about HNSW+PQ in the documentation [here](/developers/weaviate/concepts/indexing/hnsw-indexes). ::: ## The problem with PQ diff --git a/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx b/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx index c698bec69e..3f53a87d6a 100644 --- a/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx +++ b/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx @@ -56,7 +56,7 @@ We expect that BQ will generally work better for vectors with higher dimensions. When BQ is enabled, a vector cache can be used to improve query performance by storing the quantized vectors of the most recently used data objects. Note that it must be balanced with memory usage considerations. -* Read more about the `flat` index [here](/developers/weaviate/concepts/vector-index#flat-index). +* Read more about the `flat` index [here](/developers/weaviate/concepts/indexing/flat-indexes). ## OSS LLM integration with `generative-anyscale` diff --git a/blog/2024-02-20-grpc-performance-improvements/index.mdx b/blog/2024-02-20-grpc-performance-improvements/index.mdx index ccc60a8279..920d73aa23 100644 --- a/blog/2024-02-20-grpc-performance-improvements/index.mdx +++ b/blog/2024-02-20-grpc-performance-improvements/index.mdx @@ -112,7 +112,7 @@ The chart shows that each query takes 40-70% less time to return results when us Query throughput is usually measured in queries per second (QPS). The chart shows query throughput values based on averages of 10,000 queries.
-The different settings relate to a [vector index (HNSW) parameter](https://weaviate.io/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) `ef`. +The different settings relate to a [vector index (HNSW) parameter](https://weaviate.io/developers/weaviate/concepts/indexing/hnsw-indexes) `ef`. This parameter sets the size of the dynamic candidate list used in the HNSW search algorithm, and drives the speed and accuracy trade-off. In this test, it will impact the ANN-search portion of each query. Regardless, you can see that even at high `ef` numbers the gRPC-driven improvements are significant. diff --git a/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx b/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx index 9c63f59dca..0bd37b9cd8 100644 --- a/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx +++ b/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx @@ -28,7 +28,7 @@ For details, see [Multiple vectors](/developers/weaviate/config-refs/schema/mult ![HNSW and binary quantization](./img/hnsw-and-bq.png) -Weaviate improves [binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) in 1.24 to be faster, more memory efficient, and more cost-effective. Use BQ vector compression with [HNSW indexes](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) to dramatically improve your query speed. +Weaviate improves [binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) in 1.24 to be faster, more memory efficient, and more cost-effective. Use BQ vector compression with [HNSW indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) to dramatically improve your query speed. BQ compresses vector representations while preserving essential information. Uncompressed, Weaviate uses a `float32` to store each dimension. BQ uses one bit per dimension to encode Vector directionality. This means BQ compresses vectors from 32 bits per dimension to 1 bit per dimension - a savings of 32 times the space. This compression significantly reduces storage requirements. @@ -52,7 +52,7 @@ Compare each value in the first vector one with the corresponding value in the s Efficient BQ processing compliments our HNSW index to enable mind dizzyingly fast vector search and throughput! -Let’s talk numbers! You might know from our v1.23 release that combining a [flat index](/developers/weaviate/concepts/vector-index#flat-index) with BQ enables real-time search capabilities. Even so, a brute force a search on 1 million vectors, where each vector has 768 dimensions, takes about 23 milliseconds with a flat index. When you use HNSW and BQ, even with vectors six times as big (4608 dimensions), the same brute force search only takes about 90 milliseconds. +Let’s talk numbers! You might know from our v1.23 release that combining a [flat index](/developers/weaviate/concepts/indexing/flat-indexes) with BQ enables real-time search capabilities. Even so, a brute force a search on 1 million vectors, where each vector has 768 dimensions, takes about 23 milliseconds with a flat index. When you use HNSW and BQ, even with vectors six times as big (4608 dimensions), the same brute force search only takes about 90 milliseconds. A flat index combined with BQ is fast, but compare a HNSW index combined with BQ. Together HNSW and BQ enable vector search capabilities that reach nearly 10,000 queries per second at 85% recall! diff --git a/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx b/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx index 49bad256c8..c82e91b110 100644 --- a/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx +++ b/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx @@ -54,7 +54,7 @@ These numerical representations are usually presented as an array of numbers and ![embeddings](./img/embeddings.png) -At Weaviate, we’ve implemented a custom [HNSW](https://weaviate.io/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) [indexing](https://weaviate.io/developers/weaviate/concepts/vector-index) algorithm to allow for highly performant querying over your dataset which enables all sorts of business use cases at scale. +At Weaviate, we’ve implemented a custom [HNSW](https://weaviate.io/developers/weaviate/concepts/indexing/hnsw-indexes) [indexing](https://weaviate.io/developers/weaviate/concepts/indexing/vector-indexes) algorithm to allow for highly performant querying over your dataset which enables all sorts of business use cases at scale. ## Features for Enterprise diff --git a/blog/2024-04-02-binary-quantization/index.mdx b/blog/2024-04-02-binary-quantization/index.mdx index bf2b5a516b..06a52a6994 100644 --- a/blog/2024-04-02-binary-quantization/index.mdx +++ b/blog/2024-04-02-binary-quantization/index.mdx @@ -160,7 +160,7 @@ So, what's the bottom line? Can Weaviate deliver lightning-fast brute-force sear There are several advantages to brute force searching your data. Firstly, you can bypass the need for data indexing, saving the time required to build the index. While indexing in Weaviate isn't overly sluggish, brute forcing allows you to skip this step entirely. Secondly, you no longer need to store a proximity graph, resulting in further memory savings. In fact, if you opt to brute force search your data directly from disk, memory usage becomes negligible – a mere 100MB is sufficient to host your application. -Weaviate recently introduced the [flat index](/developers/weaviate/concepts/vector-index#flat-index), offering the option to brute force data either from disk (the default behavior) or by retaining only compressed data in memory and fetching a small selection of full vectors from disk for final candidate rescoring. Both approaches expedite data ingestion compared to the traditional HNSW index while also reducing memory consumption. However, if your requirements demand high performance, HNSW remains the preferred choice. Nevertheless, the flat index presents a cost-effective, high-performing alternative. Furthermore, Weaviate now supports binary quantization (BQ) for use with both the flat and HNSW indexes. +Weaviate recently introduced the [flat index](/developers/weaviate/concepts/indexing/flat-indexes), offering the option to brute force data either from disk (the default behavior) or by retaining only compressed data in memory and fetching a small selection of full vectors from disk for final candidate rescoring. Both approaches expedite data ingestion compared to the traditional HNSW index while also reducing memory consumption. However, if your requirements demand high performance, HNSW remains the preferred choice. Nevertheless, the flat index presents a cost-effective, high-performing alternative. Furthermore, Weaviate now supports binary quantization (BQ) for use with both the flat and HNSW indexes. ### Indexing Time Improvements diff --git a/developers/academy/py/compression/100_pq.mdx b/developers/academy/py/compression/100_pq.mdx index fc37b833f8..691bf97094 100644 --- a/developers/academy/py/compression/100_pq.mdx +++ b/developers/academy/py/compression/100_pq.mdx @@ -8,7 +8,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl import PyCode from '!!raw-loader!./_snippets/100_pq.py'; -[Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization) (PQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index), which can improve performance and reduce resource requirements as well as costs. +[Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization) (PQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes), which can improve performance and reduce resource requirements as well as costs. ## What is product quantization? diff --git a/developers/academy/py/compression/200_bq.mdx b/developers/academy/py/compression/200_bq.mdx index 87985d0c40..c26c544f2e 100644 --- a/developers/academy/py/compression/200_bq.mdx +++ b/developers/academy/py/compression/200_bq.mdx @@ -8,7 +8,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl import PyCode from '!!raw-loader!./_snippets/200_bq.py'; -[Binary quantization](/developers/weaviate/concepts/vector-quantization#binary-quantization) (BQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index) or the disk-based [flat index](/developers/weaviate/concepts/vector-index.md#flat-index). +[Binary quantization](/developers/weaviate/concepts/vector-quantization#binary-quantization) (BQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes) or the disk-based [flat index](/developers/weaviate/concepts/indexing/flat-indexes). For HNSW, BQ can decrease its memory footprint and thus improve performance and reduce resource requirements as well as costs. For the flat index, BQ can reduce the size of the index on disk, which can improve performance. @@ -65,4 +65,3 @@ Some BQ parameters are configurable. An important one is `rescore_limit`, which import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/compression/900_next_steps.mdx b/developers/academy/py/compression/900_next_steps.mdx index b0f61c9370..16718c6857 100644 --- a/developers/academy/py/compression/900_next_steps.mdx +++ b/developers/academy/py/compression/900_next_steps.mdx @@ -10,7 +10,7 @@ There are many more resources available to help you continue your learning journ - [How-to: Configure PQ](/developers/weaviate/configuration/compression/pq-compression.md) - [How-to: Configure BQ](/developers/weaviate/configuration/compression/bq-compression.md) -- [Concepts: vector indexes](/developers/weaviate/concepts/vector-index.md): Vector indexes are critical to how Weaviate works, and compression. Read more here. +- [Concepts: vector indexes](/developers/weaviate/concepts/indexing/vector-indexes): Vector indexes are critical to how Weaviate works, and compression. Read more here. - [Configuration references: Vector index](/developers/weaviate/config-refs/schema/vector-index.md): Configuration references for the vector index, including compression settings. import CTASocials from '../_snippets/cta_socials.mdx'; diff --git a/developers/academy/py/tokenization/900_next_steps.mdx b/developers/academy/py/tokenization/900_next_steps.mdx index 64c17c05b5..a61113e313 100644 --- a/developers/academy/py/tokenization/900_next_steps.mdx +++ b/developers/academy/py/tokenization/900_next_steps.mdx @@ -10,7 +10,7 @@ There are many more resources available to help you continue your learning journ - [Refereces: Configuration: Tokenization](/developers/weaviate/config-refs/schema/index.md#tokenization) - [Refereces: Configuration: Stopwords](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--stopwords-stopword-lists) -- [Concepts: Inverted index](/developers/weaviate/concepts/indexing.md#inverted-indexes) +- [Concepts: Inverted index](/developers/weaviate/concepts/indexing/inverted-indexes) - [Concepts: Filtering](/developers/weaviate/concepts/prefiltering.md) :::note diff --git a/developers/academy/py/vector_index/100_overview.mdx b/developers/academy/py/vector_index/100_overview.mdx index eff6aed54e..1d0fbd8d25 100644 --- a/developers/academy/py/vector_index/100_overview.mdx +++ b/developers/academy/py/vector_index/100_overview.mdx @@ -16,7 +16,7 @@ A database index is a data structure that organizes data to make searches more e /> -A [vector index](/developers/weaviate/concepts/vector-index.md) is a specialized type of index that is designed to store and search vectors. +A [vector index](/developers/weaviate/concepts/indexing/vector-indexes) is a specialized type of index that is designed to store and search vectors. The choice and configuration of your vector index can significantly impact the performance of your imports as well as searches, and the resource requirements of your Weaviate instance. @@ -92,4 +92,3 @@ We will discuss these in more detail in the following sections. import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/200_hnsw.mdx b/developers/academy/py/vector_index/200_hnsw.mdx index 3b7c1c0bc9..3b73318446 100644 --- a/developers/academy/py/vector_index/200_hnsw.mdx +++ b/developers/academy/py/vector_index/200_hnsw.mdx @@ -183,7 +183,7 @@ There are more, advanced HNSW parameters that can be set in Weaviate. These are ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) diff --git a/developers/academy/py/vector_index/220_flat.mdx b/developers/academy/py/vector_index/220_flat.mdx index ba9fcb984b..a1c2c0da67 100644 --- a/developers/academy/py/vector_index/220_flat.mdx +++ b/developers/academy/py/vector_index/220_flat.mdx @@ -62,7 +62,7 @@ Each of these parameters can be provided when creating a collection in Weaviate. ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) @@ -72,4 +72,3 @@ Each of these parameters can be provided when creating a collection in Weaviate. import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/250_dynamic.mdx b/developers/academy/py/vector_index/250_dynamic.mdx index 8d8ab5f447..34d7dd8b03 100644 --- a/developers/academy/py/vector_index/250_dynamic.mdx +++ b/developers/academy/py/vector_index/250_dynamic.mdx @@ -67,7 +67,7 @@ Additionally, you can specify any of the `flat` and `hnsw` index parameters that ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) @@ -77,4 +77,3 @@ Additionally, you can specify any of the `flat` and `hnsw` index parameters that import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/900_next_steps.mdx b/developers/academy/py/vector_index/900_next_steps.mdx index 72cb5819ba..efdbcf6972 100644 --- a/developers/academy/py/vector_index/900_next_steps.mdx +++ b/developers/academy/py/vector_index/900_next_steps.mdx @@ -18,7 +18,7 @@ Also, in a multi-tenant environment, the "dynamic" index may be a good default c These resources will help you continue your learning journey: -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) diff --git a/developers/weaviate/concepts/data.md b/developers/weaviate/concepts/data.md index 36500aacda..a6dc80dca6 100644 --- a/developers/weaviate/concepts/data.md +++ b/developers/weaviate/concepts/data.md @@ -1,5 +1,5 @@ --- -title: Data structure +title: Data structures sidebar_position: 10 image: og/docs/concepts.jpg # tags: ['basics'] diff --git a/developers/weaviate/concepts/index.md b/developers/weaviate/concepts/index.md index 7325a98385..772c45d283 100644 --- a/developers/weaviate/concepts/index.md +++ b/developers/weaviate/concepts/index.md @@ -5,16 +5,6 @@ image: og/docs/concepts.jpg # tags: ['getting started'] --- - - - The **Concepts** section explains various aspects related to Weaviate and its architecture to help you get the most out of it. You can read these sections in any order. :::info @@ -31,11 +21,11 @@ If you are after a practical guide, try the [quickstart tutorial](/developers/we - An overview of Weaviate's module system, including what can be done with modules, existing module types, and custom modules. -**[Indexing](./indexing.md)** +**[Indexing](/developers/weaviate/concepts/indexing)** - Read how data is indexed within Weaviate using inverted and ANN indexes, and about configurable settings. -**[Vector indexing](./vector-index.md)** +**[Vector indexing](/developers/weaviate/concepts/indexing/vector-indexes)** - Read more about Weaviate's vector indexing architecture, such as the HNSW algorithm, distance metrics, and configurable settings. diff --git a/developers/weaviate/concepts/indexing.md b/developers/weaviate/concepts/indexing.md deleted file mode 100644 index b0be6b7ff3..0000000000 --- a/developers/weaviate/concepts/indexing.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -title: Indexing -sidebar_position: 20 -image: og/docs/concepts.jpg -# tags: ['basics'] ---- - -Weaviate supports several types of indexes. - -1. **Vector indexes** - a vector index (e.g. HNSW or flat) is used to serve all vector-search queries. -1. **Inverted indexes** - inverted indexes enable BM25 queries, or speed up filtering. - -You can configure indexes in Weaviate per collection. - -Some things to bear in mind: - -* Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. -* A rule of thumb -- if you don't query over a specific field or vector space, don't index it. -* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](../concepts/prefiltering.md)). - -## Vector indexes - -A vector index is used to serve all vector-search queries. Weaviate supports multiple types of vector indexes: - -1. **HNSW** - an approximate nearest neighbor (ANN) search based vector index. HNSW indexes scale well with large datasets. -2. **Flat** - a vector index that is used for brute-force searches. This is useful for small datasets. -2. **Dynamic** - a vector index that is flat when the dataset is small and switches to HNSW when the dataset is large. - -For more information on vector indexes, see the [Vector Indexing](./vector-index.md) page. - -## Inverted indexes - -### Configure the inverted index - -There are three inverted index types in Weaviate: - -- `indexSearchable` - a searchable index for BM25 or hybrid search -- `indexFilterable` - a match-based index for fast [filtering](./prefiltering.md) by matching criteria -- `indexRangeFilters` - a range-based index for [filtering](./prefiltering.md) by numerical ranges - -Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. - -The filterable indexes are only capable of [filtering](./prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). - -So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. - -See the [related how-to section](../manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. - -A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ - -#### Inverted index types summary - -import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; - - - -- Enable one or both of `indexFilterable` and `indexRangeFilters` to index a property for faster filtering. - - If only one is enabled, the respective index is used for filtering. - - If both are enabled, `indexRangeFilters` is used for operations involving comparison operators, and `indexFilterable` is used for equality and inequality operations. - -This chart shows which filter makes the comparison when one or both index type is `true` for an applicable property. - -| Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | -| :- | :- | :- | :- | -| Equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | -| Not equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | -| Greater than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Greater than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | - -#### Inverted index for timestamps - -You can also enable an inverted index to search [based on timestamps](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--indextimestamps). - -Timestamps are currently indexed using the `indexFilterable` index. - -## Collections without indexes - -If you don't want to set an index at all, this is possible too. - -To create a collection without any indexes, skip indexing on the collection and on the properties. - -```js -{ - "class": "Author", - "description": "A description of this collection, in this case, it's about authors", - "vectorIndexConfig": { - "skip": true // <== disable vector index - }, - "properties": [ - { - "indexFilterable": false, // <== disable filterable index for this property - "indexSearchable": false, // <== disable searchable index for this property - "dataType": [ - "text" - ], - "description": "The name of the Author", - "name": "name" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "int" - ], - "description": "The age of the Author", - "name": "age" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "date" - ], - "description": "The date of birth of the Author", - "name": "born" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "boolean" - ], - "description": "A boolean value if the Author won a nobel prize", - "name": "wonNobelPrize" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "indexSearchable": false, // <== disable searchable index for this property - "dataType": [ - "text" - ], - "description": "A description of the author", - "name": "description" - } - ] -} -``` - -## Further resources - -:::info Related pages -- [Concepts: Vector Indexing](./vector-index.md) -- [Configuration: Vector index](../config-refs/schema/vector-index.md) -::: - - - -## Questions and feedback - -import DocsFeedback from '/_includes/docs-feedback.mdx'; - - diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx new file mode 100644 index 0000000000..9ab7f02c7b --- /dev/null +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -0,0 +1,50 @@ +--- +title: Dynamic indexes +sidebar_position: 40 +image: og/docs/indexing.jpg +# tags: ['basics']['indexes'] +--- + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +To configure a dynamic index, see: [Configure dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes). + +## Overview + +Dynamic indexes are flat indexed collections that Weaviate converts to HNSW indexed collections when the collection reaches a certain size. Flat indexes work well for collections with less than 10,000 objects. At that size, flat indexes have low memory overhead and good latency. But, search latency increases as the number of objects in a collection increases. When the collection grows to about 10,000 objects, an HNSW index usually has better latency than a flat index. + +The dynamic index helps to balance resource costs against search latency times. Flat indexes are disc-based. They are responsive at low object counts, but get slower as object counts grow. HNSW indexes reside in RAM. They are very fast, but RAM is expensive. Disk storage is orders of magnitude cheaper than RAM memory, so hosting an index on disc is significantly cheaper than hosting it in RAM. + +If your collection size grows over time, or if you have a mix of smaller and larger tenants, dynamic indexes let you take advantage of lower cost flat indexes while object counts and search latency times are low. When the object count increase, and latencies grow larger, converting the flat index to an HNSW index preserves low search latencies at the expense of increased RAM memory costs. + +The dynamic index automatically switches from a flat index to an HNSW index when the object count exceeds a pre-specified threshold. (The default is 10,000 objects.) + +Dynamic indexing requires [asynchronous indexing](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing). After the dynamic index reaches the threshold value to change from a flat index to an HNSW index, newly added objects are sent to the async queue while Weaviate converts the indexes. When the HNSW index is ready, Weaviate processes the objects in the async queue and adds them to the index. + +## Index transformations + +The dynamic index only changes the index type once. The change only happens in one direction. The dynamic index converts a flat index to an HNSW index. If the size of a collection drops below the threshold value, the dynamic index does not revert the HNSW index back to a flat index. + +## Multi-tenant collections + +The size of tenants in a multi-tenant collection can vary considerably. If you anticipate having tenants of different sizes, consider a dynamic index. The flat index reduces memory overhead for small tenants. If individual tenants grow larger than others, dynamically switching those tenants to HNSW preserves latency while saving on overall resource usage. + +## Other considerations + +- Asynchronous indexing is a prerequisite for dynamic indexing. [Enable asynchronous indexing](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing). +- Configure the flat index and HNSW index when you [define the collection](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#configure-dynamic-indexes). Many [index parameters](/developers/weaviate/config-refs/schema/vector-index) cannot be modified after the collection is created. + +## Related pages + +For more information, see the following: + +- [Configure dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes) +- [Indexing starter guide](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx new file mode 100644 index 0000000000..92351e75c6 --- /dev/null +++ b/developers/weaviate/concepts/indexing/flat-indexes.mdx @@ -0,0 +1,30 @@ +--- +title: Flat indexes +sidebar_position: 30 +image: og/docs/indexing.jpg +# tags: ['basics']['indexes'] +--- + +:::info Added in `v1.23` +::: + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + +[Binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) can improve flat indexes' search speeds. BQ improves search time by reducing the amount of data to read, and speeding up time taken to calculate the distance between vectors. + +To configure a flat index, see: [Configure flat indexes](/developers/weaviate/configuration/indexing-vector/flat-indexes). + +## Related pages + +For more information, see the following: + +- [Configure dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes) +- [Indexing starter guide](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx new file mode 100644 index 0000000000..50fccdadae --- /dev/null +++ b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx @@ -0,0 +1,156 @@ +--- +title: HNSW Indexes +sidebar_position: 20 +image: og/docs/indexing.jpg +# tags: ['basics']['indexes'] +--- + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +import HNSWHow from '/_includes/indexes/hnsw-how.mdx'; + + + +To configure an HNSW index, see: [Configure HNSW indexes](/developers/weaviate/configuration/indexing-vector/hnsw-indexes). + +## Overview + +HNSW is an algorithm that works on multi-layered graphs. It is also an index. An HNSW index is the vector index that Weaviate creates when it applies the HNSW algorithm to the vectors in a collection. Weaviate's [implementation](https://github.com/weaviate/weaviate/tree/main/adapters/repos/db/vector/hnsw) offers full [CRUD-support](https://db-engines.com/en/blog_post/87). + +When Weaviate builds the index, the HNSW algorithm creates a series of layers. The layers improve search because the database engine doesn't have to scan the entire vector space at query time. Instead, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. + +### Example - HNSW search + +Consider this diagram of a vector search that uses HNSW. + +![HNSW layers](../img/hnsw-layers.svg "HNSW layers") + +Layer zero is the lowest layer. Layer zero contains every object in the database, and the objects are well connected to each other. + +Some of the objects are also represented in the layers above layer zero. Each layer above layer zero has fewer objects and fewer connections. + +When HNSW searches the graph, it starts at the highest layer. The algorithm finds the closest matching data points in the highest layer. Then, HNSW goes one layer deeper, and finds the closest matching data points in the lower layer that correspond to the objects in the higher layer. These are the nearest neighbors. + +The HNSW algorithm searches the lower layer and creates list of nearest neighbors. The nearest neighbors list is the starting point for a similar search on the next layer down. The process repeats until the search reaches the lowest (deepest) layer. Finally, the HNSW algorithm returns the data objects that are closest to the search query. + +Since there are relatively few data objects on the higher layers, HNSW 'jumps' over large amounts of data that it doesn't need to search. In contrast, when a data store has only one layer, the search algorithm can't skip unrelated objects. Flat hierarchies mean the search engine has to scan significantly more data objects even though those object are unlikely to match the search criteria. + +Weaviate's HNSW implementation is a very fast, memory efficient, approach to similarity search. The memory cache only stores the highest layer of the index instead of storing all of the data objects from the lowest layer. As a search moves from a higher layer to a lower one, HNSW only adds the data objects that are closest to the search query. This means HNSW uses a relatively small amount of memory compared to other search algorithms. + +The diagram demonstrates how the HNSW algorithm searches. The blue search vector in the top layer connects to a partial result in layer one. The objects in layer one lead HNSW to the result set in layer zero. In this example, HNSW makes three hops through the layers (the dotted blue lines) and skips the objects that are unrelated to the search query. + +## Search quality vs search speed + +HNSW parameters can be adjusted to adjust search quality against speed. + +The `ef` parameter is a critical setting for balancing the trade-off between search speed and quality. + +The `ef` parameter dictates the size of the dynamic list (the nearest neighbors list in the example above) that the HNSW algorithm uses during the search process. A higher `ef` value results in a more extensive search. A higher `ef` value improves accuracy but it can also slow down the query. + +A lower value for `ef` makes the search faster, but it might compromise on accuracy. + +The balance is crucial in scenarios where either speed or accuracy is a priority. For instance, in applications where rapid responses are critical, a lower `ef` might be preferable, even at the expense of some accuracy. In analytical or research contexts where precision is paramount, a higher `ef` might be better, despite the increased query time. + +### Configure ef + +You can set `ef` explicitly or allow Weaviate to set it dynamically. When `ef` is configured dynamically, Weaviate optimizes the balance between speed and recall based on real-time query requirements. If your query patterns are variable, consider setting dynamic `ef`. + +To enable dynamic `ef`, set `ef: -1` in your [collection configuration](/developers/weaviate/configuration/indexing-vector/hnsw-indexes#configure-an-hnsw-index). When dynamic `ef` is enabled, Weaviate adjusts the size of the ANN list based on the query response limit. The calculation also takes into account the values of `dynamicEfMin`, `dynamicEfMax`, and `dynamicEfFactor`. + +### Configure dynamic ef + +The `ef` parameter controls the size of the ANN list at query time. You can configure a specific list size or else let Weaviate configure the list dynamically. If you choose dynamic `ef`, Weaviate provides several options to control the size of the ANN list. + +The length of the list is determined by the query response limit that you set in your query. Weaviate uses the query limit as an anchor and modifies the size of ANN list according to the values you set for the `dynamicEf` parameters. + +- `dynamicEfMin` sets a lower bound on the list length. +- `dynamicEfMax` sets an upper bound on the list length. +- `dynamicEfFactor` sets a range for the list. + +To keep search recall high, the actual dynamic `ef` value stays above `dynamicEfMin` even if the query limit is small enough to suggest a lower value. + +To keep search speed reasonable even when retrieving large result sets, the dynamic `ef` value is limited to `dynamicEfMax`. Weaviate doesn't exceed `dynamicEfMax` even if the query limit is large enough to suggest a higher value. + +If the query limit is higher than `dynamicEfMax`, `dynamicEfMax` does not have any effect. In this case, the dynamic `ef` value is equal to the query limit. + +To determine the length of the ANN list, Weaviate multiples the query limit by `dynamicEfFactor`. The list range is modified by `dynamicEfMin` and `dynamicEfMax`. + +### Dynamic ef example + +Consider this GraphQL query that sets a limit of 4. + +```graphql +{ + Get { + JeopardyQuestion(limit: 4) { + answer + question + } + } +} +``` + +Imagine the collection has dynamic `ef` configured. + +```json + "vectorIndexConfig": { + "ef": -1, + "dynamicEfMin": 5 + "dynamicEfMax": 25 + "dynamicEfFactor": 10 + } +``` + +The resulting search list has these characteristics. + +- A potential length of 40 objects ( ("dynamicEfFactor": 10) * (limit: 4) ). +- A minimum length of 5 objects ("dynamicEfMin": 5). +- A maximum length of 25 objects ("dynamicEfMax": 25). +- An actual size of 5 to 25 objects. + +#### Related environment variables + +If you use the [`docker-compose.yml` file from Weaviate](/developers/weaviate/installation/docker-compose) to run your local instance, the `QUERY_DEFAULTS_LIMIT` environment variable sets a reasonable default query limit. + +To prevent out of memory errors, the default setting for `QUERY_DEFAULTS_LIMIT` is significantly lower than `QUERY_MAXIMUM_RESULTS`. + +To change the default limit, edit the value for `QUERY_DEFAULTS_LIMIT` when you configure your Weaviate instance. + +## Deletions + +Deleted objects are marked as deleted, but Weaviate doesn't remove them immediately. Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. During cleanup, Weaviate updates the graph and actually removes the deleted objects from the collection. + +Cleanup can affect system performance if your instance has a large number of deletes. These environment variables tune the cleanup process: + +- `TOMBSTONE_DELETION_CONCURRENCY` +- `TOMBSTONE_DELETION_MAX_PER_CYCLE` +- `TOMBSTONE_DELETION_MIN_PER_CYCLE` + +For more details, see [environment variables](/developers/weaviate/config-refs/env-vars). + +## Compression + +Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. Weaviate offers several ways to compress your data: + +import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; + + + +## Other considerations + +HNSW indexes enable very fast queries, but they are not as fast at import time. Rebuilding the index when you add new vectors can be resource intensive. If you use HNSW, consider [enabling asynchronous indexing](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing) to improve system response during imports. + +## Related pages + +For more information, see the following: + +- [Configure dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes) +- [Indexing starter guide](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/concepts/img/supermarket.svg b/developers/weaviate/concepts/indexing/img/supermarket.svg similarity index 100% rename from developers/weaviate/concepts/img/supermarket.svg rename to developers/weaviate/concepts/indexing/img/supermarket.svg diff --git a/developers/weaviate/concepts/img/vectors-2d.png b/developers/weaviate/concepts/indexing/img/vectors-2d.png similarity index 100% rename from developers/weaviate/concepts/img/vectors-2d.png rename to developers/weaviate/concepts/indexing/img/vectors-2d.png diff --git a/developers/weaviate/concepts/img/vectors-2d.svg b/developers/weaviate/concepts/indexing/img/vectors-2d.svg similarity index 100% rename from developers/weaviate/concepts/img/vectors-2d.svg rename to developers/weaviate/concepts/indexing/img/vectors-2d.svg diff --git a/developers/weaviate/concepts/indexing/index.md b/developers/weaviate/concepts/indexing/index.md new file mode 100644 index 0000000000..414e7ae39b --- /dev/null +++ b/developers/weaviate/concepts/indexing/index.md @@ -0,0 +1,44 @@ +--- +title: Indexing +sidebar_position: 1 +image: og/docs/indexing.jpg +# tags: ['basics']['indexes'] +--- + +This section of the documentation discusses indexing concepts as they relate to Weaviate. + +For configuration examples without additional contextual material, see the [Vector index](/developers/weaviate/configuration/indexing-vector) and [Inverted index](/developers/weaviate/configuration/inverted-indexes) configuration pages. + +## Vector indexes + +Vector embeddings encapsulate meaning for the objects that they represent. Vector indexes make it easier to efficiently search large vector spaces in order to find objects with similar vector embeddings. + +Weaviate offers these vector index types: + +- [Hierarchical Navigable Small World (HNSW) indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) +- [Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) +- [Dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes) + +## Inverted indexes + +Inverted indexes, also known as keyword indexes, make textual and numeric searches more efficient. Inverted indexes use the [BM25](/developers/weaviate/concepts/indexing/inverted-indexes#bm25-ranking-algorithm) ranking algorithm. + +Weaviate offers these inverted index types: + +- indexSearchable +- indexFilterable +- indexRangeFilters + +## Related pages + +For more information, see the following: + +- [Configure vector indexes](/developers/weaviate/configuration/indexing-vector) +- [Configure inverted indexes](/developers/weaviate/configuration/inverted-indexes) +- [Indexing starter guide](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md new file mode 100644 index 0000000000..2657c52dea --- /dev/null +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -0,0 +1,122 @@ +--- +title: Inverted indexes +sidebar_position: 100 +image: og/docs/indexing.jpg +# tags: ['basics'] +--- + +[Inverted indexes](https://en.wikipedia.org/wiki/Inverted_index) map the contents of documents to improve search performance. The Weaviate database uses inverted indexes to search object properties efficiently. + +There are three inverted index types in Weaviate: + +- [`indexSearchable`](#indexsearchable) A default index. BM25 (keyword) and hybrid search use `indexSearchable`. +- [`indexFilterable`](#indexfilterable) A default index. Filters use the `indexFilterable` index. +- [`indexRangeFilters`](#indexrangefilters) An optional index. Numerical filters use the `indexRangeFilter`. + +## Configuration + +These indexes are set on the property level. `indexSearchable` and `indexFilterable` indexes are enabled by default. The `indexRangeFilters` index is off by default. + +If you do not plan to search or filter on an object property, disable the index. Unused indexes waste space and they slow import processing. + +To configure an inverted index see these examples: + +- [`indexSearchable`](/developers/weaviate/configuration/inverted-indexes#indexsearchable) +- [`indexFilterable`](/developers/weaviate/configuration/inverted-indexes#indexfilterable) +- [`indexRangeFilters](/developers/weaviate/configuration/inverted-indexes#indexrangefilters) + +Keyword searches and hybrid searches use the BM25 ranking algorithm in conjunction with the `indexSearchable` index. To configure the BM25 algorithm, see these examples: + +- [BM25](/developers/weaviate/configuration/inverted-indexes#bm25) + +## indexSearchable + +[Keyword searches](/developers/weaviate/search/bm25) and [hybrid searches](/developers/weaviate/search/hybrid) use the BM25 ranking algorithm in conjunction with the `indexSearchable` index. These searches match the search terms in a query to the same term in the database. They only match terms, not meanings. To match meanings, a semantic search, use a [vector similarity search](/developers/weaviate/search/similarity). + +`indexSearchable` is enabled by default. This index is required for BM25 search. + +For simple (non-BM25) filters, the `indexFilterable` index enables faster filtering. However, if the `indexFilterable` index is not enabled, Weaviate uses the `indexSearchable` index to [filter](/developers/weaviate/concepts/prefiltering). + +For configuration examples, see [`indexSearchable`](/developers/weaviate/configuration/inverted-indexes#indexsearchable). + +## indexFilterable + +`indexFilterable` is enabled by default. This index is not required for filtering of BM25 search. However, this index is much faster for filtering than the `indexSearchable` index. + +You can also enable an inverted index to search [based on timestamps](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--indextimestamps). +If you do not expect to filter on a property often, disable this filter to save space and to improve import times. For occasional filtered queries, the `indexSearchable` index is sufficient. If you expect to filter on a property regularly, leave this index enabled. + +For configuration examples, see [`indexFilterable`](/developers/weaviate/configuration/inverted-indexes#indexfilterable). + +## indexRangeFilters + +:::info Added in `1.26` +::: + +The `indexRangeFilters` index is for filtering by numerical ranges. This index is not enabled by default. + +Internally, rangeable indexes are implemented as [roaring bitmap slices](https://www.featurebase.com/blog/range-encoded-bitmaps). The roaring bitmap data structure limits the `indexRangeFilters` index to values that can be stored as 64 bit integers. This index is available for properties of type `int`, `number`, or `date`. The `indexRangeFilters` index is not available for arrays of these data types. + +The `indexRangeFilters` index is only available for new properties. Existing properties cannot be converted to use the rangeable index. + +For configuration examples, see [`indexRangeFilters](/developers/weaviate/configuration/inverted-indexes#indexrangefilters). + +## Inverted index types summary + +import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; + + + +## Multiple filter indexes + +`indexFilterable` and `indexRangeFilters` both enable faster filtering. + +- If only one of them is enabled on a property, that index is used for filtering. +- If both are enabled on a property, `indexRangeFilters` is used with numeric comparison operators, and `indexFilterable` is used for equality and inequality operations. + +This chart shows Weaviate applies the filters: + +| Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | +| :- | :- | :- | :- | +| Equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | +| Not equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | +| Greater than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Greater than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | + +## Collection level settings + +Some inverted index values are configured at the collection level. These settings allow you do create indexes for [timestamps](/developers/weaviate/config-refs/schema#indextimestamps), filter on [`null`](/developers/weaviate/config-refs/schema#indexnullstate) and filter on [property length](/developers/weaviate/config-refs/schema#indexpropertylength). + +If you so not intend to filter on these properties, do not enable these indexes. + +## BM25 ranking algorithm + +The [BM25 ranking algorithm](https://en.wikipedia.org/wiki/Okapi_BM25) ranks documents based on how frequently particular words appear in the document. Weaviate uses BM25 with [keyword searches](/developers/weaviate/search/bm25) and [hybrid searches](/developers/weaviate/search/hybrid). + +The `indexSearchable` index enables BM25 on a property. You cannot do a hybrid search or a keyword search on a property if `indexSearchable` is disabled. + +The BM25 algorithm has two parameters that change how the `indexSearchable` index is created: + +- To adjust for document length, modify `b`. Values range from 0 to 1. +- To adjust for word frequency within a document, modify `k1`. Values are usually in the range from 0 to 3. There isn't an upper limit. + +For configuration examples, see [BM25](/developers/weaviate/configuration/inverted-indexes#bm25). + + +## Further resources + +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Configuration: Inverted indexes](/developers/weaviate/configuration/inverted-indexes) +- [Concepts: Inverted Indexes](/developers/weaviate/concepts/indexing/inverted-indexes) +- [Configuration: BM25 algorithm](/developers/weaviate/configuration/inverted-indexes#bm25) +- [Blog: Ranking models](https://weaviate.io/blog/ranking-models-for-better-search) +- [Blog: Hybrid search](https://weaviate.io/blog/hybrid-search-for-web-developers) +- [Blog: Fusion algorithms](https://weaviate.io/blog/hybrid-search-fusion-algorithms) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md new file mode 100644 index 0000000000..afe3f74ac6 --- /dev/null +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -0,0 +1,151 @@ +--- +title: Vector indexes overview +sidebar_position: 10 +image: og/docs/indexing.jpg +# tags: ['vector index plugins'] +--- + +Weaviate is a vector database. Most objects in Weaviate collections have one or more vectors. Individual vectors can have thousands of dimensions. Collections can have millions of objects. The resulting vector space can be exceedingly large. + +Weaviate uses vector indexes to [efficiently search](https://weaviate.io/blog/why-is-vector-search-so-fast) the vector space. Different vector index types offer trade-offs in resource use, speed, and accuracy. + +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; + + + +## Vector indexing + +[Vector embeddings](https://weaviate.io/blog/vector-embeddings-explained) are arrays of elements that can capture meaning. The original data can come from text, images, videos, or other content types. A model transforms the underlying data into an embedding. The elements in the embedding are called dimensions. High dimension vectors, with thousands of elements, capture more information, but they are harder to work with. + +Vector databases make it easier to work with high dimensional vector embeddings. Embeddings that capture similar meanings are closer to each other than embeddings that capture different meanings. To find objects that have similar semantics, vector databases must efficiently calculate the "distance" between the objects' embeddings. + +In Weaviate, the [distance calculation method](/developers/weaviate/manage-data/collections#specify-a-distance-metric) is configurable. The [distance threshold](/developers/weaviate/search/similarity#set-a-similarity-threshold) is also configurable. A lower threshold returns more specific results. + +### Example - relative dimensions in space + +The elements in a vector define a point in a multi-dimensional space, similar to how the coordinates (X,Y) define a point on a graph and (X,Y,Z) define a point in three dimensional space. + +Consider a very simple vector embedding that uses two elements to represent the meanings of some English words. Taken together, the vectors in the collection define a two dimensional space like this one: + +![2D Vector embedding visualization](./img/vectors-2d.png "2D Vectors visualization") + +Apples and bananas are both fruits. Newspapers and magazines are both publications. In the representation, the fruits are close to each other. The publications are also close to each other, but relatively far from the fruits. The vector distances are smaller between items with similar semantics and larger between objects that don't share similar meanings. + +In a real example, the embeddings would have hundreds or thousands of elements. The vector space is difficult to visualize, but the concept is the same. Similar embeddings capture similar meanings and are closer to each other than to embeddings that capture different meanings. + +For more details on this representation, see the [GloVe model](https://github.com/stanfordnlp/GloVe) from Stanford or our [vector embeddings blog post](https://weaviate.io/blog/vector-embeddings-explained#what-exactly-are-vector-embeddings). + +### Example - supermarket layout + +Consider how items are arranged in a supermarket. Similar products are close to each other. Apples and bananas are close to each other in the fruit section. Newspapers and magazines aren't fruit. They aren't even food. They are displayed in a different section of the supermarket. + +![Supermarket map visualization as analogy for vector indexing](./img/supermarket.svg "Supermarket map visualization") + +If you walk into the supermarket to buy an apple, you move away from the publications and move towards the fruits. You minimize the distance from where you are to the object you are searching for. + +Vector search works the same way. Weaviate turns search queries into vector embeddings and then searches the collection to find objects that are close to the query in the vector space. + +## HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +import HNSWHow from '/_includes/indexes/hnsw-how.mdx'; + + + +Weaviate offers these methods to compress ("quantize") your HNSW index: + +import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; + + + +For more details, see [HNSW indexes](/developers/weaviate/concepts/indexing/hnsw-indexes). + +## Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + +[Binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) is a compression technique that also improves search speed for flat indexes. BQ reduces the amount of data the search engine reads. It also permits efficient binary calculations. These benefits of compression shorten the time needed to calculate vector distances during search. + +For more details, see [flat indexes](/developers/weaviate/concepts/indexing/flat-indexes). + +## Dynamic indexes + +:::info Added in `v1.25` +::: + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +For more details, see [dynamic indexes](/developers/weaviate/concepts/indexing/dynamic-indexes). + +## Asynchronous indexing + +:::info Added in `v1.22` +::: + +Asynchronous indexing separates indexing and object creation. Weaviate creates new objects faster. The vector index updates in the background. + +While the vector index is updating, Weaviate can search a maximum of 100,000 un-indexed objects by brute force. Brute force searches are slower, so search performance drops during indexing. The 100,000 object limit means that any un-indexed objects beyond the limit are not included in the search. + +Asynchronous indexing is not enabled by default. [Enable asynchronous indexing](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing). + +## Compression + +Weaviate stores objects and vector representations of those objects (vectors). Vectors can be very large. Vector dimensions are stored as 32 bit floats. A single vector with 1536 dimensions uses about 6 KB of storage. When collections have millions of objects, the resulting size can lead to significant costs, especially where an in-memory vector index is used. + +Consider [enabling compression](/developers/weaviate/configuration/compression) to manage system resources. + +## Vector cache considerations + +Weaviate uses an in-memory cache to hold vectors in RAM. + +Each import requires multiple searches for indexing. By default, the cache is set to one trillion (`1e12`) objects when a new collection is created. During data import, set [`vectorCacheMaxObjects`](/developers/weaviate/configuration/indexing-vector) high enough that all of the vectors can be held in memory. Import performance drops drastically when there isn't enough memory to hold all of the vectors in the cache. + +After import, when your workload is mostly querying, experiment with vector cache limits that are less than your total dataset size. + +Vectors that aren't currently in cache are added to the cache until it reaches the maximum size. If the cache fills, Weaviate drops the whole cache. Subsequent queries force data to be read from disk until the cache fills up again. + +Depending of query patterns, you may be able to tune the cache to hold frequently queried vectors in memory. Then, search only has to do slower disk lookups for less frequent queries. + +## Benchmarks + +The [ANN benchmark page](/developers/weaviate/benchmarks/ann.md) has a wide variety of vector search use cases and relative benchmarks. + +Consult the page to find a dataset similar to yours. The benchmarks are a good starting point to learn to optimal configuration settings for each type of workload. + +## Recommendations + +- Large collections or tenants with more than 10,000 objects should use the HNSW index. + +- Collections or tenants with less than 10,000 objects should use the flat index. + +- Collections, especially multi-tenant collections with a variety of tenant sizes, should use a dynamic index. + +## Other considerations + +- If a collection consists of references between two other collections, don't index it. + +- Duplicate vectors are expensive to import. Avoid importing duplicate vectors since import speeds are very slow. + +- The vector index type specifies how vectors are indexed. The `vectorizer` parameter specifies how to create a vector embedding. See [model providers](/developers/weaviate/model-providers) or set `vectorizer` to `none` to import your own vectors. + +## Further resources + +- [Concepts: Inverted indexes](/developers/weaviate/concepts/indexing/inverted-indexes) +- [Concepts: Vector quantization (compression)](/developers/weaviate/concepts/vector-quantization) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index) +- [Configuration: Schema (Configure semantic indexing)](/developers/weaviate/config-refs/schema#configure-semantic-indexing) +- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) +- [Blog post: Exploring ANN algorithms Part 1](https://weaviate.io/blog/ann-algorithms-vamana-vs-hnsw) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/developers/weaviate/concepts/interface.md b/developers/weaviate/concepts/interface.md index 4218ad5d28..9219de8f72 100644 --- a/developers/weaviate/concepts/interface.md +++ b/developers/weaviate/concepts/interface.md @@ -1,6 +1,6 @@ --- -title: Interface -sidebar_position: 85 +title: Interfaces +sidebar_position: 40 image: og/docs/concepts.jpg # tags: ['architecture', 'interface', 'API design'] --- diff --git a/developers/weaviate/concepts/modules.md b/developers/weaviate/concepts/modules.md index 1864e64f02..2da6295da2 100644 --- a/developers/weaviate/concepts/modules.md +++ b/developers/weaviate/concepts/modules.md @@ -1,6 +1,6 @@ --- title: Modules -sidebar_position: 15 +sidebar_position: 50 image: og/docs/concepts.jpg # tags: ['modules'] --- @@ -15,7 +15,7 @@ Weaviate has a modularized structure. Functionality such as vectorization or bac The core of Weaviate, without any modules attached, is a pure vector-native database. [![Weaviate modules introduction](./img/weaviate-module-diagram.svg "Weaviate Module Diagram")](./img/weaviate-module-diagram.svg) -Data is stored in Weaviate as the combination of an object and its vector, and these vectors are searchable by the provided [vector index algorithm](../concepts/vector-index.md). Without any vectorizer modules attached, Weaviate does not know how to *vectorize* an object, i.e. *how* to calculate the vectors given an object. +Data is stored in Weaviate as the combination of an object and its vector, and these vectors are searchable by the provided [vector index algorithm](/developers/weaviate/concepts/indexing/vector-indexes). Without any vectorizer modules attached, Weaviate does not know how to *vectorize* an object, i.e. *how* to calculate the vectors given an object. Depending on the type of data you want to store and search (text, images, etc.), and depending on the use case (like search, question answering, etc., depending on language, classification, ML model, training set, etc.), you can choose and attach a vectorizer module that best fits your use case. Or, you can "bring your own" vectors to Weaviate. diff --git a/developers/weaviate/concepts/prefiltering.md b/developers/weaviate/concepts/prefiltering.md index 2dbabaede9..0599f40379 100644 --- a/developers/weaviate/concepts/prefiltering.md +++ b/developers/weaviate/concepts/prefiltering.md @@ -1,6 +1,6 @@ --- title: Filtering -sidebar_position: 26 +sidebar_position: 20 image: og/docs/concepts.jpg # tags: ['architecture', 'filtered vector search', 'pre-filtering'] --- @@ -60,11 +60,11 @@ To learn more about Weaviate's roaring bitmaps implementation, see the [in-line :::info Added in `1.26` ::: -Weaviate `1.26` introduces the `indexRangeFilters` index, which is a range-based index for filtering by numerical ranges. This index is available for `int`, `number`, or `date` properties. The index is not available for arrays of these data types. +Weaviate `1.26` introduces the `indexRangeFilters` index for filtering by numerical ranges. This index is available for properties of type `int`, `number`, or `date`. The index is not available for arrays of these data types. Internally, rangeable indexes are implemented as [roaring bitmap slices](https://www.featurebase.com/blog/range-encoded-bitmaps). This data structure limits the index to values that can be stored as 64 bit integers. -`indexRangeFilters` is only available for new properties. Existing properties cannot be converted to use the rangeable index. +The `indexRangeFilters` index is only available for new properties. Existing properties cannot be converted to use the rangeable index. ## Recall on Pre-Filtered Searches diff --git a/developers/weaviate/concepts/replication-architecture/_category_.json b/developers/weaviate/concepts/replication-architecture/_category_.json index 4cf28198bc..8c7776636f 100644 --- a/developers/weaviate/concepts/replication-architecture/_category_.json +++ b/developers/weaviate/concepts/replication-architecture/_category_.json @@ -1,4 +1,4 @@ { "label": "Replication Architecture", - "position": 35 + "position": 5 } \ No newline at end of file diff --git a/developers/weaviate/concepts/resources.md b/developers/weaviate/concepts/resources.md index 6b74daaf7d..26408aee41 100644 --- a/developers/weaviate/concepts/resources.md +++ b/developers/weaviate/concepts/resources.md @@ -1,6 +1,6 @@ --- title: Resource Planning -sidebar_position: 90 +sidebar_position: 60 image: og/docs/concepts.jpg # tags: ['architecture', 'resource', 'cpu', 'memory', 'gpu'] --- @@ -45,7 +45,7 @@ When search throughput is limited, add CPUs to increase the number of queries pe Memory determines the maximum supported dataset size. Memory does not directly influence query speed. ::: -The HNSW index must be stored in memory. The memory required is directly related to the size of your dataset. There is no correlation between the size of your dataset and the current query load. You can use [`product quantization (PQ)`](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantization-pq) to compress the vectors in your dataset in increase the number of vectors your can hold in memory. +The HNSW index must be stored in memory. The memory required is directly related to the size of your dataset. There is no correlation between the size of your dataset and the current query load. You can use [`product quantization (PQ)`](/developers/weaviate/concepts/indexing/hnsw-indexes) to compress the vectors in your dataset in increase the number of vectors your can hold in memory. Weaviate let's you configure a limit to the number of vectors held in memory in order to prevent unexpected Out-of-Memory ("OOM") situations. The default value is one trillion (`1e12`) objects. per collection. To adjust the number of objects, update the value of [`vectorCacheMaxObjects`](../config-refs/schema/vector-index.md) in your index settings. @@ -102,7 +102,7 @@ To avoid out-of-memory issues during imports, set `LIMIT_RESOURCES` to `True` or The following tactics can help to reduce Weaviate's memory usage: -- **Use vector compression**. Product quantization (PQ) is a technique that reduces the size of vectors. Vector compression impacts recall performance, so we recommend testing PQ on your dataset before using it in production.

For more information, see [Product Quantization](../concepts/vector-index.md#hnsw-with-product-quantization-pq).
To configure PQ, see [Compression](../configuration/compression/pq-compression.md). +- **Use vector compression**. Product quantization (PQ) is a technique that reduces the size of vectors. Vector compression impacts recall performance, so we recommend testing PQ on your dataset before using it in production.

For more information, see [Product Quantization](/developers/weaviate/concepts/indexing/hnsw-indexes).
To configure PQ, see [Compression](../configuration/compression/pq-compression.md). - **Reduce the dimensionality of your vectors.** The most effective approach to reducing memory size, is to reduce the number of dimensions per vector. If you have high dimension vectors, consider using a model that uses fewer dimensions. For example, a model that has 384 dimensions uses far less memory than a model with 1536 dimensions. diff --git a/developers/weaviate/concepts/search/_category_.json b/developers/weaviate/concepts/search/_category_.json index 47a6117993..eb356e15df 100644 --- a/developers/weaviate/concepts/search/_category_.json +++ b/developers/weaviate/concepts/search/_category_.json @@ -1,4 +1,4 @@ { "label": "Search", - "position": 5 + "position": 7 } diff --git a/developers/weaviate/concepts/storage.md b/developers/weaviate/concepts/storage.md index 6b358f0ff9..c0db13ceef 100644 --- a/developers/weaviate/concepts/storage.md +++ b/developers/weaviate/concepts/storage.md @@ -1,6 +1,6 @@ --- title: Storage -sidebar_position: 18 +sidebar_position: 70 image: og/docs/concepts.jpg # tags: ['architecture', 'storage'] --- @@ -24,7 +24,7 @@ Each shard houses three main components: * An object store, essentially a key-value store * An [inverted index](https://en.wikipedia.org/wiki/Inverted_index) -* A vector index store (plugable, currently a [custom implementation of HNSW](/developers/weaviate/concepts/vector-index.md#hnsw)) +* A vector index store (plugable, currently a [custom implementation of HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes)) #### Object and Inverted Index Store diff --git a/developers/weaviate/concepts/vector-index.md b/developers/weaviate/concepts/vector-index.md deleted file mode 100644 index 31b8aa4a2a..0000000000 --- a/developers/weaviate/concepts/vector-index.md +++ /dev/null @@ -1,248 +0,0 @@ ---- -title: Vector Indexing -sidebar_position: 23 -image: og/docs/concepts.jpg -# tags: ['vector index plugins'] ---- - -Vector indexing is a key component of vector databases. It can help to [significantly **increase the speed** of the search process of similarity search](https://weaviate.io/blog/why-is-vector-search-so-fast) with only a minimal tradeoff in search accuracy ([HNSW index](#hierarchical-navigable-small-world-hnsw-index)), or efficiently store many subsets of data in a small memory footprint ([flat index](#flat-index)). The [dynamic index](#dynamic-index) can even start off as a flat index and then dynamically switch to the HNSW index as it scales past a threshold. - -Weaviate's vector-first storage system takes care of all storage operations with a vector index. Storing data in a vector-first manner not only allows for semantic or context-based search, but also makes it possible to store *very* large amounts of data without decreasing performance (assuming scaled well horizontally or having sufficient shards for the indexes). - -Weaviate supports these vector index types: -* [flat index](#flat-index): a simple, lightweight index that is designed for small datasets. -* [HNSW index](#hierarchical-navigable-small-world-hnsw-index): a more complex index that is slower to build, but it scales well to large datasets as queries have a logarithmic time complexity. -* [dynamic index](#dynamic-index): allows you to automatically switch from a flat index to an HNSW index as object count scales - -:::caution Experimental feature -Available starting in `v1.25`. This is an experimental feature. Use with caution. -::: - -This page explains what vector indexes are, and what purpose they serve in the Weaviate vector database. - -## Why do you need vector indexing? - -[Vector embeddings](https://weaviate.io/blog/vector-embeddings-explained) are a great way to represent meaning. Vectors embeddings are arrays of elements that can capture meaning from different data types, such as texts, images, videos, and other content. The number of elements are called dimensions. High dimension vectors capture more information, but they are harder to work with. - -Vector databases make it easier to work with high dimensional vectors. Consider search; Vector databases efficiently measure semantic similarity between data objects. When you run a [similarity search](https://weaviate.io/developers/weaviate/search/similarity), a vector database like Weaviate uses a vectorized version of the query to find objects in the database that have vectors similar to the query vector. - -Vectors are like coordinates in a multi-dimensional space. A very simple vector might represent objects, *words* in this case, in a 2-dimensional space. - -In the graph below, the words `Apple` and `Banana` are shown close to each other. `Newspaper` and `Magazine` are also close to each other, but they are far away from `Apple` and `Banana` in the same vector space. - -Within each pair, the distance between words is small because the objects have similar vector representations. The distance between the pairs is larger because the difference between the vectors is larger. Intuitively, fruits are similar to each other, but fruits are not similar to reading material. - -For more details of this representation, see: ([GloVe](https://github.com/stanfordnlp/GloVe)) and [vector embeddings](https://weaviate.io/blog/vector-embeddings-explained#what-exactly-are-vector-embeddings). - -![2D Vector embedding visualization](./img/vectors-2d.png "2D Vectors visualization") - -Another way to think of this is how products are placed in a supermarket. You'd expect to find `Apples` close to `Bananas`, because they are both fruit. But when you are searching for a `Magazine`, you would move away from the `Apples` and `Bananas`, more towards the aisle with, for example, `Newspapers`. This is how the semantics of concepts can be stored in Weaviate as well, depending on the module you're using to calculate the numbers in the vectors. Not only words or text can be indexed as vectors, but also images, video, DNA sequences, etc. Read more about which model to use [here](/developers/weaviate/modules/index.md). - -![Supermarket map visualization as analogy for vector indexing](./img/supermarket.svg "Supermarket map visualization") - -:::tip -You might be also interested in our blog post [Why is vector search to fast?](https://weaviate.io/blog/why-is-vector-search-so-fast). -::: - -## Hierarchical Navigable Small World (HNSW) index - -**Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive. - -Weaviate's `hnsw` index is a [custom implementation](../more-resources/faq.md#q-does-weaviate-use-hnswlib) of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). - -At build time, the HNSW algorithm creates a series of layers. At query time, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. - -Consider this diagram of a vector search using HNSW. - -![HNSW layers](./img/hnsw-layers.svg "HNSW layers") - -An individual object can exist in more than one layer, but every object in the database is represented in the lowest layer (layer zero in the picture). The layer zero data objects are very well connected to each other. Each layer above the lowest layer has fewer data object, and fewer connections. The data objects in the higher layers correspond to the objects in the lower layers, but each higher layer has exponentially fewer objects than the layer below. The HNSW algorithm takes advantage of the layers to efficiently process large amounts of data. - -When a search query comes in, the HNSW algorithm finds the closest matching data points in the highest layer. Then, HNSW goes one layer deeper, and finds the closest data points in that layer to the ones in the higher layer. These are the nearest neighbors. The algorithm searches the lower layer to create a new list of nearest neighbors. Then, HNSW uses the new list and repeats the process on the next layer down. When it gets to the deepest layer, the HNSW algorithm returns the data objects closest to the search query. - -Since there are relatively few data objects on the higher layers, HNSW has to search fewer objects. This means HNSW 'jumps' over large amounts of data that it doesn't need to search. When a data store has only one layer, the search algorithm can't skip unrelated objects. It has to search significantly more data objects even though they are unlikely to match. - -HNSW is very fast, memory efficient, approach to similarity search. The memory cache only stores the highest layer instead of storing all of the data objects in the lowest layer. When the search moves from a higher layer to a lower one, HNSW only adds the data objects that are closest to the search query. This means HNSW uses a relatively small amount of memory compared to other search algorithms. - -Have another look at the diagram; it demonstrates how the HNSW algorithm searches. The blue search vector in the top layer connects to a partial result in layer one. The objects in layer one lead HNSW to the result set in layer zero. HNSW makes three hops through the layers (the dotted blue lines) and skips objects that are unrelated to the search query. - -If your use case values fast data upload higher than super fast query time and high scalability, then other vector index types may be a better solution (e.g. [Spotify's Annoy](https://github.com/spotify/annoy)). - -### Managing search quality vs speed tradeoffs - -HNSW parameters can be adjusted to adjust search quality against speed. - -The `ef` parameter is a critical setting for balancing the trade-off between search speed and quality. - -The `ef` parameter dictates the size of the dynamic list used by the HNSW algorithm during the search process. A higher `ef` value results in a more extensive search, enhancing accuracy but potentially slowing down the query. - -In contrast, a lower `ef` makes the search faster but might compromise on accuracy. This balance is crucial in scenarios where either speed or accuracy is a priority. For instance, in applications where rapid responses are critical, a lower `ef` might be preferable, even at the expense of some accuracy. Conversely, in analytical or research contexts where precision is paramount, a higher `ef` would be more suitable, despite the increased query time. - -`ef` can be configured explicitly or dynamically. This feature is particularly beneficial in environments with varying query patterns. When `ef` is configured dynamically, Weaviate optimizes the balance between speed and recall based on real-time query requirements. - -To enable dynamic `ef`, set `ef`: -1. Weaviate adjusts the size of the ANN list based on the query response limit. The calculation also takes into account the values of `dynamicEfMin`, `dynamicEfMax`, and `dynamicEfFactor`. - -### Dynamic ef - -The `ef` parameter controls the size of the ANN list at query time. You can configure a specific list size or else let Weaviate configure the list dynamically. If you choose dynamic `ef`, Weaviate provides several options to control the size of the ANN list. - -The length of the list is determined by the query response limit that you set in your query. Weaviate uses the query limit as an anchor and modifies the size of ANN list according to the values you set for the `dynamicEf` parameters. - -- `dynamicEfMin` sets a lower bound on the list length. -- `dynamicEfMax` sets an upper bound on the list length. -- `dynamicEfFactor` sets a range for the list. - -To keep search recall high, the actual dynamic `ef` value stays above `dynamicEfMin` even if the query limit is small enough to suggest a lower value. - -To keep search speed reasonable even when retrieving large result sets, the dynamic `ef` value is limited to `dynamicEfMax`. Weaviate doesn't exceed `dynamicEfMax` even if the query limit is large enough to suggest a higher value. If the query limit is higher than `dynamicEfMax`, `dynamicEfMax` does not have any effect. In this case, dynamic `ef` value is equal to the query limit. - -To determine the length of the ANN list, Weaviate multiples the query limit by `dynamicEfFactor`. The list range is modified by `dynamicEfMin` and `dynamicEfMax`. - -Consider this GraphQL query that sets a limit of 4. - -```graphql -{ - Get { - JeopardyQuestion(limit: 4) { - answer - question - } - } -} -``` - -Imagine the collection has dynamic `ef` configured. - -```json - "vectorIndexConfig": { - "ef": -1, - "dynamicEfMin": 5 - "dynamicEfMax": 25 - "dynamicEfFactor": 10 - } -``` - -The resulting search list has these characteristics. - -- A potential length of 40 objects ( ("dynamicEfFactor": 10) * (limit: 4) ). -- A minimum length of 5 objects ("dynamicEfMin": 5). -- A maximum length of 25 objects ("dynamicEfMax": 25). -- An actual size of 5 to 25 objects. - -If you use the [`docker-compose.yml` file from Weaviate](/developers/weaviate/installation/docker-compose) to run your local instance, the `QUERY_DEFAULTS_LIMIT` environment variable sets a reasonable default query limit. To prevent out of memory errors,`QUERY_DEFAULTS_LIMIT` is significantly lower than `QUERY_MAXIMUM_RESULTS`. - -To change the default limit, edit the value for `QUERY_DEFAULTS_LIMIT` when you configure your Weaviate instance. - -### Deletions - -Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. Prior to cleanup, objects are marked as deleted, but they are still connected to the HNSW graph. During cleanup, the edges are reassigned and the objects are deleted for good. - -### Asynchronous indexing - -:::caution Experimental -Available starting in `v1.22`. This is an experimental feature. Use with caution. -::: - -Starting in Weaviate `1.22`, you can use asynchronous indexing by opting in. - -Asynchronous indexing decouples object creation from vector index updates. Objects are created faster, and the vector index updates in the background. Asynchronous indexing is especially useful for importing large amounts of data. - -While the vector index is updating, Weaviate can search a maximum of 100,000 un-indexed objects by brute force, that is, without using the vector index. This means that the search performance is slower until the vector index has been fully updated. Also, any additional new objects beyond the first 100,000 in the queue are not include in the search. - -:::tip -You might be also interested in our blog post [Vamana vs. HNSW - Exploring ANN algorithms Part 1](https://weaviate.io/blog/ann-algorithms-vamana-vs-hnsw). -::: - -## Flat index - -:::info Added in `v1.23` -::: - -The **flat index** is a simple, lightweight index that is fast to build and has a very small memory footprint. This index type is a good choice for use cases where each end user (i.e. tenant) has their own, isolated, dataset, such as in a SaaS product for example, or a database of isolated record sets. - -As the name suggests, the flat index is a single layer of disk-backed data objects and thus a very small memory footprint. The flat index is a good choice for small collections, such as for multi-tenancy use cases. - -A drawback of the flat index is that it does not scale well to large collections as it has a linear time complexity as a function of the number of data objects, unlike the `hnsw` index which has a logarithmic time complexity. - -## Dynamic index - -:::caution Experimental feature -Available starting in `v1.25`. This is an experimental feature. Use with caution. -::: - -import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; - - - -The flat index is ideal for use cases with a small object count and provides lower memory overhead and good latency. As the object count increases the HNSW index provides a more viable solution as HNSW speeds up search. The goal of the dynamic index is to shorten latencies during querying time at the cost of a larger memory footprint as you scale. - -By configuring a dynamic index, you can automatically switch from flat to HNSW indexes. This switch occurs when the object count exceeds a prespecified threshold (by default 10,000). This functionality only works with async indexing enabled. When the threshold is hit while importing, all the data piles up in the async queue, the HNSW index is constructed in the background and when ready the swap from flat to HNSW is completed. - -Currently, this is only a one-way upgrade from a flat to an HNSW index, it does not support changing back to a flat index even if the object count goes below the threshold due to deletion. - -This is particularly useful in a multi-tenant setup where building an HNSW index per tenant would introduce extra overhead. With a dynamic index, as individual tenants grow their index will switch from flat to HNSW, while smaller tenants' indexes remain flat. - -## Vector cache considerations - -For optimal search and import performance, previously imported vectors need to be in memory. A disk lookup for a vector is orders of magnitudes slower than memory lookup, so the disk cache should be used sparingly. However, Weaviate can limit the number of vectors in memory. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. - -During import set `vectorCacheMaxObjects` high enough that all vectors can be held in memory. Each import requires multiple searches. Import performance drops drastically when there isn't enough memory to hold all of the vectors in the cache. - -After import, when your workload is mostly querying, experiment with vector cache limits that are less than your total dataset size. - -Vectors that aren't currently in cache are added to the cache if there is still room. If the cache fills, Weaviate drops the whole cache. All future vectors have to be read from disk for the first time. Then, subsequent queries run against the cache until it fills again and the procedure repeats. Note that the cache can be a very valuable tool if you have a large dataset, and a large percentage of users only query a specific subset of vectors. In this case you might be able to serve the largest user group from cache while requiring disk lookups for "irregular" queries. - -## Vector indexing FAQ - -### Can I use vector indexing with vector quantization? - -Yes, you can read more about it in [vector quantization (compression)](./vector-quantization.md). - -### Which vector index is right for me? - -A simple heuristic is that for use cases such as SaaS products where each end user (i.e. tenant) has their own, isolated, dataset, the `flat` index is a good choice. For use cases with large collections, the `hnsw` index may be a better choice. - -Note that the vector index type parameter only specifies how the vectors of data objects are *indexed*. The index is used for data retrieval and similarity search. - -The `vectorizer` parameter determines how the data vectors are created (which numbers the vectors contain). `vectorizer` specifies a [module](/developers/weaviate/modules/index.md), such as `text2vec-contextionary`, that Weaviate uses to create the vectors. (You can also set to `vectorizer` to `none` if you want to import your own vectors). - -To learn more about configuring the collection, see [this how-to page](../manage-data/collections.mdx). - -### Which distance metrics can I use with vector indexing? - -All of [the distance metrics](/developers/weaviate/config-refs/distances.md), such as cosine similarity, can be used with any vector index type. - -### How to configure the vector index type in Weaviate? - -The index type can be specified per data collection via the [collection definition](/developers/weaviate/manage-data/collections.mdx#set-vector-index-type) settings, according to available [vector index settings](../config-refs/schema/vector-index.md). - -### When to skip indexing - -There are situations where it doesn't make sense to vectorize a collection. For example, if the collection consists solely of references between two other collections, or if the collection contains mostly duplicate elements. - -Importing duplicate vectors into HNSW is very expensive. The import algorithm checks early on if a candidate vector's distance is greater than the worst candidate's distance. When there are lots of duplicate vectors, this early exit condition is never met so each import or query results in an exhaustive search. - -To avoid indexing a collection, set `"skip"` to `"true"`. By default, collections are indexed. - -### What ANN algorithms exist? - -There are different ANN algorithms, you can find a nice overview of them on this website. - -### Are there indicative benchmarks for Weaviate's ANN performance? - -The [ANN benchmark page](/developers/weaviate/benchmarks/ann.md) contains a wide variety of vector search use cases and relative benchmarks. This page is ideal for finding a dataset similar to yours and learning what the most optimal settings are. - -## Further resources - -:::info Related pages -- [Concepts: Indexing](./indexing.md) -- [Concepts: Vector quantization (compression)](./vector-quantization.md) -- [Configuration: Vector index](../config-refs/schema/vector-index.md) -- [Configuration: Schema (Configure semantic indexing)](../config-refs/schema/index.md#configure-semantic-indexing) -::: - -## Questions and feedback - -import DocsFeedback from '/_includes/docs-feedback.mdx'; - - diff --git a/developers/weaviate/concepts/vector-quantization.md b/developers/weaviate/concepts/vector-quantization.md index 4e6105c131..3df051d2b3 100644 --- a/developers/weaviate/concepts/vector-quantization.md +++ b/developers/weaviate/concepts/vector-quantization.md @@ -5,7 +5,7 @@ image: og/docs/concepts.jpg # tags: ['vector compression', 'quantization'] --- -**Vector quantization** reduces the memory footprint of the [vector index](./vector-index.md) by compressing the vector embeddings, and thus reduces deployment costs and improves the speed of the vector similarity search process. +**Vector quantization** reduces the memory footprint of the [vector index](/developers/weaviate/concepts/indexing/vector-indexes) by compressing the vector embeddings, and thus reduces deployment costs and improves the speed of the vector similarity search process. Weaviate currently offers two vector quantization techniques: @@ -130,15 +130,13 @@ In some cases, rescoring also includes over-fetching, whereby additional candida ## Further resources -:::info Related pages -- [Concepts: Indexing](./indexing.md) -- [Concepts: Vector Indexing](./vector-index.md) -- [Configuration: Vector index](../config-refs/schema/vector-index.md) +- [Concepts: Indexing](/developers/weaviate/concepts/indexing) +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) - [Configuration: Schema (Configure semantic indexing)](../config-refs/schema/index.md#configure-semantic-indexing) - [How to configure: Binary quantization (compression)](../configuration/compression/bq-compression.md) - [How to configure: Product quantization (compression)](../configuration/compression/pq-compression.md) - [Weaviate Academy: 250 Vector Compression](../../academy/py/compression/index.md) -::: ## Questions and feedback diff --git a/developers/weaviate/config-refs/schema/index.md b/developers/weaviate/config-refs/schema/index.md index bcae066bf7..069ded185f 100644 --- a/developers/weaviate/config-refs/schema/index.md +++ b/developers/weaviate/config-refs/schema/index.md @@ -323,7 +323,7 @@ The `vectorIndexType` parameter controls the type of vector index that is used f The `vectorIndexConfig` parameter controls the configuration of the vector index. The available parameters depend on the `vectorIndexType` that is used. -See the [vector index configuration](./vector-index.md) page for more details. +See the [vector index configuration](/developers/weaviate/concepts/indexing/vector-indexes) page for more details. ### `shardingConfig` @@ -567,7 +567,7 @@ The `kagome_kr` tokenizer is not loaded by default to save resources. To use it, The `indexInverted` parameter has been deprecated from Weaviate `v1.19` onwards. ::: -Multiple [inverted index types](../../concepts/indexing.md#inverted-indexes) are available in Weaviate. Not all inverted index types are available for all data types. The available inverted index types are: +Multiple [inverted index types](/developers/weaviate/concepts/indexing/inverted-indexes) are available in Weaviate. Not all inverted index types are available for all data types. The available inverted index types are: import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; diff --git a/developers/weaviate/config-refs/schema/vector-index.md b/developers/weaviate/config-refs/schema/vector-index.md index c3e4c4cfad..471e7b1d38 100644 --- a/developers/weaviate/config-refs/schema/vector-index.md +++ b/developers/weaviate/config-refs/schema/vector-index.md @@ -19,16 +19,16 @@ Some HNSW parameters are mutable, but others cannot be modified after you create | :-- | :-- | :-- | :-- | :-- | | `cleanupIntervalSeconds` | integer | 300 | Yes | Cleanup frequency. This value does not normally need to be adjusted. A higher value means cleanup runs less frequently, but it does more in a single batch. A lower value means cleanup is more frequent, but it may be less efficient on each run. | | `distance` | string | `cosine` | No | Distance metric. The metric that measures the distance between two arbitrary vectors. For available distance metrics, see [supported distance metrics](/developers/weaviate/config-refs/distances.md). | -| `ef` | integer | -1 | Yes | Balance search speed and recall. `ef` is the size of the dynamic list that the HNSW uses during search. Search is more accurate when `ef` is higher, but it is also slower. `ef` values greater than 512 show diminishing improvements in recall.

Dynamic `ef`. Weaviate automatically adjusts the `ef` value and creates a dynamic `ef` list when `ef` is set to -1. For more details, see [dynamic ef](../../concepts/vector-index.md#dynamic-ef). | +| `ef` | integer | -1 | Yes | Balance search speed and recall. `ef` is the size of the dynamic list that the HNSW uses during search. Search is more accurate when `ef` is higher, but it is also slower. `ef` values greater than 512 show diminishing improvements in recall.

Dynamic `ef`. Weaviate automatically adjusts the `ef` value and creates a dynamic `ef` list when `ef` is set to -1. For more details, see [dynamic ef](/developers/weaviate/concepts/indexing/hnsw-indexes#dynamic-ef). | | `efConstruction` | integer | 128 | No | Balance index search speed and build speed. A high `efConstruction` value means you can lower your `ef` settings, but importing is slower.

`efConstruction` must be greater than 0. | | `maxConnections` | integer | 32 | No | Maximum number of connections per element. `maxConnections` is the connection limit per layer for layers above the zero layer. The zero layer can have (2 * maxConnections) connections.

`maxConnections` must be greater than 0. | -| `dynamicEfMin` | integer | 100 | Yes | *New in `v1.10.0`.*

Lower bound for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Protects against a creating search list that is too short.

This setting is only used when `ef` is -1. | -| `dynamicEfMax` | integer | 500 | Yes | *New in `v1.10.0`.*

Upper bound for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Protects against creating a search list that is too long.

If `dynamicEfMax` is higher than the limit, `dynamicEfMax` does not have any effect. In this case, `ef` is the limit.

This setting is only used when `ef` is -1. | -| `dynamicEfFactor` | integer | 8 | Yes | *New in `v1.10.0`.*

Multiplier for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Sets the potential length of the search list.

This setting is only used when `ef` is -1. | +| `dynamicEfMin` | integer | 100 | Yes | *New in `v1.10.0`.*

Lower bound for [dynamic `ef`](/developers/weaviate/concepts/indexing/hnsw-indexes#dynamic-ef). Protects against a creating search list that is too short.

This setting is only used when `ef` is -1. | +| `dynamicEfMax` | integer | 500 | Yes | *New in `v1.10.0`.*

Upper bound for [dynamic `ef`](/developers/weaviate/concepts/indexing/hnsw-indexes#dynamic-ef). Protects against creating a search list that is too long.

If `dynamicEfMax` is higher than the limit, `dynamicEfMax` does not have any effect. In this case, `ef` is the limit.

This setting is only used when `ef` is -1. | +| `dynamicEfFactor` | integer | 8 | Yes | *New in `v1.10.0`.*

Multiplier for [dynamic `ef`](/developers/weaviate/concepts/indexing/hnsw-indexes#dynamic-ef). Sets the potential length of the search list.

This setting is only used when `ef` is -1. | | `flatSearchCutoff` | integer | 40000 | Yes | Optional. Threshold for the [flat-search cutoff](/developers/weaviate/concepts/prefiltering.md#flat-search-cutoff). To force a vector index search, set `"flatSearchCutoff": 0`. | -| `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](../../concepts/vector-index.md#when-to-skip-indexing). | -| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/vector-index.md#vector-cache-considerations). | -| `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/concepts/vector-index.md#hnsw-with-product-quantizationpq) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | +| `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](/developers/weaviate/concepts/indexing/hnsw-indexes#when-to-skip-indexing). | +| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/hnsw-indexes). | +| `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/concepts/indexing/hnsw-indexes) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | ### Database parameters for HNSW @@ -140,7 +140,7 @@ Flat indexes are recommended for use cases where the number of objects per index | Parameter | Type | Default | Changeable | Details | | :-- | :-- | :-- | :-- | :-- | -| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/vector-indexes). | | `bq` | object | -- | No | Enable and configure [binary quantization (BQ)](../../concepts/vector-quantization.md#binary-quantization) compression.

For BQ configuration details, see [BQ configuration parameters](#bq-configuration-parameters). | ### BQ configuration parameters @@ -194,10 +194,11 @@ Use these parameters to configure the index type and their properties. They can
How to select the index type -Generally, the `hnsw` index type is recommended for most use cases. The `flat` index type is recommended for use cases where the data the number of objects per index is low, such as in multi-tenancy cases. You can also opt for the `dynamic` index which will initially configure a `flat` index and once the object count exceeds a specified threshold it will automatically convert to an `hnsw` index. +The `hnsw` index type is recommended for most use cases. -See [this section](../../concepts/vector-index.md#which-vector-index-is-right-for-me) for more information about the different index types and how to choose between them. +The `flat` index type is recommended for use cases where the number of objects per index is low, such as in multi-tenancy cases. +The `dynamic` index is useful if your collection size starts small and grows. A dynamic index starts as a flat index and converts to an hnsw index when the object count exceeds a specified threshold.
If faster import speeds are desired, [asynchronous indexing](#asynchronous-indexing) allows de-coupling of indexing from object creation. @@ -208,9 +209,9 @@ If faster import speeds are desired, [asynchronous indexing](#asynchronous-index Available starting in `v1.22`. This is an experimental feature. Use with caution. ::: -Starting in Weaviate `1.22`, you can use asynchronous indexing by opting in. +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; -To enable asynchronous indexing, set the `ASYNC_INDEXING` environment variable to `true` in your Weaviate configuration (the `docker-compose.yml` file if you use Docker Compose). This setting enables asynchronous indexing for all collections. +
Example Docker Compose configuration @@ -302,8 +303,8 @@ import MultiVectorSupport from '/_includes/multi-vector-support.mdx'; ## Related pages -- [Concepts: Indexing](../../concepts/indexing.md) -- [Concepts: Vector Indexing](../../concepts/vector-index.md) +- [Concepts: Indexing](/developers/weaviate/concepts/indexing) +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) ## Questions and feedback diff --git a/developers/weaviate/configuration/backups.md b/developers/weaviate/configuration/backups.md index 8b3ff79108..f5303653a6 100644 --- a/developers/weaviate/configuration/backups.md +++ b/developers/weaviate/configuration/backups.md @@ -1,6 +1,6 @@ --- title: Backups -sidebar_position: 12 +sidebar_position: 20 image: og/docs/configuration.jpg # tags: ['configuration', 'backups'] --- diff --git a/developers/weaviate/configuration/compression/_category_.json b/developers/weaviate/configuration/compression/_category_.json new file mode 100644 index 0000000000..8f1f43419e --- /dev/null +++ b/developers/weaviate/configuration/compression/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Compression", + "position": 1 +} \ No newline at end of file diff --git a/developers/weaviate/configuration/compression/bq-compression.md b/developers/weaviate/configuration/compression/bq-compression.md index 21c0d6a01b..9605e8bd8b 100644 --- a/developers/weaviate/configuration/compression/bq-compression.md +++ b/developers/weaviate/configuration/compression/bq-compression.md @@ -17,7 +17,7 @@ import GoCode from '!!raw-loader!/_includes/code/howto/configure.bq-compression. import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/bq-compression.java'; :::info Added in `v1.23` -BQ is available for the [`flat` index](/developers/weaviate/concepts/vector-index.md#flat-index) type from `v1.23` onwards and for the [`hnsw` index](/developers/weaviate/concepts/vector-index.md#hnsw-index) type from `v1.24`. +BQ is available for the [`flat` index](/developers/weaviate/concepts/indexing/flat-indexes) type from `v1.23` onwards and for the [`hnsw` index](/developers/weaviate/concepts/indexing/hnsw-indexes) type from `v1.24`. ::: Binary quantization (BQ) is a vector compression technique that can reduce the size of a vector. @@ -104,7 +104,7 @@ The following parameters are available for BQ compression, under `vectorIndexCon | `bq` : `enabled` | boolean | `false` | Enable BQ. Weaviate uses binary quantization (BQ) compression when `true`.

The Python client v4 does not use the `enabled` parameter. To enable BQ with the v4 client, set a `quantizer` in the collection definition. | | `bq` : `rescoreLimit` | integer | -1 | The minimum number of candidates to fetch before rescoring. | | `bq` : `cache` | boolean | `false` | Whether to use the vector cache. | -| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/vector-indexes). | For example: @@ -173,7 +173,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Tutorial: Schema](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/configuration/compression/index.md b/developers/weaviate/configuration/compression/index.md index c8e20a68e8..6838404a0a 100644 --- a/developers/weaviate/configuration/compression/index.md +++ b/developers/weaviate/configuration/compression/index.md @@ -5,10 +5,10 @@ image: og/docs/configuration.jpg # tags: ['configuration', 'compression', 'pq'] --- -Uncompressed vectors can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. +Uncompressed vectors can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. To balance resource costs and system performance, consider one of these options: -- [Binary Quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) -- [Product Quantization (PQ)](/developers/weaviate/configuration/compression/pq-compression) -- [Scalar Quantization (SQ)](/developers/weaviate/configuration/compression/sq-compression) +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/compression/pq-compression.md b/developers/weaviate/configuration/compression/pq-compression.md index fe361101e2..e21ad7b033 100644 --- a/developers/weaviate/configuration/compression/pq-compression.md +++ b/developers/weaviate/configuration/compression/pq-compression.md @@ -375,7 +375,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Guide: Schemas and collection definitions](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/configuration/compression/sq-compression.md b/developers/weaviate/configuration/compression/sq-compression.md index 5c4d4d3626..4312ced9e9 100644 --- a/developers/weaviate/configuration/compression/sq-compression.md +++ b/developers/weaviate/configuration/compression/sq-compression.md @@ -60,7 +60,7 @@ To tune SQ, set these `vectorIndexConfig` parameters. | `sq`: `rescoreLimit` | integer | -1 | The minimum number of candidates to fetch before rescoring. | | `sq`: `trainingLimit` | integer | 100000 | The size of the training set to determine scalar bucket boundaries. | | `sq`: `cache` | boolean | `false` | Use the vector cache when true. | -| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/vector-indexes). | @@ -91,7 +91,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Tutorial: Schema](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/configuration/indexing-vector/_category_.json b/developers/weaviate/configuration/indexing-vector/_category_.json new file mode 100644 index 0000000000..f10274ad88 --- /dev/null +++ b/developers/weaviate/configuration/indexing-vector/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Vector indexes", + "position": 3 +} \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md b/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md new file mode 100644 index 0000000000..a3440d5c12 --- /dev/null +++ b/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md @@ -0,0 +1,122 @@ +--- +title: Dynamic indexes +sidebar_position: 30 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v2.ts'; + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. + +import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; + + + +## Enable dynamic indexes + +Enable a dynamic index that uses default parameters for the dynamic, HNSW, and flat indexes: + + + + + + + + + + + + + + + + +## Configure dynamic indexes + +Configure the dynamic, HNSW, and flat indexes: + + + + + + + + + + + + + + + + +See also: + +- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) + +## Asynchronous indexing + +import AsyncIndexing from '/_includes/indexes/async-indexing.mdx'; + + + +## Related pages + +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing-vector/flat-indexes.md b/developers/weaviate/configuration/indexing-vector/flat-indexes.md new file mode 100644 index 0000000000..3d5afeaf2f --- /dev/null +++ b/developers/weaviate/configuration/indexing-vector/flat-indexes.md @@ -0,0 +1,165 @@ +--- +title: Flat indexes +sidebar_position: 20 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + +import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; + + + +## Enable a flat index + +Enable a flat index for a collection: + + + + + + + + + + + + + + + + +## Configure a flat index + +Configure a flat index for a collection: + + + + + + + + + + + + + + + + +See also: + +- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) + +## Multiple named vectors + +import MultiNameVec from '/_includes/indexes/multiple-named-vectors.mdx'; + + + +## Compression + +import IndexCompression from '/_includes/indexes/index-compression.mdx'; + + + +### Enable compression + +Enable compression on a flat index: + + + + + + + + + + + + + + + + +## Related pages + +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing-vector/hnsw-indexes.md b/developers/weaviate/configuration/indexing-vector/hnsw-indexes.md new file mode 100644 index 0000000000..b94057a13b --- /dev/null +++ b/developers/weaviate/configuration/indexing-vector/hnsw-indexes.md @@ -0,0 +1,166 @@ +--- +title: HNSW indexes +sidebar_position: 13 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; + + + +## Enable an HNSW index + +HNSW is the default vector index. You do not have to explicitly enable HNSW if you want to use the default vector index type. + + + + + + + + + + + + + + + + +## Configure an HNSW index + +Configure an HNSW index for a collection: + + + + + + + + + + + + + + + + +See also: + + - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) + +## Multiple named vectors + +import MultiNameVec from '/_includes/indexes/multiple-named-vectors.mdx'; + + + +## Compression + +import IndexCompression from '/_includes/indexes/index-compression.mdx'; + + + +### Enable compression + +Enable compression on an HNSW index: + + + + + + + + + + + + + + + + +## Related pages + +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing-vector/index.md b/developers/weaviate/configuration/indexing-vector/index.md new file mode 100644 index 0000000000..3e36104f5e --- /dev/null +++ b/developers/weaviate/configuration/indexing-vector/index.md @@ -0,0 +1,76 @@ +--- +title: Vector index overview +sidebar_position: 10 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; + + + +HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. + +When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. + +## HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +[Configure an HNSW index](/developers/weaviate/configuration/indexing-vector/hnsw-indexes.md). + +See also: + + - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) + +## Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + +[Configure a flat index](/developers/weaviate/configuration/indexing-vector/flat-indexes.md). + +See also: + +- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) + +## Dynamic indexes + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. + +[Configure a dynamic index](/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md). + +See also: + +- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) + +## Compression + +import IndexCompression from '/_includes/indexes/index-compression.mdx'; + + + +## Asynchronous indexing + +import AsyncIndexing from '/_includes/indexes/async-indexing.mdx'; + + + +## Related pages + +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) +- [Configure inverted indexes](/developers/weaviate/configuration/inverted-indexes) + + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/inverted-indexes.md b/developers/weaviate/configuration/inverted-indexes.md new file mode 100644 index 0000000000..d7f257e2af --- /dev/null +++ b/developers/weaviate/configuration/inverted-indexes.md @@ -0,0 +1,247 @@ +--- +title: Inverted indexes +sidebar_position: 15 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v2.ts'; + +Weaviate uses [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make textual and numeric searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data. + +These indexes are normally configured on a property level: + +- [indexSearchable](#indexSearchable) +- [indexFilterable](#indexfilterable) +- [indexRangeFilters](#indexrangefilters) + +To tune inverted indexes at the collection level, adjust these configuration settings: + +- [BM25 search algorithm](#bm25) +- [indexNullState](#collection-level-properties) +- [indexPropertyLength](#collection-level-properties) +- [indexTimestamps](#collection-level-properties) + +## indexSearchable + +The `indexSearchable` index improves property search times. This index is enabled by default. [Keyword search](/developers/weaviate/search/bm25) and [hybrid search](/developers/weaviate/search/hybrid) use this index. + +If you don't anticipate searching on a property field, you can disable this index to save disk space and import time. The property is still searchable. The search is just less efficient. + + + + + + + + + + + + + + + + +## indexFilterable + +The `indexFilterable` index improves [filtering](/developers/weaviate/search/filters). This index is enabled by default. + +If you don't anticipate searching on a property field, disable this index to save disk space and import time. The property is still filterable. + +Set these indexes on the property level. + + + + + + + + + + + + + + + + +## indexRangeFilters + +The `indexRangeFilters` is a range-based index for filtering by [numerical ranges](/developers/weaviate/release-notes/release_1_26#improved-range-queries). This index is not enabled by default. + +Set these indexes on the property level. + + + + + + + + + + + + + + + + +## bm25 + +The [`bm25`](/developers/weaviate/config-refs/schema#bm25) ranking algorithm is configured on the collection level. The BM25 algorithm does not specify values for `b` or `k1`. Use these ["free parameters"](https://en.wikipedia.org/wiki/Free_parameter) to tailor the ranking algorithm for your data. + +To adjust for document length, modify `b`. Values range from 0 to 1. + +To adjust for word frequency within a document, modify `k1`. Values are usually in the range from 0 to 3. There isn't an upper limit. + + + + + + + + + + + + + + + + +## Collection level properties + +These properties are configured on the collection level. + + + + + + + + + + + + + + + + +## Related pages + +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) +- [Configure vector indexes](/developers/weaviate/configuration/indexing-vector) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/modules.md b/developers/weaviate/configuration/modules.md index bb403fa34c..a8a7c0200b 100644 --- a/developers/weaviate/configuration/modules.md +++ b/developers/weaviate/configuration/modules.md @@ -1,6 +1,6 @@ --- title: Modules -sidebar_position: 11 +sidebar_position: 17 image: og/docs/configuration.jpg # tags: ['configuration', 'modules'] --- diff --git a/developers/weaviate/configuration/tenant-offloading.md b/developers/weaviate/configuration/tenant-offloading.md index c79b740e6c..dd4512452c 100644 --- a/developers/weaviate/configuration/tenant-offloading.md +++ b/developers/weaviate/configuration/tenant-offloading.md @@ -1,6 +1,6 @@ --- title: Tenant Offloading -sidebar_position: 5 +sidebar_position: 25 image: og/docs/configuration.jpg --- diff --git a/developers/weaviate/introduction.md b/developers/weaviate/introduction.md index 5935218f42..5a141d6aee 100644 --- a/developers/weaviate/introduction.md +++ b/developers/weaviate/introduction.md @@ -128,7 +128,7 @@ Weaviate is an open source vector database that stores both objects and vectors. **Weaviate in a nutshell**: * Weaviate is an open source [vector database](https://weaviate.io/blog/what-is-a-vector-database). -* Weaviate allows you to store and retrieve data objects based on their semantic properties by indexing them with [vectors](./concepts/vector-index.md). +* Weaviate allows you to store and retrieve data objects based on their semantic properties by indexing them with [vectors](/developers/weaviate/concepts/indexing/vector-indexes). * Weaviate can be used stand-alone (aka _bring your vectors_) or with a variety of [modules](./modules/index.md) that can do the vectorization for you and extend the core capabilities. * Weaviate has a [GraphQL-API](./api/graphql/index.md) to access your data easily. * Weaviate is fast (check our [open source benchmarks](./benchmarks/index.md)). @@ -186,7 +186,7 @@ Within Weaviate, all individual data objects are based on a class property struc You can add data to Weaviate through the [RESTful API](/developers/weaviate/api/rest) end-points and retrieve data through the [GraphQL interface](./api/graphql/index.md). -Weaviate's [vector indexing mechanism is modular](./concepts/vector-index.md), and the current available plugin is the Hierarchical Navigable Small World (HNSW) multilayered graph. +Weaviate's [vector indexing mechanism is modular](/developers/weaviate/concepts/indexing/vector-indexes), and the current available plugin is the Hierarchical Navigable Small World (HNSW) multilayered graph. ## What are Weaviate modules? diff --git a/developers/weaviate/manage-data/collections.mdx b/developers/weaviate/manage-data/collections.mdx index 0b171dd6d3..faeda92722 100644 --- a/developers/weaviate/manage-data/collections.mdx +++ b/developers/weaviate/manage-data/collections.mdx @@ -382,7 +382,7 @@ The vector index type can be set for each collection at creation time, between `
Additional information -- Read more about index types & compression in [Concepts: Vector index](../concepts/vector-index.md). +- Read more about index types & compression in [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes).
@@ -440,7 +440,7 @@ Various vector index parameters are configurable at collection creation time, in
Additional information -- Read more about index types & compression in [Concepts: Vector index](../concepts/vector-index.md). +- Read more about index types & compression in [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes).
diff --git a/developers/weaviate/more-resources/faq.md b/developers/weaviate/more-resources/faq.md index 75074259aa..07b4389b5e 100644 --- a/developers/weaviate/more-resources/faq.md +++ b/developers/weaviate/more-resources/faq.md @@ -330,9 +330,9 @@ If you need a higher search quality for a given limit you can consider the follo > More information: > > - [Weaviate, an ANN Database with CRUD support – DB-Engines.com](https://db-engines.com/en/blog_post/87) ⬅️ best resource on the topic -> - [Weaviate's HNSW implementation in the docs](/developers/weaviate/concepts/vector-index.md#hnsw) +> - [Weaviate's HNSW implementation in the docs](/developers/weaviate/concepts/indexing/hnsw-indexes) > -> _Note I: HNSW is just one implementation in Weaviate, but Weaviate can support multiple indexing algoritmns as outlined [here](/developers/weaviate/concepts/vector-index.md)_ +> _Note I: HNSW is just one implementation in Weaviate, but Weaviate can support multiple indexing algoritmns as outlined [here](/developers/weaviate/concepts/indexing/vector-indexes)_
diff --git a/developers/weaviate/more-resources/migration/index.md b/developers/weaviate/more-resources/migration/index.md index 6ad50decab..548c74118b 100644 --- a/developers/weaviate/more-resources/migration/index.md +++ b/developers/weaviate/more-resources/migration/index.md @@ -624,7 +624,7 @@ With the modularization, it becomes possible to vectorize non-text objects. Sear * The `vectorizer` indicates which module (if any) are responsible for vectorization. * The `moduleConfig` allows configuration per module (by name). * See [here](#text2vec-contextionary) for Contextionary specific property configuration. - * The `vectorIndexType` allows the choosing the vector index (defaults to [HNSW](/developers/weaviate/concepts/vector-index.md#hnsw)) + * The `vectorIndexType` allows the choosing the vector index (defaults to [HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes)) * The `vectorIndexConfig` is an arbitrary object passed to the index for config (defaults can be found [here](/developers/weaviate/config-refs/schema/vector-index.md#how-to-configure-hnsw) ) All changes are in this example: diff --git a/developers/weaviate/more-resources/performance.md b/developers/weaviate/more-resources/performance.md index ea2e17fad2..502929bc44 100644 --- a/developers/weaviate/more-resources/performance.md +++ b/developers/weaviate/more-resources/performance.md @@ -28,7 +28,7 @@ The inverted index currently does not do any weighing (e.g. tf-idf) for sorting, ## Vector index Everything that has a vector, thus every data object in Weaviate, is also indexed in the vector index. Weaviate currently supports [HNSW](https://arxiv.org/abs/1603.09320) and flat vector indexes. -See [Concepts: vector index](../concepts/vector-index.md) for more information about the vector index. +See [Concepts: vector index](/developers/weaviate/concepts/indexing/vector-indexes) for more information about the vector index. ## Costs of queries and operations diff --git a/developers/weaviate/starter-guides/managing-resources/compression.mdx b/developers/weaviate/starter-guides/managing-resources/compression.mdx index ea11df9ce9..4b40adf30c 100644 --- a/developers/weaviate/starter-guides/managing-resources/compression.mdx +++ b/developers/weaviate/starter-guides/managing-resources/compression.mdx @@ -11,11 +11,11 @@ Use compression to lower system requirements and save on infrastructure costs. Weaviate stores objects and vector representations of those objects (vectors). Vectors can be very large. Vector dimensions are stored as 32 bit floats. A single vector with 1536 dimensions uses about 6 KB of storage. When collections have millions of objects, the resulting size can lead to significant costs, especially where an in-memory vector index is used. -Weaviate creates indexes to search the vector space for your collection. By default, the vector index is an [Hierarchical Navigable Small World (HNSW)](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) index which includes the vector as well as a graph structure. HNSW indexes allow fast vector searches while maintaining excellent recall, but they can be expensive to use as they are stored in memory. +Weaviate creates indexes to search the vector space for your collection. By default, the vector index is an [Hierarchical Navigable Small World (HNSW)](/developers/weaviate/concepts/indexing/hnsw-indexes) index which includes the vector as well as a graph structure. HNSW indexes allow fast vector searches while maintaining excellent recall, but they can be expensive to use as they are stored in memory. In many cases, you can use compression or a different index type to change the way Weaviate stores and searches your data, and still maintain high levels of recall. Updating the default settings can result in significant cost savings and performance improvements. -This page discusses compression algorithms. For more on indexes, see [Vector indexing](/developers/weaviate/concepts/vector-index). +This page discusses compression algorithms. For more on indexes, see [Vector indexing](/developers/weaviate/concepts/indexing/vector-indexes). ## Compression algorithms diff --git a/developers/weaviate/starter-guides/managing-resources/index.md b/developers/weaviate/starter-guides/managing-resources/index.md index 86853eecb5..46f3efbbcb 100644 --- a/developers/weaviate/starter-guides/managing-resources/index.md +++ b/developers/weaviate/starter-guides/managing-resources/index.md @@ -168,7 +168,7 @@ Consider a strategy of deactivating tenants that are not frequently accessed, an - [Starter guide: Compression](./compression.mdx) - [Starter guide: Indexing](./indexing.mdx) - [Starter guide: Tenant states](./tenant-states.mdx) -- [Concepts: Vector Index](../../concepts/vector-index.md) +- [Concepts: Vector Index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector Quantization](../../concepts/vector-quantization.md) - [Concepts: Multi-Tenancy](../../concepts/data.md#multi-tenancy) - [How-to: Set the vector index type](../../manage-data/collections.mdx#set-vector-index-type) diff --git a/developers/weaviate/starter-guides/managing-resources/indexing.mdx b/developers/weaviate/starter-guides/managing-resources/indexing.mdx index 049cec4eb5..47fdda309f 100644 --- a/developers/weaviate/starter-guides/managing-resources/indexing.mdx +++ b/developers/weaviate/starter-guides/managing-resources/indexing.mdx @@ -30,15 +30,15 @@ Vector indexes can use *hot* or *warm* resources, depending on the index type. I ## Vector indexes -Weaviate offers three types of vector indexes, [Hierarchical Navigable Small World (HNSW)](#hnsw-indexes), [flat](#flat-indexes) and [dynamic](#dynamic-indexes). +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; -- HNSW indexes enable fast, scalable vector searching that works well even with very large data sets. -- Flat indexes are memory-efficient indexes that work best with small data sets. -- Dynamic indexes provide a best-of-both-worlds approach by switching from flat to HNSW indexes when a [collection](../../concepts/data.md#collections) (or [tenant](../../concepts/data.md#multi-tenancy)) reaches a threshold size. + #### HNSW indexes -[HNSW](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + HNSW indexes achieve this by building a multi-layered graph of objects, allowing for fast, approximate nearest neighbor searches. @@ -52,7 +52,9 @@ import CompressionAlgorithms from '/_includes/starter-guides/compression-types.m #### Flat indexes -[Flat indexes](/developers/weaviate/concepts/vector-index#flat-index) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Vector search speeds with flat indexes scale linearly with the number of objects. +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + As a result, flat indexes are best suited for cases where the number of objects is low and will not grow significantly. @@ -63,17 +65,21 @@ As a result, flat indexes are best suited for cases where the number of objects :::info Added in `v1.25` ::: -import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; - - -A [dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offers a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index upon reaching a threshold size. + This can be particularly useful in multi-tenant configurations, where different tenants may have different numbers of objects. With a dynamic index, you can avoid the overhead of an HNSW index when it's not needed. -The threshold size is 10,000 objects by default. You can configure the threshold size when you create the dynamic index. +The `dynamic` vector index type requires asynchronous indexing. To enable asynchronous indexing, set the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) to `true`. + +Configure the settings for the flat and HNSW indexes when you create the collection definition. The dynamic index uses the flat index configuration when the collection is small. It converts the flat index to an HNSW index when the collection size reaches the dynamic index threshold. -This table shows how a dynamic index changes as the number of objects in a collection grows. The assumed set up is a dynamic index with: +The threshold size is 10,000 objects by default. Configure the threshold size when you create the dynamic index. + +##### Dynamic index behavior + +This table shows how a dynamic index changes as the number of objects in a collection grows. The table assumes a dynamic index configure like this: - A threshold of 10,000 objects. - Flat index + BQ configuration. @@ -86,10 +92,6 @@ This table shows how a dynamic index changes as the number of objects in a colle | 100,000 | HNSW | Training | The collection object count == PQ/SQ training threshold. | | 100,001 | HNSW | PQ/SQ | PQ/SQ is active. | -:::info Dynamic index requires flat and HNSW index settings -A dynamic index requires its flat and HNSW index settings at creation time. The dynamic index will use the flat index settings initially, then automatically switch to the HNSW index with provided settings when the threshold is reached. -::: - ### Asynchronous vector indexing :::info Added in `v1.22` @@ -124,7 +126,9 @@ For details, see [set inverted index parameters](/developers/weaviate/manage-dat Property level configuration is more limited. Individual indexes can be turned on or off at the property level, and keyword tokenization options can be set. -`indexSearchable` determines whether a property is indexed for keyword searches. `indexFilterable` determines whether a property is indexed to speed up match-based filtering. `indexRangeFilters` determines whether a property is indexed for numerical range filters. +- `indexSearchable` indexes are used with keyword searches +- `indexFilterable` indexes are used for filtering +- `indexRangeFilters` indexes are used for numerical range filters For more on filters, see [Filtering](/developers/weaviate/concepts/prefiltering). @@ -141,8 +145,8 @@ To configure indexing, follow the steps on these pages: For more documentation details, see: -- [Vector indexing](/developers/weaviate/concepts/vector-index) -- [Inverted indexes](/developers/weaviate/concepts/indexing) +- [Vector indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Inverted indexes](/developers/weaviate/concepts/indexing/inverted-indexes) ### Weaviate academy diff --git a/developers/weaviate/starter-guides/schema.md b/developers/weaviate/starter-guides/schema.md index dcb8c03593..93a3f96275 100644 --- a/developers/weaviate/starter-guides/schema.md +++ b/developers/weaviate/starter-guides/schema.md @@ -207,9 +207,9 @@ import SchemaWithMT from '/_includes/code/tutorial.schema.multi-tenancy.mdx'; ### Index settings -Weaviate uses two types of indexes: [vector indexes](../concepts/vector-index.md) and [inverted indexes](../concepts/indexing.md#inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. +Weaviate uses two types of indexes: [vector indexes](/developers/weaviate/concepts/indexing/vector-indexes) and [inverted indexes](/developers/weaviate/concepts/indexing/inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. -The default vector index type is [HNSW](../concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index). The other options are [flat](../concepts/vector-index.md#flat-index), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](../concepts/vector-index.md#dynamic-index), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. +The default vector index type is [HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes). The other options are [flat](/developers/weaviate/concepts/indexing/flat-indexes), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](/developers/weaviate/concepts/indexing/dynamic-indexes), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. import SchemaWithIndexSettings from '/_includes/code/tutorial.schema.index-settings.mdx'; diff --git a/site.redirects.js b/site.redirects.js index 61a530528c..6c5745b48d 100644 --- a/site.redirects.js +++ b/site.redirects.js @@ -19,7 +19,7 @@ const siteRedirects = { from: '/developers/weaviate/current/core-knowledge/basics', }, { - to: '/developers/weaviate/concepts/vector-index', + to: '/developers/weaviate/concepts/indexing/vector-indexes', from: [ '/developers/weaviate/current/vector-index-plugins', '/developers/weaviate/current/vector-index-plugins/hnsw', @@ -219,7 +219,7 @@ const siteRedirects = { // Remove BPR page { - to: '/developers/weaviate/concepts/vector-index', + to: '/developers/weaviate/concepts/indexing/vector-indexes', from: '/developers/weaviate/concepts/binary-passage-retrieval', }, diff --git a/static/og/docs/indexes.jpg b/static/og/docs/indexes.jpg new file mode 100644 index 0000000000..ff6608ebe7 Binary files /dev/null and b/static/og/docs/indexes.jpg differ