DR-629 index refactor

_includes/code/howto/indexes/indexes-dynamic-v2.ts

@@ -0,0 +1,94 @@
+// TODO: Configure as part of the test harness
+// TODO: Needs tests
+
+// Imports
+import weaviate, { WeaviateClient } from 'weaviate-ts-client';
+
+// Create client connection
+function getClient(){
+  const client: WeaviateClient = weaviate.client({
+  scheme: 'http',
+  host: 'localhost:8080',
+})
+
+  return client;
+}
+
+// Delete pre-existing collections
+async function deleteClass(client: WeaviateClient, className: string){
+ if (await client.schema.exists(className)) {
+   await client.schema.classDeleter().withClassName(className).do();
+   }
+}
+
+////////////////////
+// ENABLE DYNAMIC //
+////////////////////
+
+// START EnableDynamic
+async function createDynamicCollection(client: WeaviateClient, className: string){
+ const setIndexType = {
+   class: className,
+   // Add property definitions
+   vectorizer: 'text2vec-openai',
+   vectorIndexType: 'dynamic',
+ };
+
+ // Add the class to the schema
+ await client.schema.classCreator().withClass(setIndexType).do();
+}
+// END EnableDynamic
+
+////////////////////
+// CONFIGURE DYNAMIC //
+////////////////////
+
+// START ConfigDynamic
+async function configureDynamicCollection(client: WeaviateClient, className: string){
+ const setIndexType = {
+   class: className,
+   // Add property definitions
+   vectorizer: 'text2vec-openai',
+   vectorIndexType: 'dynamic',
+   vectorIndexConfig: {
+    distance: 'cosine',
+    vector_cache_max_objects: 100000,
+    bq: { enabled: true, },
+  },
+  vectorIndexConfigDynamic: {
+   distance: 'cosine',
+   ef_construction: '256',  // Dynamic list size during construction
+   max_connections: '128',  // Maximum number of connections per node
+   ef: '-1',  // Dynamic list size during search; -1 enables dynamic Ef
+   dynamic_ef_factor: '15',  // Multiplier for dynamic Ef
+   dynamic_ef_min: '200',  // Minimum threshold for dynamic Ef
+   dynamic_ef_max: '1000',  // Maximum threshold for dynamic Ef
+   quantizer: 'Configure.VectorIndex.Quantizer.pq()',  // Quantizer configuration
+  },
+ };
+
+ // Add the class to the schema
+ await client.schema.classCreator().withClass(setIndexType).do();
+}
+// END ConfigDynamic
+
+/////////////////////////////
+/// AVOID TOP LEVEL AWAIT ///
+/////////////////////////////
+
+// Main
+async function main(){
+  const className = "ConfigCollection";
+
+  const client = await getClient();
+
+  // Run enable dynamic collection code
+  await deleteClass(client, className)
+  createDynamicCollection(client, className);
+
+  // // Run configure dynamic collection code
+  // await deleteClass(client, className)
+  // configureDynamicCollection(client, className);
+}
+
+main()

_includes/code/howto/indexes/indexes-dynamic-v3.py

@@ -0,0 +1,101 @@
+# TODO: Configure as part of the test harness
+
+class_name = "ConfigCollection"
+
+########################
+### CLIENT CONNECTION ##
+########################
+
+import os
+import weaviate
+
+client = weaviate.Client(
+    url="http://localhost:8080",
+    additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")},
+)
+
+######################
+### ENABLE DYNAMIC ###
+######################
+
+# Delete data from prior runs
+if client.schema.exists(class_name):
+    client.schema.delete_class(class_name)
+
+# START EnableDynamic
+class_obj = {
+    "class": class_name,
+    "vectorIndexType": "dynamic",
+}
+
+client.schema.create_class(class_obj)
+# END EnableDynamic
+
+class_response = client.schema.get()
+schema_response = client.schema.get(class_name)
+
+classes = []
+for c in class_response["classes"]:
+    classes.append(c["class"])
+assert class_name in classes, "Class missing"
+
+correct_index = False
+if (schema_response["class"] == class_name) and (
+    schema_response["vectorIndexType"] == "dynamic"
+):
+    correct_index = True
+assert correct_index, "Wrong index type"
+
+#########################
+### CONFIGURE DYNAMIC ###
+#########################
+
+# Delete data from prior runs
+if client.schema.exists(class_name):
+    client.schema.delete_class(class_name)
+
+# START ConfigDynamic
+class_obj = {
+    "description": "Dynamic configuration example",
+    "class": class_name,
+    "vectorIndexType": "dynamic",
+    "vectorIndexConfig": {
+        "hnsw": {
+              "cleanupIntervalSeconds": 300,
+              "distanceMetric": "cosine",
+              "dynamicEfMin": 100,
+              "dynamicEfMax": 600,
+              "dynamicEfFactor": 8,
+              "ef": -1,
+              "efConstruction": 128,
+              "flatSearchCutoff": 40000,
+              "maxConnections": 32,
+              "skip": False,
+              "vectorCacheMaxObjects": 1000000000000,
+              "pq": { "enabled": True, },
+        },
+        "flat": {
+            "distanceMetric": "cosine",
+            "vector_cache_max_objects": 100000,
+            "bq": { "enabled": True, },
+        },
+    }
+}
+
+client.schema.create_class(class_obj)
+# END ConfigDynamic
+
+class_response = client.schema.get()
+schema_response = client.schema.get(class_name)
+
+classes = []
+for c in class_response["classes"]:
+    classes.append(c["class"])
+assert class_name in classes, "Class missing"
+
+correct_index = False
+if (schema_response["class"] == class_name) and (
+    schema_response["vectorIndexType"] == "dynamic"
+):
+    correct_index = True
+assert correct_index, "Wrong index type"

_includes/code/howto/indexes/indexes-dynamic-v3.ts

@@ -0,0 +1,94 @@
+// TODO: Configure as part of the test harness
+// TODO: Needs tests
+
+// Imports
+import weaviate, { WeaviateClient,vectorizer, configure } from 'weaviate-client';
+
+// Delete pre-existing collections
+async function deleteCollection(client: WeaviateClient, collectionName: string){
+  if(await client.collections.exists(collectionName)){
+    await client.collections.delete(collectionName)
+   }
+}
+
+// Create client connection
+async function getClient(){
+  const client: WeaviateClient = await weaviate.connectToLocal();
+  return client;
+}
+
+////////////////////
+// ENABLE DYNAMIC //
+////////////////////
+
+// START EnableDynamic
+// Add this import line
+// import { vectorizer, dataType, configure } from 'weaviate-client';
+
+async function createDynamicCollection(client: WeaviateClient, collectionName: string){
+ await client.collections.create({
+   name: collectionName,
+   vectorizers: vectorizer.text2VecOpenAI({
+     vectorIndexConfig: configure.vectorIndex.dynamic(),
+       }),
+     })
+}
+// END EnableDynamic
+
+
+///////////////////////
+// CONFIGURE DYNAMIC //
+///////////////////////
+
+// START ConfigDynamic
+// Add this import line
+// import { vectorizer, dataType, configure } from 'weaviate-client';
+
+async function configureDynamicCollection(client: WeaviateClient, collectionName: string){
+ await client.collections.create({
+   name: collectionName,
+   vectorizers: vectorizer.text2VecOpenAI({
+     vectorIndexConfig: configure.vectorIndex.dynamic({
+      distanceMetric: 'cosine',
+      hnsw: {
+        distanceMetric: 'cosine',
+        efConstruction: 256,         // Dynamic list size during construction
+        maxConnections: 128,         // Maximum number of connections per node
+        ef: -1,                      // Dynamic list size during search; -1 enables dynamic Ef
+        dynamicEfFactor: 15,         // Multiplier for dynamic Ef
+        dynamicEfMin: 200,           // Minimum threshold for dynamic Ef
+        dynamicEfMax: 1000,          // Maximum threshold for dynamic Ef
+        quantizer: configure.vectorIndex.quantizer.pq() // Compression
+      },
+      flat: {
+       distanceMetric: 'cosine',
+       vectorCacheMaxObjects: 1000000,
+       quantizer: configure.vectorIndex.quantizer.bq(),
+       },
+     }),
+       }),
+     })
+}
+// END ConfigDynamic
+
+/////////////////////////////
+/// AVOID TOP LEVEL AWAIT ///
+/////////////////////////////
+
+// Main
+async function main(){
+  const collectionName = "ConfigCollection";
+
+  const client = await getClient();
+
+  // // Run enable dynamic collection code
+  // await deleteCollection(client, collectionName)
+  // await createDynamicCollection(client, collectionName);
+
+  // Run configure dynamic collection code
+  await deleteCollection(client, collectionName)
+  await configureDynamicCollection(client, collectionName);
+
+}
+
+main()

_includes/code/howto/indexes/indexes-dynamic-v4.py

@@ -0,0 +1,92 @@
+# TODO: Configure as part of the test harness
+
+collection_name = "ConfigCollection"
+
+########################
+### CLIENT CONNECTION ##
+########################
+
+import os
+import weaviate
+
+cohere_api_key = os.environ["COHERE_API_KEY"]
+client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key})
+
+######################
+### ENABLE DYNAMIC ###
+######################
+
+# Delete data from prior runs
+if client.collections.exists(collection_name):
+    client.collections.delete(collection_name)
+
+# START EnableDynamic
+from weaviate.classes.config import Configure, Property, DataType, VectorDistances
+
+client.collections.create(
+    name=collection_name,
+    description="Configuration example",
+    vector_index_config=Configure.VectorIndex.dynamic(),
+    # Configure vectorizer, properties
+    properties=[
+        Property(name="title", data_type=DataType.TEXT),
+        Property(name="body", data_type=DataType.TEXT),
+    ]
+)
+# END EnableDynamic
+
+collection = client.collections.get(collection_name)
+collections_response = client.collections.list_all()
+schema_response = collection.config.get()
+
+assert collection_name in collections_response.keys(), "Collection missing"
+assert (
+    str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC"
+), "Wrong index type"
+
+######################
+### ENABLE DYNAMIC ###
+######################
+
+# Delete data from prior runs
+if client.collections.exists(collection_name):
+    client.collections.delete(collection_name)
+
+# START ConfigDynamic
+from weaviate.classes.config import Configure, VectorDistances
+
+client.collections.create(
+    name=collection_name,
+    vector_index_config=Configure.VectorIndex.dynamic(
+        distance_metric=VectorDistances.COSINE,
+        threshold=20000,
+        hnsw=Configure.VectorIndex.hnsw(
+            # Any hnsw configuration parameters
+            dynamic_ef_factor=15,  # Multiplier for dynamic Ef
+            dynamic_ef_min=200,    # Minimum threshold for dynamic Ef
+            dynamic_ef_max=1000,   # Maximum threshold for dynamic Ef
+        ),
+        flat=Configure.VectorIndex.flat(
+            # Any flat index configuration parameters
+            vector_cache_max_objects=100000,
+            quantizer=Configure.VectorIndex.Quantizer.bq()
+        ),
+    ),
+    # Configure vectorizer, properties
+)
+# END ConfigDynamic
+
+collection = client.collections.get(collection_name)
+collections_response = client.collections.list_all()
+schema_response = collection.config.get()
+
+assert collection_name in collections_response.keys(), "Collection missing"
+assert (
+    str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC"
+), "Wrong index type"
+
+################
+### CLEAN UP ###
+################
+
+client.close()