[ENH] Turn on spann by default

chromadb/test/property/invariants.py

@@ -246,16 +246,20 @@ def fd_not_exceeding_threadpool_size(threadpool_size: int) -> None:
     )
 
 def get_space(collection: Collection):
+    # TODO: this is a hack to get the space
+    # We should update the tests to not pass space via metadata instead use collection
+    # configuration_json
+    space = None
     if "hnsw:space" in collection.metadata:
-        return collection.metadata["hnsw:space"]
+        space = collection.metadata["hnsw:space"]
     if collection._model.configuration_json is None:
-        return None
+        return space
     if 'spann' in collection._model.configuration_json and collection._model.configuration_json.get('spann') is not None and 'space' in collection._model.configuration_json.get('spann'):
-        return collection._model.configuration_json.get('spann').get('space')
+        space = collection._model.configuration_json.get('spann').get('space')
     elif 'hnsw' in collection._model.configuration_json and collection._model.configuration_json.get('hnsw') is not None and 'space' in collection._model.configuration_json.get('hnsw'):
-        return collection._model.configuration_json.get('hnsw').get('space')
-    else:
-        return None
+        if space is None:
+            space = collection._model.configuration_json.get('hnsw').get('space')
+    return space
 
 def ann_accuracy(
     collection: Collection,

rust/frontend/sample_configs/distributed.yaml

@@ -46,4 +46,4 @@ scorecard:
       - "collection_id:*"
     score: 100
 enable_span_indexing: true
-default_knn_index: "hnsw"
+default_knn_index: "spann"

rust/frontend/sample_configs/tilt_config.yaml

@@ -54,4 +54,4 @@ scorecard:
 circuit_breaker:
   requests: 1000
 enable_span_indexing: true
-default_knn_index: "hnsw"
+default_knn_index: "spann"

rust/frontend/src/impls/service_based_frontend.rs

@@ -1438,9 +1438,13 @@ mod tests {
         assert!(segments.iter().any(
             |s| s.r#type == SegmentType::BlockfileMetadata && s.scope == SegmentScope::METADATA
         ));
-        assert!(segments
-            .iter()
-            .any(|s| s.r#type == SegmentType::HnswDistributed && s.scope == SegmentScope::VECTOR));
+        assert!(
+            segments.iter().any(
+                |s| s.r#type == SegmentType::HnswDistributed && s.scope == SegmentScope::VECTOR
+            ) || segments
+                .iter()
+                .any(|s| s.r#type == SegmentType::Spann && s.scope == SegmentScope::VECTOR)
+        );
         assert!(segments
             .iter()
             .any(|s| s.r#type == SegmentType::BlockfileRecord && s.scope == SegmentScope::RECORD));

rust/index/src/spann/types.rs

@@ -2022,6 +2022,11 @@ impl<'me> SpannIndexReader<'me> {
         dimensionality: usize,
         ef_search: usize,
     ) -> Result<HnswIndexRef, SpannIndexReaderError> {
+        // We take a lock here to synchronize concurrent open of the same index.
+        // Otherwise, we could end up with a corrupted index since the filesystem
+        // operations are not guaranteed to be atomic.
+        // The lock is a partitioned mutex to allow for higher concurrency across collections.
+        let _guard = hnsw_provider.write_mutex.lock(id).await;
         match hnsw_provider.get(id, cache_key).await {
             Some(index) => Ok(index),
             None => {