diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index ddc92d0ed..e5af0b408 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -33,6 +33,10 @@ services: OBJECTS_TTL_DELETE_SCHEDULE: "@every 12h" # for objectTTL tests to work EXPORT_ENABLED: 'true' EXPORT_DEFAULT_PATH: "/var/lib/weaviate/exports" + # for config.delete_vector_index tests to work. The endpoint is experimental and off by + # default; remove this once it is promoted to a supported release. Ignored by servers that + # do not know the flag. + ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT: 'true' contextionary: environment: diff --git a/integration/test_collection_config.py b/integration/test_collection_config.py index b1bd30e12..bba965f67 100644 --- a/integration/test_collection_config.py +++ b/integration/test_collection_config.py @@ -1,5 +1,6 @@ import datetime -from typing import Generator, List, Optional, Union +import time +from typing import Any, Dict, Generator, List, Optional, Union import pytest as pytest from _pytest.fixtures import SubRequest @@ -11,6 +12,7 @@ OpenAICollection, _sanitize_collection_name, ) +from weaviate.collections import Collection from weaviate.collections.classes.config import ( _BQConfig, _CollectionConfig, @@ -21,6 +23,7 @@ _VectorIndexConfigDynamic, _VectorIndexConfigFlat, _VectorIndexConfigHNSW, + _VectorIndexConfigNone, _VectorIndexConfigHNSWUpdate, Configure, Reconfigure, @@ -37,6 +40,7 @@ Rerankers, _RerankerProvider, Tokenization, + _NamedVectorConfig, _NamedVectorConfigCreate, _VectorizerConfigCreate, IndexName, @@ -2694,3 +2698,60 @@ def test_text_analyzer_roundtrip_from_dict( assert config == new assert config.to_dict() == new.to_dict() client.collections.delete(name) + + +def _vector_config_without_index( + collection: Collection[Any, Any], vector_name: str, timeout: float = 30 +) -> Dict[str, _NamedVectorConfig]: + """Poll the collection config until `vector_name` no longer has an index. + + The drop is applied asynchronously, a 200 from the endpoint only means that Weaviate accepted + the request. It then becomes visible in two steps: the vector first stays in the schema with a + `vector_index_config` of `_VectorIndexConfigNone`, and once the index is gone from disk the + entry is removed from the schema altogether. Both shapes must parse, so accept either. + """ + start = time.time() + while True: + vector_config = collection.config.get().vector_config + assert vector_config is not None + if vector_name not in vector_config or isinstance( + vector_config[vector_name].vector_index_config, _VectorIndexConfigNone + ): + return vector_config + if time.time() - start > timeout: + pytest.fail(f"vector index of {vector_name} was not dropped within {timeout}s") + time.sleep(0.2) + + +def test_delete_vector_index(collection_factory: CollectionFactory) -> None: + """Test that dropping the index of a named vector leaves the rest of the collection usable.""" + collection_dummy = collection_factory("dummy") + if collection_dummy._connection._weaviate_version.is_lower_than(1, 39, 0): + pytest.skip("delete vector index not supported before 1.39.0") + + collection = collection_factory( + properties=[Property(name="name", data_type=DataType.TEXT)], + vector_config=[ + Configure.Vectors.self_provided(name="dropped"), + Configure.Vectors.self_provided(name="kept"), + ], + ) + collection.data.insert( + properties={"name": "banana"}, + vector={"dropped": [1, 2], "kept": [3, 4]}, + ) + + config = collection.config.get() + assert config.vector_config is not None + assert not isinstance( + config.vector_config["dropped"].vector_index_config, _VectorIndexConfigNone + ) + + assert collection.config.delete_vector_index("dropped") is None + + vector_config = _vector_config_without_index(collection, "dropped") + # vectors that were not dropped keep their index + assert not isinstance(vector_config["kept"].vector_index_config, _VectorIndexConfigNone) + + # searching the vector that still has an index keeps working + assert len(collection.query.near_vector([3, 4], target_vector="kept").objects) == 1 diff --git a/mock_tests/test_collection.py b/mock_tests/test_collection.py index 7769e23e0..14450bd86 100644 --- a/mock_tests/test_collection.py +++ b/mock_tests/test_collection.py @@ -1,9 +1,11 @@ import datetime -from typing import Any, Dict, Literal +import json +from typing import Any, Dict, List, Literal import grpc import pytest from pytest_httpserver import HTTPServer +from werkzeug import Request, Response import weaviate import weaviate.classes as wvc @@ -504,6 +506,84 @@ async def test_async_collection_exists(weaviate_mock: HTTPServer) -> None: assert e.value.status_code == 500 +def test_delete_vector_index(weaviate_mock: HTTPServer) -> None: + # the collection name is capitalized by the client before it hits the path + weaviate_mock.expect_request( + "/v1/schema/Test/vectors/vec/index", method="DELETE" + ).respond_with_json(response_json={}, status=200) + weaviate_mock.expect_request( + "/v1/schema/Test/vectors/missing/index", method="DELETE" + ).respond_with_json( + response_json={"error": [{"message": "vector index missing not found"}]}, status=422 + ) + weaviate_mock.expect_request( + "/v1/schema/Test/vectors/disabled/index", method="DELETE" + ).respond_with_json( + response_json={ + "error": [ + { + "message": "alter schema drop vector index endpoint is experimental and disabled by default" + } + ] + }, + status=500, + ) + + with weaviate.connect_to_local( + port=MOCK_PORT, host=MOCK_IP, grpc_port=MOCK_PORT_GRPC, skip_init_checks=True + ) as client: + assert client.collections.use("test").config.delete_vector_index("vec") is None + + # a non-OK answer (e.g. unknown vector name) surfaces as UnexpectedStatusCodeError + with pytest.raises(weaviate.exceptions.UnexpectedStatusCodeError) as e: + client.collections.use("test").config.delete_vector_index("missing") + assert e.value.status_code == 422 + + # a disabled experimental endpoint answers 500; the server message must reach the + # exception rather than being masked as a missing vector + with pytest.raises(weaviate.exceptions.UnexpectedStatusCodeError) as disabled: + client.collections.use("test").config.delete_vector_index("disabled") + assert disabled.value.status_code == 500 + assert "experimental and disabled by default" in disabled.value.message + + with pytest.raises(weaviate.exceptions.WeaviateInvalidInputError): + client.collections.use("test").config.delete_vector_index(42) # type: ignore[arg-type] + + +def test_create_from_dict_skips_dropped_vectors(weaviate_mock: HTTPServer) -> None: + """Entries with vectorIndexType "none" cannot be re-created and are stripped before the POST.""" + bodies: List[Dict[str, Any]] = [] + + def handler(request: Request) -> Response: + body = request.get_json() + bodies.append(body) + return Response(json.dumps({"class": body["class"]}), content_type="application/json") + + weaviate_mock.expect_request("/v1/schema", method="POST").respond_with_handler(handler) + + hnsw_entry = {"vectorizer": {"none": {}}, "vectorIndexType": "hnsw", "vectorIndexConfig": {}} + dropped_entry = {"vectorizer": {"none": {}}, "vectorIndexType": "none"} + + with weaviate.connect_to_local( + port=MOCK_PORT, host=MOCK_IP, grpc_port=MOCK_PORT_GRPC, skip_init_checks=True + ) as client: + with pytest.warns(UserWarning, match=r"Col001.*dropped"): + client.collections.create_from_dict( + { + "class": "TestDropped", + "vectorConfig": {"dropped": dropped_entry, "kept": hnsw_entry}, + } + ) + assert bodies[-1]["vectorConfig"] == {"kept": hnsw_entry} + + # once every vector is stripped the empty block is omitted, not sent as {} + with pytest.warns(UserWarning, match=r"Col001.*only"): + client.collections.create_from_dict( + {"class": "TestAllDropped", "vectorConfig": {"only": dropped_entry}} + ) + assert "vectorConfig" not in bodies[-1] + + def test_grpc_client_version_header( metadata_capture_collection: tuple[ weaviate.collections.Collection, MockMetadataCaptureWeaviateService diff --git a/test/collection/test_config_methods.py b/test/collection/test_config_methods.py index 2e40acacc..748fdf7e1 100644 --- a/test/collection/test_config_methods.py +++ b/test/collection/test_config_methods.py @@ -1,10 +1,162 @@ +from typing import Any, Dict + +import pytest + +from weaviate.collections.classes.config import VectorIndexType, _VectorIndexConfigNone +from weaviate.exceptions import SchemaValidationError from weaviate.collections.classes.config_methods import ( _collection_config_from_json, + _collection_config_simple_from_json, _collection_configs_simple_from_json, _nested_properties_from_config, _properties_from_config, ) +HNSW_CONFIG = { + "skip": False, + "cleanupIntervalSeconds": 300, + "maxConnections": 64, + "efConstruction": 128, + "ef": -1, + "dynamicEfMin": 100, + "dynamicEfMax": 500, + "dynamicEfFactor": 8, + "vectorCacheMaxObjects": 1000000000000, + "flatSearchCutoff": 40000, + "distance": "cosine", +} + + +def _schema_with_vector_config(vector_config: Dict[str, Any]) -> Dict[str, Any]: + """Build a minimal collection schema, as returned by Weaviate, around the given vectorConfig.""" + return { + "class": "TestCollection", + "vectorConfig": vector_config, + "properties": [], + "invertedIndexConfig": { + "bm25": {"b": 0.75, "k1": 1.2}, + "cleanupIntervalSeconds": 60, + "stopwords": {"preset": "en", "additions": None, "removals": None}, + }, + "multiTenancyConfig": {"enabled": False}, + "replicationConfig": {"factor": 1, "deletionStrategy": "NoAutomatedResolution"}, + "shardingConfig": { + "virtualPerPhysical": 128, + "desiredCount": 1, + "actualCount": 1, + "desiredVirtualCount": 128, + "actualVirtualCount": 128, + "key": "_id", + "strategy": "hash", + "function": "murmur3", + }, + } + + +def test_collection_config_from_json_with_dropped_vector_index() -> None: + """A vector whose index was dropped is returned without a vectorIndexConfig.""" + # Shape returned by Weaviate after `collection.config.delete_vector_index("dropped")`: + # the entry stays in the schema, `vectorIndexType` becomes "none" and `vectorIndexConfig` + # is omitted entirely. + schema = _schema_with_vector_config( + { + "dropped": {"vectorizer": {"none": {}}, "vectorIndexType": "none"}, + "kept": { + "vectorizer": {"none": {}}, + "vectorIndexType": "hnsw", + "vectorIndexConfig": HNSW_CONFIG, + }, + } + ) + + config = _collection_config_from_json(schema) + + assert config.vector_config is not None + assert isinstance(config.vector_config["dropped"].vector_index_config, _VectorIndexConfigNone) + assert config.vector_config["dropped"].vector_index_config.vector_index_type() == "none" + assert not isinstance(config.vector_config["kept"].vector_index_config, _VectorIndexConfigNone) + + # The dropped vector must round-trip back to the "none" index type the server reported. + as_dict = config.to_dict() + assert as_dict["vectorConfig"]["dropped"]["vectorIndexType"] == VectorIndexType.NONE.value + assert "vectorIndexConfig" not in as_dict["vectorConfig"]["dropped"] + assert as_dict["vectorConfig"]["kept"]["vectorIndexType"] == VectorIndexType.HNSW.value + + +def test_collection_config_from_json_missing_vector_index_config_raises() -> None: + """A non-dropped vector missing its vectorIndexConfig must fail fast, not parse as None.""" + schema = _schema_with_vector_config( + {"broken": {"vectorizer": {"none": {}}, "vectorIndexType": "hnsw"}} + ) + + with pytest.raises(SchemaValidationError, match="broken"): + _collection_config_from_json(schema) + + +def test_collection_config_from_json_unknown_vector_index_type_raises() -> None: + """An index type the client does not know is reported as such, not as a missing config.""" + # `vectorIndexConfig` is present and populated; only the type is unknown to this client. + schema = _schema_with_vector_config( + { + "future": { + "vectorizer": {"none": {}}, + "vectorIndexType": "spann", + "vectorIndexConfig": {"distance": "cosine", "searchListSize": 100}, + } + } + ) + + with pytest.raises(SchemaValidationError, match="unknown vectorIndexType"): + _collection_config_from_json(schema) + + +def _schema_without_any_vector() -> Dict[str, Any]: + """Schema of a named-vector collection whose vectors were all dropped. + + Once the drops finalize the server removes every `vectorConfig` entry, so the block is + omitted, and a named-vector collection never has a top-level `vectorizer`, `vectorIndexType` + or `vectorIndexConfig`. (A legacy single-vector collection cannot reach this shape: the server + rejects dropping its index, so it always keeps a top-level `vectorizer`.) + """ + return { + "class": "TestCollection", + "properties": [], + "invertedIndexConfig": { + "bm25": {"b": 0.75, "k1": 1.2}, + "cleanupIntervalSeconds": 60, + "stopwords": {"preset": "en", "additions": None, "removals": None}, + }, + "multiTenancyConfig": {"enabled": False}, + "replicationConfig": {"factor": 1, "deletionStrategy": "NoAutomatedResolution"}, + "shardingConfig": { + "virtualPerPhysical": 128, + "desiredCount": 1, + "actualCount": 1, + "desiredVirtualCount": 128, + "actualVirtualCount": 128, + "key": "_id", + "strategy": "hash", + "function": "murmur3", + }, + } + + +def test_collection_config_from_json_all_vectors_dropped() -> None: + """A collection whose vectors were all dropped has no top-level vectorizer.""" + config = _collection_config_from_json(_schema_without_any_vector()) + + assert config.vectorizer is None + assert config.vector_index_type is None + assert config.vector_config is None + + +def test_collection_config_simple_from_json_all_vectors_dropped() -> None: + """`collections.list_all()` must not choke on a collection whose vectors were all dropped.""" + config = _collection_config_simple_from_json(_schema_without_any_vector()) + + assert config.vectorizer is None + assert config.vector_config is None + def test_collection_config_simple_from_json_with_none_vectorizer_config() -> None: """Test that _collection_configs_simple_from_json handles None vectorizer config.""" diff --git a/test/collection/test_config_update.py b/test/collection/test_config_update.py index 066d4ecb3..d2b9fc11a 100644 --- a/test/collection/test_config_update.py +++ b/test/collection/test_config_update.py @@ -230,3 +230,62 @@ def test_switching_quantizer_still_rejected_when_pq_enabled() -> None: ) with pytest.raises(WeaviateInvalidInputError): update.merge_with_existing(schema) + + +@pytest.mark.parametrize("use_deprecated_syntax", [False, True]) +def test_updating_dropped_vector_index(use_deprecated_syntax: bool) -> None: + """A vector whose index was dropped has no index config to merge into.""" + schema = multi_vector_schema() + # shape reported by Weaviate for a vector dropped via `config.delete_vector_index` + schema["vectorConfig"]["boi"] = {"vectorizer": {"none": {}}, "vectorIndexType": "none"} + + hnsw = Reconfigure.VectorIndex.hnsw(ef=128) + update = ( + _CollectionConfigUpdate( + vectorizer_config=[ + Reconfigure.NamedVectors.update(name="boi", vector_index_config=hnsw) + ] + ) + if use_deprecated_syntax + else _CollectionConfigUpdate( + vector_config=[Reconfigure.Vectors.update(name="boi", vector_index_config=hnsw)] + ) + ) + + with pytest.raises(WeaviateInvalidInputError, match="delete_vector_index"): + update.merge_with_existing(schema) + + +def test_updating_vector_next_to_dropped_vector_index() -> None: + """Vectors that still have an index remain updatable next to a dropped one.""" + schema = multi_vector_schema() + schema["vectorConfig"]["boi"] = {"vectorizer": {"none": {}}, "vectorIndexType": "none"} + + update = _CollectionConfigUpdate( + vector_config=[ + Reconfigure.Vectors.update( + name="yeh", vector_index_config=Reconfigure.VectorIndex.hnsw(ef=128) + ) + ] + ) + new_schema = update.merge_with_existing(schema) + + assert new_schema["vectorConfig"]["yeh"]["vectorIndexConfig"]["ef"] == 128 + assert new_schema["vectorConfig"]["boi"] == { + "vectorizer": {"none": {}}, + "vectorIndexType": "none", + } + + +def test_updating_vector_when_none_left() -> None: + """Once every vector is dropped the server omits vectorConfig; update must not raise KeyError.""" + update = _CollectionConfigUpdate( + vector_config=[ + Reconfigure.Vectors.update( + name="gone", vector_index_config=Reconfigure.VectorIndex.hnsw(ef=128) + ) + ] + ) + + with pytest.raises(WeaviateInvalidInputError, match="does not exist"): + update.merge_with_existing({"class": "Test", "properties": []}) diff --git a/weaviate/collections/classes/config.py b/weaviate/collections/classes/config.py index e9effaf15..f2c188164 100644 --- a/weaviate/collections/classes/config.py +++ b/weaviate/collections/classes/config.py @@ -1546,6 +1546,23 @@ def mutual_exclusivity( ) return v + @staticmethod + def __existing_vector_index_config(schema: Dict[str, Any], name: str) -> Dict[str, Any]: + # `vectorConfig` is omitted entirely once every named vector has been dropped. + if "vectorConfig" not in schema or name not in schema["vectorConfig"]: + raise WeaviateInvalidInputError( + f"Vector config with name {name} does not exist in the existing vector config" + ) + existing = schema["vectorConfig"][name] + if "vectorIndexConfig" not in existing: + # the index was dropped with `collection.config.delete_vector_index`, Weaviate reports + # such a vector as `vectorIndexType: "none"` without any index config to merge into + raise WeaviateInvalidInputError( + f"Vector config with name {name} has no vector index, it was deleted with " + "collection.config.delete_vector_index() and cannot be re-created" + ) + return cast(Dict[str, Any], existing["vectorIndexConfig"]) + def __check_quantizers( self, quantizer: Optional[_QuantizerConfigUpdate], @@ -1658,18 +1675,10 @@ def merge_with_existing(self, schema: Dict[str, Any]) -> Dict[str, Any]: ) else: for vc in self.vectorizerConfig: - if vc.name not in schema["vectorConfig"]: - raise WeaviateInvalidInputError( - f"Vector config with name {vc.name} does not exist in the existing vector config" - ) - self.__check_quantizers( - vc.vectorIndexConfig.quantizer, - schema["vectorConfig"][vc.name]["vectorIndexConfig"], - ) + existing = self.__existing_vector_index_config(schema, vc.name) + self.__check_quantizers(vc.vectorIndexConfig.quantizer, existing) schema["vectorConfig"][vc.name]["vectorIndexConfig"] = ( - vc.vectorIndexConfig.merge_with_existing( - schema["vectorConfig"][vc.name]["vectorIndexConfig"] - ) + vc.vectorIndexConfig.merge_with_existing(existing) ) schema["vectorConfig"][vc.name]["vectorIndexType"] = ( vc.vectorIndexConfig.vector_index_type() @@ -1681,18 +1690,10 @@ def merge_with_existing(self, schema: Dict[str, Any]) -> Dict[str, Any]: else self.vectorConfig ) for vc in vcs: - if vc.name not in schema["vectorConfig"]: - raise WeaviateInvalidInputError( - f"Vector config with name {vc.name} does not exist in the existing vector config" - ) - self.__check_quantizers( - vc.vectorIndexConfig.quantizer, - schema["vectorConfig"][vc.name]["vectorIndexConfig"], - ) + existing = self.__existing_vector_index_config(schema, vc.name) + self.__check_quantizers(vc.vectorIndexConfig.quantizer, existing) schema["vectorConfig"][vc.name]["vectorIndexConfig"] = ( - vc.vectorIndexConfig.merge_with_existing( - schema["vectorConfig"][vc.name]["vectorIndexConfig"] - ) + vc.vectorIndexConfig.merge_with_existing(existing) ) schema["vectorConfig"][vc.name]["vectorIndexType"] = ( vc.vectorIndexConfig.vector_index_type() @@ -2098,6 +2099,21 @@ def vector_index_type() -> str: VectorIndexConfigDynamic = _VectorIndexConfigDynamic +@dataclass +class _VectorIndexConfigNone(_ConfigBase): + """The index of this vector was dropped with `collection.config.delete_vector_index`. + + The vector data is still stored, but there is no index left to configure or search. + """ + + @staticmethod + def vector_index_type() -> str: + return VectorIndexType.NONE.value + + +VectorIndexConfigNone = _VectorIndexConfigNone + + @dataclass class _GenerativeConfig(_ConfigBase): generative: Union[GenerativeSearches, str] @@ -2147,11 +2163,15 @@ class _NamedVectorConfig(_ConfigBase): VectorIndexConfigFlat, VectorIndexConfigDynamic, VectorIndexConfigHFresh, + VectorIndexConfigNone, ] def to_dict(self) -> Dict: ret_dict = super().to_dict() ret_dict["vectorIndexType"] = self.vector_index_config.vector_index_type() + if isinstance(self.vector_index_config, _VectorIndexConfigNone): + # match the server: a dropped index is reported without any `vectorIndexConfig` + ret_dict.pop("vectorIndexConfig", None) return ret_dict diff --git a/weaviate/collections/classes/config_methods.py b/weaviate/collections/classes/config_methods.py index 691cf208d..7d7d75d2f 100644 --- a/weaviate/collections/classes/config_methods.py +++ b/weaviate/collections/classes/config_methods.py @@ -44,8 +44,10 @@ _VectorIndexConfigFlat, _VectorIndexConfigHFresh, _VectorIndexConfigHNSW, + _VectorIndexConfigNone, _VectorizerConfig, ) +from weaviate.exceptions import SchemaValidationError def _is_primitive(d_type: str) -> bool: @@ -282,8 +284,32 @@ def __get_vector_config( vec_config = {} props = vec_config.pop("properties", None) - vector_index_config = __get_vector_index_config(named_vector) - assert vector_index_config is not None + vector_index_config: Union[ + _VectorIndexConfigHNSW, + _VectorIndexConfigFlat, + _VectorIndexConfigDynamic, + _VectorIndexConfigHFresh, + _VectorIndexConfigNone, + None, + ] = __get_vector_index_config(named_vector) + if vector_index_config is None: + # A vector whose index was dropped with `collection.config.delete_vector_index` is + # returned as `vectorIndexType: "none"` without any `vectorIndexConfig`. + if named_vector.get("vectorIndexType") == VectorIndexType.NONE.value: + vector_index_config = _VectorIndexConfigNone() + elif "vectorIndexConfig" in named_vector: + # the config is present; this client version does not know the index type + raise SchemaValidationError( + f"Named vector {name!r} has an unknown vectorIndexType " + f"{named_vector.get('vectorIndexType')!r}; upgrade the client to a version " + "that supports it" + ) + else: + raise SchemaValidationError( + f"Named vector {name!r} has vectorIndexType " + f"{named_vector.get('vectorIndexType')!r} but no vectorIndexConfig in the " + "schema returned by Weaviate" + ) try: vec: Union[str, Vectorizers] = Vectorizers(vectorizer_str) except ValueError: @@ -306,6 +332,11 @@ def __get_vector_config( def __get_vectorizer(schema: Dict[str, Any]) -> Optional[Union[str, Vectorizers]]: if "vectorConfig" in schema: return None + # A named-vector collection whose vectors were all dropped with + # `collection.config.delete_vector_index` comes back with neither a `vectorConfig` block nor a + # top-level `vectorizer`. Return None instead of raising KeyError on the missing key. + if "vectorizer" not in schema: + return None vectorizer = str(schema["vectorizer"]) try: diff --git a/weaviate/collections/classes/config_vector_index.py b/weaviate/collections/classes/config_vector_index.py index ff6a0ba40..fe0166558 100644 --- a/weaviate/collections/classes/config_vector_index.py +++ b/weaviate/collections/classes/config_vector_index.py @@ -36,12 +36,16 @@ class VectorIndexType(str, Enum): FLAT: Flat index. DYNAMIC: Dynamic index. HFRESH: HFRESH index. + NONE: The index of this vector has been dropped, see ``collection.config.delete_vector_index``. + The vector data is still stored, but it cannot be searched. This value is reported by the + server only, it cannot be used to configure a vector. """ HNSW = "hnsw" FLAT = "flat" DYNAMIC = "dynamic" HFRESH = "hfresh" + NONE = "none" class _MultiVectorConfigCreateBase(_ConfigCreateModel): diff --git a/weaviate/collections/collections/base.py b/weaviate/collections/collections/base.py index 32296c137..c6af43c32 100644 --- a/weaviate/collections/collections/base.py +++ b/weaviate/collections/collections/base.py @@ -97,6 +97,10 @@ def create_from_dict( This method is helpful for those making the v3 -> v4 migration and for those interfacing with any experimental Weaviate features that are not yet fully supported by the Weaviate Python client. + Vector entries whose index was dropped with `collection.config.delete_vector_index` are + skipped with a warning: there is no API to re-create a vector without an index, so the new + collection simply does not contain them. + Args: config: The dictionary representation of the collection's configuration. @@ -115,6 +119,10 @@ def create_from_config( ]: """Use this method to create a collection in Weaviate and immediately return a collection object using a pre-defined Weaviate collection configuration object. + Vector entries whose index was dropped with `collection.config.delete_vector_index` are + skipped with a warning: there is no API to re-create a vector without an index, so the new + collection simply does not contain them. + Args: config: The collection's configuration. diff --git a/weaviate/collections/collections/executor.py b/weaviate/collections/collections/executor.py index 588737600..e22401378 100644 --- a/weaviate/collections/collections/executor.py +++ b/weaviate/collections/collections/executor.py @@ -37,6 +37,7 @@ _collection_configs_from_json, _collection_configs_simple_from_json, ) +from weaviate.collections.classes.config_vector_index import VectorIndexType from weaviate.collections.classes.internal import References from weaviate.collections.classes.types import ( Properties, @@ -103,6 +104,7 @@ def __create( Collection[Properties, References], Awaitable[CollectionAsync[Properties, References]], ]: + config = self.__without_dropped_vectors(config) result = self._connection.post( path="/schema", weaviate_object=config, @@ -137,6 +139,34 @@ async def execute_(): assert isinstance(collection, Collection) return collection + @staticmethod + def __without_dropped_vectors(config: dict) -> dict: + """Strip vector entries whose index was dropped (`vectorIndexType: "none"`). + + The server rejects the `"none"` sentinel on create and there is no API to re-create a + vector without an index. Keeping the entry with a real index type instead would silently + re-create an index that was deliberately dropped, so the whole entry is skipped — matching + where the server's own cleanup ends up once a drop finalizes. + """ + vector_config = config.get("vectorConfig") + if not isinstance(vector_config, dict): + return config + dropped = [ + name + for name, vc in vector_config.items() + if isinstance(vc, dict) and vc.get("vectorIndexType") == VectorIndexType.NONE.value + ] + if not dropped: + return config + _Warnings.create_skips_vectors_without_index(dropped) + config = { + **config, + "vectorConfig": {n: vc for n, vc in vector_config.items() if n not in dropped}, + } + if not config["vectorConfig"]: + del config["vectorConfig"] + return config + def __delete(self, *, name: str) -> executor.Result[None]: return executor.execute( response_callback=lambda res: None, diff --git a/weaviate/collections/config/async_.pyi b/weaviate/collections/config/async_.pyi index 015b70dab..13891e3c5 100644 --- a/weaviate/collections/config/async_.pyi +++ b/weaviate/collections/config/async_.pyi @@ -90,3 +90,4 @@ class _ConfigCollectionAsync(_ConfigCollectionExecutor[ConnectionAsync]): self, *, vector_config: Union[_VectorConfigCreate, List[_VectorConfigCreate]] ) -> None: ... async def delete_property_index(self, property_name: str, index_name: IndexName) -> bool: ... + async def delete_vector_index(self, vector_name: str) -> None: ... diff --git a/weaviate/collections/config/executor.py b/weaviate/collections/config/executor.py index 103ab70ac..7302bcdbf 100644 --- a/weaviate/collections/config/executor.py +++ b/weaviate/collections/config/executor.py @@ -666,3 +666,46 @@ def resp(res: Response) -> bool: error_msg="Property may not exist", status_codes=_ExpectedStatusCodes(ok_in=[200], error="property exists"), ) + + def delete_vector_index( + self, + vector_name: str, + ) -> executor.Result[None]: + """Delete the index of a named vector of the collection in Weaviate. + + This is a destructive and irreversible operation. The vectors themselves are kept, but + their index is removed from disk and cannot be re-created afterwards, neither through + this method nor through `collection.config.update()`. Searches and writes targeting the + vector are rejected once the index is gone. + + The drop is applied asynchronously. A successful call means that Weaviate accepted the + request, not that the index is already gone. `collection.config.get()` first reports the + vector with a `vector_index_config` of `VectorIndexConfigNone` and drops it from + `vector_config` altogether once the index has been removed from disk. + + Only named vectors can be dropped. The endpoint is experimental and may be disabled + server-side, in which case Weaviate rejects the request. + + Args: + vector_name: The name of the named vector whose index to delete. + + Raises: + weaviate.exceptions.WeaviateConnectionError: If the network connection to Weaviate fails. + weaviate.exceptions.UnexpectedStatusCodeError: If Weaviate reports a non-OK status, e.g. + if the vector does not exist or if the endpoint is disabled on the server. + weaviate.exceptions.WeaviateInvalidInputError: If `vector_name` is not a string. + """ + _validate_input([_ValidateArgument(expected=[str], name="vector_name", value=vector_name)]) + + path = f"/schema/{_capitalize_first_letter(self._name)}/vectors/{vector_name}/index" + + def resp(res: Response) -> None: + return None + + return executor.execute( + response_callback=resp, + method=self._connection.delete, + path=path, + error_msg="Vector index may not have been deleted", + status_codes=_ExpectedStatusCodes(ok_in=[200], error="delete vector index"), + ) diff --git a/weaviate/collections/config/sync.pyi b/weaviate/collections/config/sync.pyi index e54d8c8fc..9cee186c5 100644 --- a/weaviate/collections/config/sync.pyi +++ b/weaviate/collections/config/sync.pyi @@ -88,3 +88,4 @@ class _ConfigCollection(_ConfigCollectionExecutor[ConnectionSync]): self, *, vector_config: Union[_VectorConfigCreate, List[_VectorConfigCreate]] ) -> None: ... def delete_property_index(self, property_name: str, index_name: IndexName) -> bool: ... + def delete_vector_index(self, vector_name: str) -> None: ... diff --git a/weaviate/outputs/config.py b/weaviate/outputs/config.py index 17ebebf0e..8799ae724 100644 --- a/weaviate/outputs/config.py +++ b/weaviate/outputs/config.py @@ -25,6 +25,7 @@ VectorIndexConfigFlat, VectorIndexConfigHFresh, VectorIndexConfigHNSW, + VectorIndexConfigNone, VectorIndexType, VectorizerConfig, Vectorizers, @@ -57,6 +58,7 @@ "VectorIndexConfigHNSW", "VectorIndexConfigHFresh", "VectorIndexConfigFlat", + "VectorIndexConfigNone", "VectorIndexType", "Vectorizers", "VectorizerConfig", diff --git a/weaviate/warnings.py b/weaviate/warnings.py index 1c0a1ae0b..426f99175 100644 --- a/weaviate/warnings.py +++ b/weaviate/warnings.py @@ -282,6 +282,16 @@ def datetime_year_zero(date: str) -> None: stacklevel=1, ) + @staticmethod + def create_skips_vectors_without_index(vectors: list) -> None: + warnings.warn( + message=f"""Col001: The vector config(s) {vectors} have no vector index (it was dropped with + `collection.config.delete_vector_index`) and cannot be re-created. The collection is created + without these vectors; inserts and queries targeting them will fail.""", + category=UserWarning, + stacklevel=1, + ) + @staticmethod def batch_refresh_failed(err: str) -> None: warnings.warn(