Skip to content
This repository was archived by the owner on Sep 9, 2026. It is now read-only.

Commit c988644

Browse files
dirkkuldirkkul
authored
feat: update weaviate server to v1.16.0 and client to 3.9.0 (#736)
1 parent 8c7d38a commit c988644

10 files changed

Lines changed: 61 additions & 41 deletions

File tree

‎docarray/array/storage/weaviate/backend.py‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ class WeaviateConfig:
3737
# vectorIndexConfig parameters
3838
ef: Optional[int] = None
3939
ef_construction: Optional[int] = None
40-
timeout_config: Optional[Tuple[int, int]] = None
40+
timeout_config: Optional[Tuple[int, int]] = field(default=(10, 60))
4141
max_connections: Optional[int] = None
4242
dynamic_ef_min: Optional[int] = None
4343
dynamic_ef_max: Optional[int] = None
@@ -48,6 +48,13 @@ class WeaviateConfig:
4848
skip: Optional[bool] = None
4949
columns: Optional[Union[List[Tuple[str, str]], Dict[str, str]]] = None
5050
distance: Optional[str] = None
51+
# weaviate python client parameters
52+
batch_size: Optional[int] = field(default=50)
53+
dynamic_batching: Optional[bool] = field(default=False)
54+
55+
def __post_init__(self):
56+
if isinstance(self.timeout_config, list):
57+
self.timeout_config = tuple(self.timeout_config)
5158

5259

5360
_banned_classname_chars = [
@@ -98,7 +105,6 @@ def _init_storage(
98105
:raises ValueError: only one of name or docs can be used for initialization,
99106
raise an error if both are provided
100107
"""
101-
102108
config = copy.deepcopy(config)
103109
if not config:
104110
config = WeaviateConfig()
@@ -367,17 +373,14 @@ def _doc2weaviate_create_payload(self, value: 'Document'):
367373
vector=self._map_embedding(value.embedding),
368374
)
369375

370-
def _map_id(self, doc_id: str):
371-
"""the function maps doc id to weaviate id
372-
373-
:param doc_id: id of the document
374-
:return: weaviate object id
375-
"""
376-
# appending class name to doc id to handle the case:
377-
# daw1 = DocumentArrayWeaviate([Document(id=str(i), text='hi') for i in range(3)])
378-
# daw2 = DocumentArrayWeaviate([Document(id=str(i), text='bye') for i in range(3)])
379-
# daw2[0, 'text'] == 'hi' # this will be False if we don't append class name
380-
return str(uuid.uuid5(uuid.NAMESPACE_URL, doc_id + self._class_name))
376+
@staticmethod
377+
def _map_id(doc_id: str):
378+
# if doc_id is a random ID in hex format, just translate back to UUID str
379+
# otherwise, create UUID5 from doc_id
380+
try:
381+
return str(uuid.UUID(hex=doc_id))
382+
except ValueError:
383+
return str(uuid.uuid5(uuid.NAMESPACE_URL, doc_id))
381384

382385
def _map_embedding(self, embedding: 'ArrayType'):
383386
if embedding is not None:

‎docarray/array/storage/weaviate/find.py‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def _find_similar_vectors(
5959
if query_params:
6060
query_dict.update(query_params)
6161

62-
_additional = ['id', 'certainty']
62+
_additional = ['id', 'distance']
6363
if additional:
6464
_additional = _additional + additional
6565

@@ -70,15 +70,14 @@ def _find_similar_vectors(
7070
.with_near_vector(query_dict)
7171
)
7272

73-
if filter:
73+
if filter is not None:
7474
query_builder = query_builder.with_where(filter)
7575

76-
if sort:
76+
if sort is not None:
7777
query_builder = query_builder.with_sort(sort)
7878

7979
results = query_builder.do()
8080

81-
docs = []
8281
if 'errors' in results:
8382
errors = '\n'.join(map(lambda error: error['message'], results['errors']))
8483
raise ValueError(
@@ -88,16 +87,19 @@ def _find_similar_vectors(
8887
found_results = results.get('data', {}).get('Get', {}).get(self._class_name, [])
8988

9089
# The serialized document is stored in results['data']['Get'][self._class_name]
90+
91+
docs = []
92+
9193
for result in found_results:
9294
doc = Document.from_base64(result['_serialized'], **self._serialize_config)
93-
certainty = result['_additional']['certainty']
9495

95-
doc.scores['weaviate_certainty'] = NamedScore(value=certainty)
96+
distance = result['_additional']['distance']
97+
doc.scores['cosine_similarity'] = NamedScore(value=distance)
98+
doc.scores['distance'] = NamedScore(value=distance)
9699

97-
if certainty is None:
98-
doc.scores['cosine_similarity'] = NamedScore(value=None)
99-
else:
100-
doc.scores['cosine_similarity'] = NamedScore(value=2 * certainty - 1)
100+
certainty = result['_additional'].get('certainty', None)
101+
if certainty is not None:
102+
doc.scores['weaviate_certainty'] = NamedScore(value=certainty)
101103

102104
doc.tags['wid'] = result['_additional']['id']
103105

‎docarray/array/storage/weaviate/getsetdel.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ def _getitem(self, wid: str) -> 'Document':
1717
:return: Document
1818
"""
1919
try:
20-
resp = self._client.data_object.get_by_id(wid, with_vector=True)
20+
resp = self._client.data_object.get_by_id(
21+
wid, with_vector=True, class_name=self._class_name
22+
)
2123
return Document.from_base64(
2224
resp['properties']['_serialized'], **self._serialize_config
2325
)
@@ -60,9 +62,12 @@ def _del_doc_by_id(self, _id: str):
6062
6163
:param _id: the id of the document to delete
6264
"""
63-
wid = self._map_id(_id)
64-
if self._client.data_object.exists(wid):
65-
self._client.data_object.delete(wid)
65+
if self._client.data_object.exists(
66+
self._map_id(_id), class_name=self._class_name
67+
):
68+
self._client.data_object.delete(
69+
self._map_id(_id), class_name=self._class_name
70+
)
6671

6772
def _clear_storage(self):
6873
"""Concrete implementation of base class' ``_clear_storage``"""

‎docarray/array/storage/weaviate/seqlike.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,13 @@ def __contains__(self, x: Union[str, 'Document']):
4848
:return: True if ``x`` is contained in self
4949
"""
5050
if isinstance(x, str):
51-
return self._client.data_object.exists(self._map_id(x))
51+
return self._client.data_object.exists(
52+
self._map_id(x), class_name=self._class_name
53+
)
5254
elif isinstance(x, Document):
53-
return self._client.data_object.exists(self._map_id(x.id))
55+
return self._client.data_object.exists(
56+
self._map_id(x.id), class_name=self._class_name
57+
)
5458
else:
5559
return False
5660

@@ -65,7 +69,9 @@ def _extend(self, values: Iterable['Document'], **kwargs) -> None:
6569
6670
:param values: Documents to be added
6771
"""
68-
with self._client.batch(batch_size=50) as _b:
72+
with self._client.batch(
73+
batch_size=self._config.batch_size, dynamic=self._config.dynamic_batching
74+
) as _b:
6975
for d in values:
7076
_b.add_data_object(**self._doc2weaviate_create_payload(d))
7177
self._offset2ids.append(d.id)

‎docs/advanced/document-store/weaviate.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ services:
3131
- '8080'
3232
- --scheme
3333
- http
34-
image: semitechnologies/weaviate:1.13.2
34+
image: semitechnologies/weaviate:1.16.0
3535
ports:
3636
- "8080:8080"
3737
restart: on-failure:0

‎scripts/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
version: "3.3"
22
services:
33
weaviate:
4-
image: semitechnologies/weaviate:1.13.2
4+
image: semitechnologies/weaviate:1.16.0
55
ports:
66
- "41234:41234"
77
environment:

‎setup.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@
7373
'annlite',
7474
],
7575
'weaviate': [
76-
'weaviate-client~=3.3.0',
76+
'weaviate-client~=3.9.0',
7777
],
7878
'elasticsearch': [
7979
'elasticsearch>=8.2.0',
@@ -105,7 +105,7 @@
105105
'onnxruntime',
106106
'jupyterlab',
107107
'transformers>=4.16.2',
108-
'weaviate-client~=3.3.0',
108+
'weaviate-client~=3.9.0',
109109
'annlite',
110110
'elasticsearch>=8.2.0',
111111
'redis>=4.3.0',

‎tests/unit/array/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
version: "3.3"
22
services:
33
weaviate:
4-
image: semitechnologies/weaviate:1.13.2
4+
image: semitechnologies/weaviate:1.16.0
55
ports:
66
- "8080:8080"
77
environment:

‎tests/unit/array/mixins/test_find.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ def inv_cosine(*args):
2929
'storage, config',
3030
[
3131
('memory', None),
32-
('weaviate', {'n_dim': 32}),
32+
('weaviate', {'n_dim': 32, 'distance': 'cosine'}),
3333
('annlite', {'n_dim': 32}),
3434
('qdrant', {'n_dim': 32}),
3535
('elasticsearch', {'n_dim': 32}),
@@ -51,7 +51,10 @@ def test_find(storage, config, limit, query, start_storage):
5151

5252
da.extend([Document(embedding=v) for v in embeddings])
5353

54-
result = da.find(query, limit=limit)
54+
if storage == 'weaviate':
55+
result = da.find(query, limit=limit, additional=['certainty'])
56+
else:
57+
result = da.find(query, limit=limit)
5558
n_rows_query, n_dim = ndarray.get_array_rows(query)
5659

5760
if n_rows_query == 1 and n_dim == 1:
@@ -66,14 +69,14 @@ def test_find(storage, config, limit, query, start_storage):
6669
assert len(result) == n_rows_query
6770

6871
# check returned objects are sorted according to the storage backend metric
69-
# weaviate uses cosine similarity by default
72+
# weaviate uses distance by default
7073
# annlite uses cosine distance by default
7174
if n_dim == 1:
7275
if storage == 'weaviate':
7376
cosine_similarities = [
7477
t['cosine_similarity'].value for t in result[:, 'scores']
7578
]
76-
assert sorted(cosine_similarities, reverse=True) == cosine_similarities
79+
assert sorted(cosine_similarities, reverse=False) == cosine_similarities
7780
if storage == 'redis':
7881
cosine_distances = [t['score'].value for t in da[:, 'scores']]
7982
assert sorted(cosine_distances, reverse=False) == cosine_distances
@@ -86,7 +89,7 @@ def test_find(storage, config, limit, query, start_storage):
8689
cosine_similarities = [
8790
t['cosine_similarity'].value for t in da[:, 'scores']
8891
]
89-
assert sorted(cosine_similarities, reverse=True) == cosine_similarities
92+
assert sorted(cosine_similarities, reverse=False) == cosine_similarities
9093
if storage == 'redis':
9194
for da in result:
9295
cosine_distances = [t['score'].value for t in da[:, 'scores']]

‎tests/unit/array/storage/weaviate/test_query_params.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,14 @@ def find_random(da, target_certainty):
66
return da.find(
77
DocumentArray([Document(embedding=np.random.randint(10, size=10))]),
88
query_params={"certainty": target_certainty},
9+
additional=['certainty'],
910
)[0]
1011

1112

1213
def test_certainty_filter(start_storage):
1314
nrof_docs = 100
1415
target_certainty = 0.98
15-
da = DocumentArray(storage="weaviate", config={"n_dim": 10})
16+
da = DocumentArray(storage='weaviate', config={'n_dim': 10, 'distance': 'cosine'})
1617

1718
with da:
1819
da.extend(

0 commit comments

Comments
 (0)