Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 2098aa7

Browse filesBrowse files
davidbpalaeddine-13
davidbp
and
alaeddine-13
authored
feat: filter in weav (#373)
* feat: filter in weav * test: test filter weaviate without vector * docs: add example of filter without vector * refactor: remove unneeded list return * test: test private filter * refactor: remove unneeded list in weaviate find * docs: docs fixes * docs: add python marker to code Co-authored-by: AlaeddineAbdessalem <alaeddine-13@live.fr>
1 parent 1df47c4 commit 2098aa7
Copy full SHA for 2098aa7

3 files changed

+140-5Lines changed: 140 additions & 5 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎docarray/array/storage/weaviate/find.py‎

Copy file name to clipboardExpand all lines: docarray/array/storage/weaviate/find.py
+39-3Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,7 @@ def _find_similar_vectors(
5858
f'find failed, please check your filter query. Errors: \n{errors}'
5959
)
6060

61-
found_results = (
62-
results.get('data', {}).get('Get', {}).get(self._class_name, []) or []
63-
)
61+
found_results = results.get('data', {}).get('Get', {}).get(self._class_name, [])
6462

6563
# The serialized document is stored in results['data']['Get'][self._class_name]
6664
for result in found_results:
@@ -80,6 +78,44 @@ def _find_similar_vectors(
8078

8179
return DocumentArray(docs)
8280

81+
def _filter(
82+
self,
83+
filter: Dict,
84+
) -> 'DocumentArray':
85+
"""Returns a subset of documents by filtering by the given filter (Weaviate `where` filter).
86+
87+
:param filter: the input filter to apply in each stored document
88+
:return: a `DocumentArray` containing the `Document` objects that verify the filter.
89+
"""
90+
if not filter:
91+
return self
92+
93+
results = (
94+
self._client.query.get(self._class_name, '_serialized')
95+
.with_additional('id')
96+
.with_where(filter)
97+
.do()
98+
)
99+
100+
docs = []
101+
if 'errors' in results:
102+
errors = '\n'.join(map(lambda error: error['message'], results['errors']))
103+
raise ValueError(
104+
f'filter failed, please check your filter query. Errors: \n{errors}'
105+
)
106+
107+
found_results = results.get('data', {}).get('Get', {}).get(self._class_name, [])
108+
109+
# The serialized document is stored in results['data']['Get'][self._class_name]
110+
for result in found_results:
111+
doc = Document.from_base64(result['_serialized'], **self._serialize_config)
112+
113+
doc.tags['wid'] = result['_additional']['id']
114+
115+
docs.append(doc)
116+
117+
return DocumentArray(docs)
118+
83119
def _find(
84120
self,
85121
query: 'WeaviateArrayType',
Collapse file

‎docs/advanced/document-store/weaviate.md‎

Copy file name to clipboardExpand all lines: docs/advanced/document-store/weaviate.md
+57-2Lines changed: 57 additions & 2 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -169,13 +169,66 @@ print(results[0].text)
169169
Persist Documents with Weaviate.
170170
```
171171

172-
(weaviate-filter)=
173172
## Vector search with filter
174173

175174
Search with `.find` can be restricted by user-defined filters. Such filters can be constructed following the guidelines
176175
in [Weaviate's Documentation](https://weaviate.io/developers/weaviate/current/graphql-references/filters.html).
177176

178-
### Example of `.find` with a filter
177+
### Example of `.find` with a filter only
178+
179+
Consider you store Documents with a certain tag `price` into weaviate and you want to retrieve all Documents
180+
with `price` lower or equal to some `max_price` value.
181+
182+
183+
You can index such Documents as follows:
184+
```python
185+
from docarray import Document, DocumentArray
186+
import numpy as np
187+
188+
n_dim = 3
189+
da = DocumentArray(
190+
storage='weaviate',
191+
config={
192+
'n_dim': n_dim,
193+
'columns': [('price', 'float')],
194+
},
195+
)
196+
197+
with da:
198+
da.extend([Document(id=f'r{i}', tags={'price': i}) for i in range(10)])
199+
200+
print('\nIndexed Embeddings:\n')
201+
for price in da[:, 'tags__price']:
202+
print(f'\t price={price}')
203+
```
204+
205+
Then you can retrieve all documents whose price is lower than or equal to `max_price` by applying the following
206+
filter:
207+
208+
```python
209+
max_price = 3
210+
n_limit = 4
211+
212+
filter = {'path': 'price', 'operator': 'LessThanEqual', 'valueNumber': max_price}
213+
results = da.find(filter=filter)
214+
215+
print('\n Returned examples that verify filter "price at most 7":\n')
216+
for price in results[:, 'tags__price']:
217+
print(f'\t price={price}')
218+
```
219+
220+
This would print
221+
222+
```
223+
Returned examples that satisfy condition "price at most 3":
224+
225+
price=0
226+
price=1
227+
price=2
228+
price=3
229+
```
230+
231+
### Example of `.find` with query vector and filter
179232

180233
Consider Documents with embeddings `[0,0,0]` up to ` [9,9,9]` where the document with embedding `[i,i,i]`
181234
has as tag `price` with value `i`. We can create such example with the following code:
@@ -236,3 +289,5 @@ Embeddings Nearest Neighbours with "price" at most 7:
236289
embedding=[5. 5. 5.], price=5
237290
embedding=[4. 4. 4.], price=4
238291
```
292+
293+
Collapse file

‎tests/unit/array/mixins/test_find.py‎

Copy file name to clipboardExpand all lines: tests/unit/array/mixins/test_find.py
+44Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,45 @@ def test_search_pre_filtering(
319319
)
320320

321321

322+
@pytest.mark.parametrize(
323+
'storage,filter_gen,numeric_operators,operator',
324+
[
325+
*[
326+
tuple(
327+
[
328+
'weaviate',
329+
lambda operator, threshold: {
330+
'path': ['price'],
331+
'operator': operator,
332+
'valueNumber': threshold,
333+
},
334+
numeric_operators_weaviate,
335+
operator,
336+
]
337+
)
338+
for operator in numeric_operators_weaviate.keys()
339+
]
340+
],
341+
)
342+
def test_filtering(storage, filter_gen, operator, numeric_operators, start_storage):
343+
n_dim = 128
344+
da = DocumentArray(
345+
storage=storage, config={'n_dim': n_dim, 'columns': [('price', 'float')]}
346+
)
347+
348+
da.extend([Document(id=f'r{i}', tags={'price': i}) for i in range(50)])
349+
thresholds = [10, 20, 30]
350+
351+
for threshold in thresholds:
352+
353+
filter = filter_gen(operator, threshold)
354+
results = da.find(filter=filter)
355+
356+
assert all(
357+
[numeric_operators[operator](r.tags['price'], threshold) for r in results]
358+
)
359+
360+
322361
def test_weaviate_filter_query(start_storage):
323362
n_dim = 128
324363
da = DocumentArray(
@@ -335,6 +374,11 @@ def test_weaviate_filter_query(start_storage):
335374
with pytest.raises(ValueError):
336375
da.find(np.random.rand(n_dim), filter={'wrong': 'filter'})
337376

377+
with pytest.raises(ValueError):
378+
da._filter(filter={'wrong': 'filter'})
379+
380+
assert isinstance(da._filter(filter={}), type(da))
381+
338382

339383
@pytest.mark.parametrize('storage', ['memory', 'elasticsearch'])
340384
def test_unsupported_pre_filtering(storage, start_storage):

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.