Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit cf98028

Browse filesBrowse files
authored
fix(dataarray): fix load from a database with subindices (#581)
1 parent 57073c0 commit cf98028
Copy full SHA for cf98028

2 files changed

+70-3Lines changed: 70 additions & 3 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎docarray/array/storage/base/backend.py‎

Copy file name to clipboardExpand all lines: docarray/array/storage/base/backend.py
+9-2Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ def _init_storage(
2222
):
2323
self._load_offset2ids()
2424

25-
def _init_subindices(self, *args, **kwargs):
25+
def _init_subindices(
26+
self, _docs: Optional['DocumentArraySourceType'] = None, *args, **kwargs
27+
):
2628
self._subindices = {}
2729
subindex_configs = kwargs.get('subindex_configs', None)
2830
if subindex_configs:
@@ -39,7 +41,12 @@ def _init_subindices(self, *args, **kwargs):
3941
config, config_subindex, config_joined, name
4042
)
4143
self._subindices[name] = self.__class__(config=config_joined)
42-
self._subindices[name].extend(self.traverse_flat(name[1:]))
44+
if _docs:
45+
from docarray import DocumentArray
46+
47+
self._subindices[name].extend(
48+
DocumentArray(_docs).traverse_flat(name[1:])
49+
)
4350

4451
@abstractmethod
4552
def _ensure_unique_config(
Collapse file

‎tests/unit/array/test_backend_configuration.py‎

Copy file name to clipboardExpand all lines: tests/unit/array/test_backend_configuration.py
+61-1Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,20 @@
1+
import os
2+
import random
3+
14
import pytest
25
import requests
36

4-
from docarray import DocumentArray, Document
7+
from docarray import Document, DocumentArray, dataclass
8+
from docarray.typing import Image, Text
9+
10+
11+
@dataclass
12+
class MyDocument:
13+
image: Image
14+
paragraph: Text
15+
16+
17+
cur_dir = os.path.dirname(os.path.abspath(__file__))
518

619

720
def test_weaviate_hnsw(start_storage):
@@ -138,3 +151,50 @@ def test_cast_columns_qdrant(start_storage, type_da, type_column, request):
138151
index.extend(docs)
139152

140153
assert len(index) == N
154+
155+
156+
def test_random_subindices_config():
157+
database_index = random.randint(0, 100)
158+
database_name = "jina" + str(database_index) + ".db"
159+
table_index = random.randint(0, 100)
160+
table_name = "test" + str(table_index)
161+
subindice_image_index = random.randint(0, 100)
162+
subindice_image_name = "test" + str(subindice_image_index)
163+
subindice_paragraph_index = random.randint(0, 100)
164+
subindice_paragraph_name = "test" + str(subindice_paragraph_index)
165+
sqlite3_config = {'connection': database_name, 'table_name': table_name}
166+
167+
common_subindex_config = {
168+
'@.[image]': {'connection': database_name, 'table_name': subindice_image_name},
169+
'@.[paragraph]': {
170+
'connection': database_name,
171+
'table_name': subindice_paragraph_name,
172+
},
173+
}
174+
# extend with Documents, including embeddings
175+
_docs = [
176+
(
177+
MyDocument(
178+
image=os.path.join(cur_dir, '../document/toydata/test.png'),
179+
paragraph='hello world',
180+
)
181+
)
182+
]
183+
184+
da = DocumentArray(
185+
storage='sqlite', # use SQLite as vector database
186+
config=sqlite3_config,
187+
subindex_configs=common_subindex_config, # set up subindices for image and description
188+
)
189+
da.summary()
190+
191+
for item in _docs:
192+
d = Document(item)
193+
da.append(d)
194+
195+
da = DocumentArray(
196+
storage='sqlite', # use SQLite as vector database
197+
config=sqlite3_config,
198+
subindex_configs=common_subindex_config, # set up subindices for image and description
199+
)
200+
da.summary()

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.