Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit b3649b4

Browse filesBrowse files
Joan Fontanalssamsja
Joan Fontanals
and
samsja
authored
feat: make DocList an actual Python List (#1457)
Signed-off-by: Joan Fontanals Martinez <joan.martinez@jina.ai> Signed-off-by: samsja <sami.jaghouar@hotmail.fr> Co-authored-by: samsja <sami.jaghouar@hotmail.fr>
1 parent 7ba430c commit b3649b4
Copy full SHA for b3649b4

9 files changed

+62-142Lines changed: 62 additions & 142 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎docarray/array/doc_list/doc_list.py‎

Copy file name to clipboardExpand all lines: docarray/array/doc_list/doc_list.py
+33-49Lines changed: 33 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
import io
2-
from functools import wraps
32
from typing import (
43
TYPE_CHECKING,
54
Any,
6-
Callable,
75
Iterable,
86
List,
97
MutableSequence,
@@ -15,15 +13,13 @@
1513
overload,
1614
)
1715

16+
from typing_extensions import SupportsIndex
1817
from typing_inspect import is_union_type
1918

2019
from docarray.array.any_array import AnyDocArray
2120
from docarray.array.doc_list.io import IOMixinArray
2221
from docarray.array.doc_list.pushpull import PushPullMixin
23-
from docarray.array.doc_list.sequence_indexing_mixin import (
24-
IndexingSequenceMixin,
25-
IndexIterType,
26-
)
22+
from docarray.array.list_advance_indexing import IndexIterType, ListAdvancedIndexing
2723
from docarray.base_doc import AnyDoc, BaseDoc
2824
from docarray.typing import NdArray
2925

@@ -40,25 +36,11 @@
4036
T_doc = TypeVar('T_doc', bound=BaseDoc)
4137

4238

43-
def _delegate_meth_to_data(meth_name: str) -> Callable:
44-
"""
45-
create a function that mimic a function call to the data attribute of the
46-
DocList
47-
48-
:param meth_name: name of the method
49-
:return: a method that mimic the meth_name
50-
"""
51-
func = getattr(list, meth_name)
52-
53-
@wraps(func)
54-
def _delegate_meth(self, *args, **kwargs):
55-
return getattr(self._data, meth_name)(*args, **kwargs)
56-
57-
return _delegate_meth
58-
59-
6039
class DocList(
61-
IndexingSequenceMixin[T_doc], PushPullMixin, IOMixinArray, AnyDocArray[T_doc]
40+
ListAdvancedIndexing[T_doc],
41+
PushPullMixin,
42+
IOMixinArray,
43+
AnyDocArray[T_doc],
6244
):
6345
"""
6446
DocList is a container of Documents.
@@ -129,8 +111,13 @@ class Image(BaseDoc):
129111
def __init__(
130112
self,
131113
docs: Optional[Iterable[T_doc]] = None,
114+
validate_input_docs: bool = True,
132115
):
133-
self._data: List[T_doc] = list(self._validate_docs(docs)) if docs else []
116+
if validate_input_docs:
117+
docs = self._validate_docs(docs) if docs else []
118+
else:
119+
docs = docs if docs else []
120+
super().__init__(docs)
134121

135122
@classmethod
136123
def construct(
@@ -143,9 +130,7 @@ def construct(
143130
:param docs: a Sequence (list) of Document with the same schema
144131
:return: a `DocList` object
145132
"""
146-
new_docs = cls.__new__(cls)
147-
new_docs._data = docs if isinstance(docs, list) else list(docs)
148-
return new_docs
133+
return cls(docs, False)
149134

150135
def __eq__(self, other: Any) -> bool:
151136
if self.__len__() != other.__len__():
@@ -168,12 +153,6 @@ def _validate_one_doc(self, doc: T_doc) -> T_doc:
168153
raise ValueError(f'{doc} is not a {self.doc_type}')
169154
return doc
170155

171-
def __len__(self):
172-
return len(self._data)
173-
174-
def __iter__(self):
175-
return iter(self._data)
176-
177156
def __bytes__(self) -> bytes:
178157
with io.BytesIO() as bf:
179158
self._write_bytes(bf=bf)
@@ -185,7 +164,7 @@ def append(self, doc: T_doc):
185164
as the `.doc_type` of this `DocList` otherwise it will fail.
186165
:param doc: A Document
187166
"""
188-
self._data.append(self._validate_one_doc(doc))
167+
super().append(self._validate_one_doc(doc))
189168

190169
def extend(self, docs: Iterable[T_doc]):
191170
"""
@@ -194,31 +173,28 @@ def extend(self, docs: Iterable[T_doc]):
194173
fail.
195174
:param docs: Iterable of Documents
196175
"""
197-
self._data.extend(self._validate_docs(docs))
176+
super().extend(self._validate_docs(docs))
198177

199-
def insert(self, i: int, doc: T_doc):
178+
def insert(self, i: SupportsIndex, doc: T_doc):
200179
"""
201180
Insert a Document to the `DocList`. The Document must be from the same
202181
class as the doc_type of this `DocList` otherwise it will fail.
203182
:param i: index to insert
204183
:param doc: A Document
205184
"""
206-
self._data.insert(i, self._validate_one_doc(doc))
207-
208-
pop = _delegate_meth_to_data('pop')
209-
remove = _delegate_meth_to_data('remove')
210-
reverse = _delegate_meth_to_data('reverse')
211-
sort = _delegate_meth_to_data('sort')
185+
super().insert(i, self._validate_one_doc(doc))
212186

213187
def _get_data_column(
214188
self: T,
215189
field: str,
216190
) -> Union[MutableSequence, T, 'TorchTensor', 'NdArray']:
217-
"""Return all values of the fields from all docs this doc_list contains
218-
219-
:param field: name of the fields to extract
220-
:return: Returns a list of the field value for each document
221-
in the doc_list like container
191+
"""Return all v @classmethod
192+
def __class_getitem__(cls, item: Union[Type[BaseDoc], TypeVar, str]):alues of the fields from all docs this doc_list contains
193+
@classmethod
194+
def __class_getitem__(cls, item: Union[Type[BaseDoc], TypeVar, str]):
195+
:param field: name of the fields to extract
196+
:return: Returns a list of the field value for each document
197+
in the doc_list like container
222198
"""
223199
field_type = self.__class__.doc_type._get_field_type(field)
224200

@@ -299,7 +275,7 @@ def from_protobuf(cls: Type[T], pb_msg: 'DocListProto') -> T:
299275
return super().from_protobuf(pb_msg)
300276

301277
@overload
302-
def __getitem__(self, item: int) -> T_doc:
278+
def __getitem__(self, item: SupportsIndex) -> T_doc:
303279
...
304280

305281
@overload
@@ -308,3 +284,11 @@ def __getitem__(self: T, item: IndexIterType) -> T:
308284

309285
def __getitem__(self, item):
310286
return super().__getitem__(item)
287+
288+
@classmethod
289+
def __class_getitem__(cls, item: Union[Type[BaseDoc], TypeVar, str]):
290+
291+
if isinstance(item, type) and issubclass(item, BaseDoc):
292+
return AnyDocArray.__class_getitem__.__func__(cls, item) # type: ignore
293+
else:
294+
return super().__class_getitem__(item)
Collapse file

‎docarray/array/doc_list/io.py‎

Copy file name to clipboardExpand all lines: docarray/array/doc_list/io.py
+1-9Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@ def __getitem__(self, item: slice):
9999

100100
class IOMixinArray(Iterable[T_doc]):
101101
doc_type: Type[T_doc]
102-
_data: List[T_doc]
103102

104103
@abstractmethod
105104
def __len__(self):
@@ -329,14 +328,7 @@ def to_json(self) -> bytes:
329328
"""Convert the object into JSON bytes. Can be loaded via `.from_json`.
330329
:return: JSON serialization of `DocList`
331330
"""
332-
return orjson_dumps(self._data)
333-
334-
def _docarray_to_json_compatible(self) -> List[T_doc]:
335-
"""
336-
Convert itself into a json compatible object
337-
:return: A list of documents
338-
"""
339-
return self._data
331+
return orjson_dumps(self)
340332

341333
@classmethod
342334
def from_csv(
Collapse file

‎docarray/array/doc_vec/column_storage.py‎

Copy file name to clipboardExpand all lines: docarray/array/doc_vec/column_storage.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
Union,
1111
)
1212

13-
from docarray.array.doc_vec.list_advance_indexing import ListAdvancedIndexing
13+
from docarray.array.list_advance_indexing import ListAdvancedIndexing
1414
from docarray.typing import NdArray
1515
from docarray.typing.tensor.abstract_tensor import AbstractTensor
1616

Collapse file

‎docarray/array/doc_vec/doc_vec.py‎

Copy file name to clipboardExpand all lines: docarray/array/doc_vec/doc_vec.py
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from docarray.array.any_array import AnyDocArray
2222
from docarray.array.doc_list.doc_list import DocList
2323
from docarray.array.doc_vec.column_storage import ColumnStorage, ColumnStorageView
24-
from docarray.array.doc_vec.list_advance_indexing import ListAdvancedIndexing
24+
from docarray.array.list_advance_indexing import ListAdvancedIndexing
2525
from docarray.base_doc import BaseDoc
2626
from docarray.base_doc.mixins.io import _type_to_protobuf
2727
from docarray.typing import NdArray
@@ -271,9 +271,9 @@ def _get_data_column(
271271
in the array like container
272272
"""
273273
if field in self._storage.any_columns.keys():
274-
return self._storage.any_columns[field].data
274+
return self._storage.any_columns[field]
275275
elif field in self._storage.docs_vec_columns.keys():
276-
return self._storage.docs_vec_columns[field].data
276+
return self._storage.docs_vec_columns[field]
277277
elif field in self._storage.columns.keys():
278278
return self._storage.columns[field]
279279
else:
Collapse file

‎docarray/array/doc_vec/list_advance_indexing.py‎

Copy file name to clipboardExpand all lines: docarray/array/doc_vec/list_advance_indexing.py
-41Lines changed: 0 additions & 41 deletions
This file was deleted.

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.