Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit d3f2c61

Browse filesBrowse files
hanxiaohanxiao
authored
docs(document): add doc for document api (#8)
1 parent 0815a79 commit d3f2c61
Copy full SHA for d3f2c61

36 files changed

+797-470Lines changed: 797 additions & 470 deletions
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎docarray/array/chunk.py‎

Copy file name to clipboardExpand all lines: docarray/array/chunk.py
+16-1Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
1-
from typing import TYPE_CHECKING
1+
import itertools
2+
from typing import (
3+
TYPE_CHECKING,
4+
Generator,
5+
Iterator,
6+
Sequence,
7+
)
28

39
from .document import DocumentArray
410

@@ -24,6 +30,15 @@ def __init__(self, docs, reference_doc: 'Document'):
2430
"""
2531
self._ref_doc = reference_doc
2632
super().__init__(docs)
33+
if (
34+
isinstance(
35+
docs, (DocumentArray, Sequence, Generator, Iterator, itertools.chain)
36+
)
37+
and self._ref_doc is not None
38+
):
39+
for d in docs:
40+
d.parent_id = self._ref_doc.id
41+
d.granularity = self._ref_doc.granularity + 1
2742

2843
def append(self, document: 'Document'):
2944
"""Add a sub-document (i.e chunk) to the current Document.
Collapse file

‎docarray/array/document.py‎

Copy file name to clipboardExpand all lines: docarray/array/document.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ def __bool__(self):
242242
return len(self) > 0
243243

244244
def __repr__(self):
245-
return f'<{typename(self)} (length={len(self)}) at {id(self)}>'
245+
return f'<{self.__class__.__name__} (length={len(self)}) at {id(self)}>'
246246

247247
def __add__(self, other: 'Document'):
248248
v = type(self)()
Collapse file

‎docarray/array/match.py‎

Copy file name to clipboardExpand all lines: docarray/array/match.py
+15-2Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
1-
from typing import TYPE_CHECKING
1+
import itertools
2+
from typing import (
3+
TYPE_CHECKING,
4+
Generator,
5+
Iterator,
6+
Sequence,
7+
)
28

39
from .. import DocumentArray
410

@@ -18,13 +24,20 @@ class MatchArray(DocumentArray):
1824
def __init__(self, docs, reference_doc: 'Document'):
1925
self._ref_doc = reference_doc
2026
super().__init__(docs)
27+
if (
28+
isinstance(
29+
docs, (DocumentArray, Sequence, Generator, Iterator, itertools.chain)
30+
)
31+
and self._ref_doc is not None
32+
):
33+
for d in docs:
34+
d.adjacency = self._ref_doc.adjacency + 1
2135

2236
def append(self, document: 'Document'):
2337
"""Add a matched document to the current Document.
2438
2539
:param document: Sub-document to be added
2640
"""
27-
document.granularity = self._ref_doc.granularity
2841
document.adjacency = self._ref_doc.adjacency + 1
2942
super().append(document)
3043

Collapse file

‎docarray/array/mixins/io/dataframe.py‎

Copy file name to clipboardExpand all lines: docarray/array/mixins/io/dataframe.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def to_dataframe(self, **kwargs) -> 'DataFrame':
2121
"""
2222
from pandas import DataFrame
2323

24-
return DataFrame.from_dict(self.to_list_safe(), **kwargs)
24+
return DataFrame.from_dict(self.to_list(), **kwargs)
2525

2626
@classmethod
2727
def from_dataframe(cls: Type['T'], df: 'DataFrame') -> 'T':
Collapse file

‎docarray/array/mixins/io/json.py‎

Copy file name to clipboardExpand all lines: docarray/array/mixins/io/json.py
+4-4Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,24 +55,24 @@ def from_json(cls: Type['T'], file: Union[str, TextIO]) -> 'T':
5555
return cls.load_json(file)
5656

5757
@classmethod
58-
def from_list_safe(cls: Type['T'], values: List) -> 'T':
58+
def from_list(cls: Type['T'], values: List) -> 'T':
5959
from .... import Document
6060

6161
return cls(Document.from_dict(v) for v in values)
6262

63-
def to_list_safe(self) -> List:
63+
def to_list(self, strict: bool = True) -> List:
6464
"""Convert the object into a Python list.
6565
6666
.. note::
6767
Array like object such as :class:`numpy.ndarray` will be converted to Python list.
6868
6969
:return: a Python list
7070
"""
71-
return [d.to_dict() for d in self]
71+
return [d.to_dict(strict=strict) for d in self]
7272

7373
def to_json(self) -> str:
7474
"""Convert the object into a JSON string. Can be loaded via :meth:`.load_json`.
7575
7676
:return: a Python list
7777
"""
78-
return json.dumps(self.to_list_safe())
78+
return json.dumps(self.to_list())
Collapse file

‎docarray/base.py‎

Copy file name to clipboardExpand all lines: docarray/base.py
+16-12Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ def __init__(
1616
_obj: Optional['T'] = None,
1717
copy: bool = False,
1818
field_resolver: Optional[Dict[str, str]] = None,
19+
unknown_fields_handler: str = 'catch',
1920
**kwargs,
2021
):
2122
self._data = None
@@ -32,23 +33,26 @@ def __init__(
3233
kwargs = {field_resolver.get(k, k): v for k, v in kwargs.items()}
3334

3435
_unknown_kwargs = None
35-
if hasattr(self, '_unresolved_fields_dest'):
36-
_unresolved = set(kwargs.keys()).difference(
37-
{f.name for f in fields(self._data_class)}
38-
)
39-
if _unresolved:
40-
_unknown_kwargs = {k: kwargs[k] for k in _unresolved}
41-
for k in _unresolved:
42-
kwargs.pop(k)
36+
_unresolved = set(kwargs.keys()).difference(
37+
{f.name for f in fields(self._data_class)}
38+
)
39+
40+
if _unresolved:
41+
if unknown_fields_handler == 'raise':
42+
raise AttributeError(f'unknown attributes: {_unresolved}')
43+
44+
_unknown_kwargs = {k: kwargs[k] for k in _unresolved}
45+
for k in _unresolved:
46+
kwargs.pop(k)
4347

4448
self._data = self._data_class(self)
4549
for k, v in kwargs.items():
4650
setattr(self._data, k, v)
4751

48-
if _unknown_kwargs:
52+
if _unknown_kwargs and unknown_fields_handler == 'catch':
4953
getattr(self, self._unresolved_fields_dest).update(_unknown_kwargs)
5054

51-
if _obj is None and not kwargs:
55+
if _obj is None and not kwargs and self._data is None:
5256
self._data = self._data_class(self)
5357

5458
if self._data is None:
@@ -100,8 +104,8 @@ def __hash__(self):
100104

101105
def __repr__(self):
102106
content = str(self.non_empty_fields)
103-
content += f' at {id(self)}'
104-
return f'<{typename(self)} {content.strip()}>'
107+
content += f' at {getattr(self, "id", id(self))}'
108+
return f'<{self.__class__.__name__} {content.strip()}>'
105109

106110
def __bytes__(self):
107111
return self.to_bytes()
Collapse file

‎docarray/document/data.py‎

Copy file name to clipboardExpand all lines: docarray/document/data.py
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ def __setattr__(self, key, value):
8484
self.text = value
8585
else:
8686
self.blob = value
87+
value = None
8788
elif key == 'chunks':
8889
from ..array.chunk import ChunkArray
8990

Collapse file

‎docarray/document/mixins/plot.py‎

Copy file name to clipboardExpand all lines: docarray/document/mixins/plot.py
+1-96Lines changed: 1 addition & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -7,66 +7,6 @@
77
class PlotMixin:
88
"""Provide helper functions for :class:`Document` to plot and visualize itself. """
99

10-
@property
11-
def _mermaid_id(self):
12-
if not hasattr(self, '__mermaid_id'):
13-
self.__mermaid_id = random_identity()
14-
return self.__mermaid_id
15-
16-
def __mermaid_str__(self):
17-
results = []
18-
_id = f'{self._mermaid_id[:3]}~Document~'
19-
20-
for idx, c in enumerate(self.chunks):
21-
results.append(
22-
f'{_id} --> "{idx + 1}/{len(self.chunks)}" {c._mermaid_id[:3]}~Document~: chunks'
23-
)
24-
results.append(c.__mermaid_str__())
25-
26-
for idx, c in enumerate(self.matches):
27-
results.append(
28-
f'{_id} ..> "{idx + 1}/{len(self.matches)}" {c._mermaid_id[:3]}~Document~: matches'
29-
)
30-
results.append(c.__mermaid_str__())
31-
32-
content = self.to_dict()
33-
if 'chunks' in content:
34-
content.pop('chunks')
35-
if 'matches' in content:
36-
content.pop('matches')
37-
if content:
38-
results.append(f'class {_id}{{')
39-
for k, v in content.items():
40-
if isinstance(v, (str, int, float, bytes)):
41-
results.append(f'+{k} {str(v)[:10]}')
42-
else:
43-
results.append(f'+{k}({type(getattr(self, k, v))})')
44-
results.append('}')
45-
46-
return '\n'.join(results)
47-
48-
def _mermaid_to_url(self, img_type: str) -> str:
49-
"""
50-
Rendering the current flow as a url points to a SVG, it needs internet connection
51-
52-
:param img_type: the type of image to be generated
53-
:return: the url pointing to a SVG
54-
"""
55-
mermaid_str = (
56-
"""
57-
%%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#FFC666'}}}%%
58-
classDiagram
59-
60-
"""
61-
+ self.__mermaid_str__()
62-
)
63-
64-
encoded_str = base64.b64encode(bytes(mermaid_str.strip(), 'utf-8')).decode(
65-
'utf-8'
66-
)
67-
68-
return f'https://mermaid.ink/{img_type}/{encoded_str}'
69-
7010
def _ipython_display_(self):
7111
"""Displays the object in IPython as a side effect"""
7212
self.summary()
@@ -92,7 +32,7 @@ def _plot_recursion(self, _str_list, indent, box_char='├─'):
9232
_str_list, indent=len(prefix) + 4, box_char='└─'
9333
)
9434

95-
def plot_image(self):
35+
def plot(self):
9636
""" Plot image data from :attr:`.blob` or :attr:`.uri`. """
9737
from IPython.display import Image, display
9838

@@ -104,38 +44,3 @@ def plot_image(self):
10444
display(Image(self.uri))
10545
else:
10646
raise ValueError('`uri` and `blob` is empty')
107-
108-
def plot(self, output: Optional[str] = None, inline_display: bool = False) -> None:
109-
"""
110-
Visualize the Document recursively.
111-
112-
:param output: a filename specifying the name of the image to be created,
113-
the suffix svg/jpg determines the file type of the output image
114-
:param inline_display: show image directly inside the Jupyter Notebook
115-
"""
116-
image_type = 'svg'
117-
if (
118-
not output.endswith('.svg')
119-
and not output.endswith('.jpg')
120-
and not output.endswith('.jpeg')
121-
):
122-
raise ValueError('`output` can be only SVG/JPG format')
123-
elif output.endswith('.jpg') or output.endswith('.jpeg'):
124-
image_type = 'img'
125-
126-
url = self._mermaid_to_url(image_type)
127-
showed = False
128-
if inline_display:
129-
try:
130-
from IPython.display import Image, display
131-
132-
display(Image(url=url))
133-
showed = True
134-
except:
135-
# no need to panic users
136-
pass
137-
138-
if output:
139-
download_mermaid_url(url, output)
140-
elif not showed:
141-
print(f'Document visualization: {url}')
Collapse file

‎docarray/document/mixins/porting.py‎

Copy file name to clipboardExpand all lines: docarray/document/mixins/porting.py
+18-7Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1+
import dataclasses
12
import pickle
2-
from typing import Optional, TYPE_CHECKING, Type, Dict
3+
from typing import Optional, TYPE_CHECKING, Type, Dict, Any
34

45
from ...helper import compress_bytes, decompress_bytes
56

@@ -26,13 +27,16 @@ def from_json(cls: Type['T'], obj: str) -> 'T':
2627
json_format.Parse(obj, pb_msg)
2728
return cls.from_protobuf(pb_msg)
2829

29-
def to_dict(self):
30-
from google.protobuf.json_format import MessageToDict
30+
def to_dict(self, strict: bool = True) -> Dict[str, Any]:
31+
if strict:
32+
from google.protobuf.json_format import MessageToDict
3133

32-
return MessageToDict(
33-
self.to_protobuf(),
34-
preserving_proto_field_name=True,
35-
)
34+
return MessageToDict(
35+
self.to_protobuf(),
36+
preserving_proto_field_name=True,
37+
)
38+
else:
39+
return dataclasses.asdict(self._data)
3640

3741
def to_bytes(
3842
self, protocol: str = 'pickle', compress: Optional[str] = None
@@ -54,6 +58,13 @@ def from_bytes(
5458
protocol: str = 'pickle',
5559
compress: Optional[str] = None,
5660
) -> 'T':
61+
"""Build Document object from binary bytes
62+
63+
:param data: binary bytes
64+
:param protocol: protocol to use
65+
:param compress: compress method to use
66+
:return: a Document object
67+
"""
5768
bstr = decompress_bytes(data, algorithm=compress)
5869
if protocol == 'pickle':
5970
d = pickle.loads(bstr)
Collapse file

‎docarray/document/mixins/property.py‎

Copy file name to clipboardExpand all lines: docarray/document/mixins/property.py
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
class PropertyMixin(_PropertyMixin):
1313
def _clear_content(self):
14+
self._data.content = None
1415
self._data.text = None
1516
self._data.blob = None
1617
self._data.buffer = None

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.