Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 98d1f1f

Browse filesBrowse files
JohannesMessnerJohannesMessner
authored
fix: from_dataframe with numpy==1.26.1 and type handling in python 3.9 (#1823)
Signed-off-by: Johannes Messner <messnerjo@gmail.com>
1 parent 7479f59 commit 98d1f1f
Copy full SHA for 98d1f1f

File tree

Expand file treeCollapse file tree

7 files changed

+66
-15
lines changed
Filter options
Expand file treeCollapse file tree

7 files changed

+66
-15
lines changed

‎.github/workflows/ci.yml

Copy file name to clipboardExpand all lines: .github/workflows/ci.yml
+2-1Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ jobs:
9494
strategy:
9595
fail-fast: false
9696
matrix:
97-
python-version: [3.8]
97+
python-version: [3.9]
9898
pydantic-version: ["pydantic-v2", "pydantic-v1"]
9999
test-path: [tests/integrations, tests/units, tests/documentation]
100100
steps:
@@ -112,6 +112,7 @@ jobs:
112112
./scripts/install_pydantic_v2.sh ${{ matrix.pydantic-version }}
113113
poetry run pip uninstall -y torch
114114
poetry run pip install torch
115+
poetry run pip install numpy==1.26.1
115116
sudo apt-get update
116117
sudo apt-get install --no-install-recommends ffmpeg
117118

‎docarray/base_doc/doc.py

Copy file name to clipboardExpand all lines: docarray/base_doc/doc.py
+5-5Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import typing_extensions
2323
from pydantic import BaseModel, Field
2424
from pydantic.fields import FieldInfo
25-
from typing_inspect import is_optional_type
25+
from typing_inspect import get_args, is_optional_type
2626

2727
from docarray.utils._internal.pydantic import is_pydantic_v2
2828

@@ -185,7 +185,7 @@ def _get_field_annotation(cls, field: str) -> Type:
185185
if is_optional_type(
186186
annotation
187187
): # this is equivalent to `outer_type_` in pydantic v1
188-
return annotation.__args__[0]
188+
return get_args(annotation)[0]
189189
else:
190190
return annotation
191191
else:
@@ -205,12 +205,12 @@ def _get_field_inner_type(cls, field: str) -> Type:
205205
if is_optional_type(
206206
annotation
207207
): # this is equivalent to `outer_type_` in pydantic v1
208-
return annotation.__args__[0]
208+
return get_args(annotation)[0]
209209
elif annotation == Tuple:
210-
if len(annotation.__args__) == 0:
210+
if len(get_args(annotation)) == 0:
211211
return Any
212212
else:
213-
annotation.__args__[0]
213+
get_args(annotation)[0]
214214
else:
215215
return annotation
216216
else:

‎docarray/base_doc/mixins/io.py

Copy file name to clipboardExpand all lines: docarray/base_doc/mixins/io.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,7 @@ def _get_content_from_node_proto(
336336
field_type = None
337337

338338
if isinstance(field_type, GenericAlias):
339-
field_type = field_type.__args__[0]
339+
field_type = get_args(field_type)[0]
340340

341341
return_field = arg_to_container[content_key](
342342
cls._get_content_from_node_proto(node, field_type=field_type)

‎docarray/display/document_summary.py

Copy file name to clipboardExpand all lines: docarray/display/document_summary.py
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from typing import Any, List, Optional, Type, Union
1+
from typing import Any, List, Optional, Type, Union, get_args
22

33
from rich.highlighter import RegexHighlighter
44
from rich.theme import Theme
@@ -83,7 +83,7 @@ def _get_schema(
8383

8484
if is_union_type(field_type) or is_optional_type(field_type):
8585
sub_tree = Tree(node_name, highlight=True)
86-
for arg in field_type.__args__:
86+
for arg in get_args(field_type):
8787
if safe_issubclass(arg, BaseDoc):
8888
sub_tree.add(
8989
DocumentSummary._get_schema(

‎docarray/helper.py

Copy file name to clipboardExpand all lines: docarray/helper.py
+46-1Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,23 @@
1515
Union,
1616
)
1717

18+
import numpy as np
19+
1820
from docarray.utils._internal._typing import safe_issubclass
21+
from docarray.utils._internal.misc import (
22+
is_jax_available,
23+
is_tf_available,
24+
is_torch_available,
25+
)
26+
27+
if is_torch_available():
28+
import torch
29+
30+
if is_jax_available():
31+
import jax
32+
33+
if is_tf_available():
34+
import tensorflow as tf
1935

2036
if TYPE_CHECKING:
2137
from docarray import BaseDoc
@@ -54,6 +70,35 @@ def _access_path_to_dict(access_path: str, value) -> Dict[str, Any]:
5470
return result
5571

5672

73+
def _is_none_like(val: Any) -> bool:
74+
"""
75+
:param val: any value
76+
:return: true iff `val` equals to `None`, `'None'` or `''`
77+
"""
78+
# Convoluted implementation, but fixes https://github.com/docarray/docarray/issues/1821
79+
80+
# tensor-like types can have unexpected (= broadcast) `==`/`in` semantics,
81+
# so treat separately
82+
is_np_arr = isinstance(val, np.ndarray)
83+
if is_np_arr:
84+
return False
85+
86+
is_torch_tens = is_torch_available() and isinstance(val, torch.Tensor)
87+
if is_torch_tens:
88+
return False
89+
90+
is_tf_tens = is_tf_available() and isinstance(val, tf.Tensor)
91+
if is_tf_tens:
92+
return False
93+
94+
is_jax_arr = is_jax_available() and isinstance(val, jax.numpy.ndarray)
95+
if is_jax_arr:
96+
return False
97+
98+
# "normal" case
99+
return val in ['', 'None', None]
100+
101+
57102
def _access_path_dict_to_nested_dict(access_path2val: Dict[str, Any]) -> Dict[Any, Any]:
58103
"""
59104
Convert a dict, where the keys are access paths ("__"-separated) to a nested dictionary.
@@ -76,7 +121,7 @@ def _access_path_dict_to_nested_dict(access_path2val: Dict[str, Any]) -> Dict[An
76121
for access_path, value in access_path2val.items():
77122
field2val = _access_path_to_dict(
78123
access_path=access_path,
79-
value=value if value not in ['', 'None'] else None,
124+
value=None if _is_none_like(value) else value,
80125
)
81126
_update_nested_dicts(to_update=nested_dict, update_with=field2val)
82127
return nested_dict

‎tests/units/array/test_array_from_to_pandas.py

Copy file name to clipboardExpand all lines: tests/units/array/test_array_from_to_pandas.py
+8-3Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,8 @@ class BasisUnion(BaseDoc):
136136

137137

138138
@pytest.mark.parametrize('tensor_type', [NdArray, TorchTensor])
139-
def test_from_to_pandas_tensor_type(tensor_type):
139+
@pytest.mark.parametrize('tensor_len', [0, 5])
140+
def test_from_to_pandas_tensor_type(tensor_type, tensor_len):
140141
class MyDoc(BaseDoc):
141142
embedding: tensor_type
142143
text: str
@@ -145,9 +146,13 @@ class MyDoc(BaseDoc):
145146
da = DocVec[MyDoc](
146147
[
147148
MyDoc(
148-
embedding=[1, 2, 3, 4, 5], text='hello', image=ImageDoc(url='aux.png')
149+
embedding=list(range(tensor_len)),
150+
text='hello',
151+
image=ImageDoc(url='aux.png'),
152+
),
153+
MyDoc(
154+
embedding=list(range(tensor_len)), text='hello world', image=ImageDoc()
149155
),
150-
MyDoc(embedding=[5, 4, 3, 2, 1], text='hello world', image=ImageDoc()),
151156
],
152157
tensor_type=tensor_type,
153158
)

‎tests/units/typing/tensor/test_ndarray.py

Copy file name to clipboardExpand all lines: tests/units/typing/tensor/test_ndarray.py
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -200,9 +200,9 @@ def test_parametrized_instance():
200200
def test_parametrized_equality():
201201
t1 = parse_obj_as(NdArray[128], np.zeros(128))
202202
t2 = parse_obj_as(NdArray[128], np.zeros(128))
203-
t3 = parse_obj_as(NdArray[256], np.zeros(256))
203+
t3 = parse_obj_as(NdArray[128], np.ones(128))
204204
assert (t1 == t2).all()
205-
assert not t1 == t3
205+
assert not (t1 == t3).any()
206206

207207

208208
def test_parametrized_operations():

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.