From a703ff341df516eff831d0fea87f617ccae086ff Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 27 Mar 2024 14:04:08 -0600 Subject: [PATCH 1/7] BUG: introduce PyArray_ViewableTypes to fix issues around stringdtype views --- numpy/_core/_internal.py | 2 +- numpy/_core/src/multiarray/ctors.c | 2 +- numpy/_core/src/multiarray/multiarraymodule.c | 22 ++++++++++++++++++- numpy/_core/src/multiarray/multiarraymodule.h | 3 +++ numpy/_core/tests/test_stringdtype.py | 17 ++++++++++++++ 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/numpy/_core/_internal.py b/numpy/_core/_internal.py index 8d6dc04851b5..058e93644dec 100644 --- a/numpy/_core/_internal.py +++ b/numpy/_core/_internal.py @@ -560,7 +560,7 @@ def _view_is_safe(oldtype, newtype): return if newtype.hasobject or oldtype.hasobject: - raise TypeError("Cannot change data-type for object array.") + raise TypeError("Cannot change data-type for array of references.") return diff --git a/numpy/_core/src/multiarray/ctors.c b/numpy/_core/src/multiarray/ctors.c index 4c9d76991296..322244e60c1e 100644 --- a/numpy/_core/src/multiarray/ctors.c +++ b/numpy/_core/src/multiarray/ctors.c @@ -1922,7 +1922,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) /* If a writeable array was requested, and arr is not */ ((flags & NPY_ARRAY_WRITEABLE) && (!(arrflags & NPY_ARRAY_WRITEABLE))) || - !PyArray_EquivTypes(oldtype, newtype); + !PyArray_ViewableTypes(oldtype, newtype); if (copy) { if (flags & NPY_ARRAY_ENSURENOCOPY ) { diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index 3c153adb83a8..f559c13d9811 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -1481,6 +1481,26 @@ PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) } +/* + * This function returns true if a view can be safely created + * between the two types. This implies that PyArray_EquivTypes + * is true as well. + */ +NPY_NO_EXPORT unsigned char +PyArray_ViewableTypes(PyArray_Descr *type1, PyArray_Descr *type2) +{ + if (!PyArray_EquivTypes(type1, type2)) { + return 0; + } + // a view of a stringdtype array has a corrupt arena, unless + // type1 and type2 are exactly the same object + if ((type1 != type2) && (type1->kind == 'T')) { + return 0; + } + return 1; +} + + /*NUMPY_API*/ NPY_NO_EXPORT unsigned char PyArray_EquivTypenums(int typenum1, int typenum2) @@ -1609,7 +1629,7 @@ _array_fromobject_generic( /* One more chance for faster exit if user specified the dtype. */ oldtype = PyArray_DESCR(oparr); - if (PyArray_EquivTypes(oldtype, dtype)) { + if (PyArray_ViewableTypes(oldtype, dtype)) { if (copy != NPY_COPY_ALWAYS && STRIDING_OK(oparr, order)) { if (oldtype == dtype) { Py_INCREF(op); diff --git a/numpy/_core/src/multiarray/multiarraymodule.h b/numpy/_core/src/multiarray/multiarraymodule.h index ba03d367eeb8..7e00e46ea2e3 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.h +++ b/numpy/_core/src/multiarray/multiarraymodule.h @@ -21,4 +21,7 @@ NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_convert_if_no_array; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_cpu; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_err_msg_substr; +NPY_NO_EXPORT unsigned char +PyArray_ViewableTypes(PyArray_Descr *type1, PyArray_Descr *type2); + #endif /* NUMPY_CORE_SRC_MULTIARRAY_MULTIARRAYMODULE_H_ */ diff --git a/numpy/_core/tests/test_stringdtype.py b/numpy/_core/tests/test_stringdtype.py index e38575dbc0dd..8d72df6401f3 100644 --- a/numpy/_core/tests/test_stringdtype.py +++ b/numpy/_core/tests/test_stringdtype.py @@ -458,6 +458,23 @@ def test_creation_functions(): assert np.empty(3, dtype="T")[0] == "" +def test_create_with_copy_none(string_list): + arr = np.array(string_list, dtype=StringDType()) + arr_rev = np.array(string_list[::-1], dtype=StringDType()) + + arr_copy = np.array(arr, copy=None, dtype=arr_rev.dtype) + np.testing.assert_array_equal(arr, arr_copy) + assert arr_copy.base is None + + with pytest.raises(ValueError, match="Unable to avoid copy"): + np.array(arr, copy=False, dtype=arr_rev.dtype) + + arr_view = np.array(arr, copy=None, dtype=arr.dtype) + np.testing.assert_array_equal(arr, arr) + np.testing.assert_array_equal(arr_view[::-1], arr_rev) + assert arr_view is arr + + @pytest.mark.parametrize( "strings", [ From ae4be90ae2383ed12c58609a428e2f3c7d63000a Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 27 Mar 2024 19:29:33 -0600 Subject: [PATCH 2/7] MNT: make PyArray_ViewableTypes use view_offset from casting setup --- numpy/_core/src/multiarray/convert_datatype.c | 5 ++- numpy/_core/src/multiarray/ctors.c | 8 +++- numpy/_core/src/multiarray/multiarraymodule.c | 39 ++++++++++++++++--- .../_core/src/multiarray/stringdtype/casts.c | 3 +- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/numpy/_core/src/multiarray/convert_datatype.c b/numpy/_core/src/multiarray/convert_datatype.c index 92e6964fc21f..5f0a8138c10e 100644 --- a/numpy/_core/src/multiarray/convert_datatype.c +++ b/numpy/_core/src/multiarray/convert_datatype.c @@ -466,6 +466,7 @@ _get_cast_safety_from_castingimpl(PyArrayMethodObject *castingimpl, * Check for less harmful non-standard returns. The following two returns * should never happen: * 1. No-casting must imply a view offset of 0. + * (with an exception for stringdtype) * 2. Equivalent-casting + 0 view offset is (usually) the definition * of a "no" cast. However, changing the order of fields can also * create descriptors that are not equivalent but views. @@ -473,7 +474,9 @@ _get_cast_safety_from_castingimpl(PyArrayMethodObject *castingimpl, * principle, casting `kind != 'T' || to->kind != 'T') { + assert(casting != NPY_NO_CASTING); + } } else { assert(casting != NPY_EQUIV_CASTING diff --git a/numpy/_core/src/multiarray/ctors.c b/numpy/_core/src/multiarray/ctors.c index 322244e60c1e..2c8373a25aca 100644 --- a/numpy/_core/src/multiarray/ctors.c +++ b/numpy/_core/src/multiarray/ctors.c @@ -1908,6 +1908,12 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) } arrflags = PyArray_FLAGS(arr); + unsigned char viewable = PyArray_ViewableTypes(oldtype, newtype); + if (viewable < 0) { + Py_DECREF(newtype); + return NULL; + } + /* If a guaranteed copy was requested */ copy = (flags & NPY_ARRAY_ENSURECOPY) || /* If C contiguous was requested, and arr is not */ @@ -1922,7 +1928,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) /* If a writeable array was requested, and arr is not */ ((flags & NPY_ARRAY_WRITEABLE) && (!(arrflags & NPY_ARRAY_WRITEABLE))) || - !PyArray_ViewableTypes(oldtype, newtype); + !viewable; if (copy) { if (flags & NPY_ARRAY_ENSURENOCOPY ) { diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index f559c13d9811..9c1157cfb87b 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -1489,15 +1489,38 @@ PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) NPY_NO_EXPORT unsigned char PyArray_ViewableTypes(PyArray_Descr *type1, PyArray_Descr *type2) { - if (!PyArray_EquivTypes(type1, type2)) { + if (type1 == type2) { + return 1; + } + + if (Py_TYPE(Py_TYPE(type1)) == &PyType_Type) { + /* + * 2021-12-17: This case is nonsense and should be removed eventually! + * + * boost::python has/had a bug effectively using EquivTypes with + * `type(arbitrary_obj)`. That is clearly wrong as that cannot be a + * `PyArray_Descr *`. We assume that `type(type(type(arbitrary_obj))` + * is always in practice `type` (this is the type of the metaclass), + * but for our descriptors, `type(type(descr))` is DTypeMeta. + * + * In that case, we just return False. There is a possibility that + * this actually _worked_ effectively (returning 1 sometimes). + * We ignore that possibility for simplicity; it really is not our bug. + */ return 0; } - // a view of a stringdtype array has a corrupt arena, unless - // type1 and type2 are exactly the same object - if ((type1 != type2) && (type1->kind == 'T')) { + + npy_intp view_offset; + NPY_CASTING safety = PyArray_GetCastInfo(type1, type2, NULL, &view_offset); + if (safety < 0) { + PyErr_Clear(); return 0; } - return 1; + if (view_offset != 0) { + return 0; + } + /* If casting is "no casting" this dtypes are considered equivalent. */ + return PyArray_MinCastSafety(safety, NPY_NO_CASTING) == NPY_NO_CASTING; } @@ -1629,7 +1652,11 @@ _array_fromobject_generic( /* One more chance for faster exit if user specified the dtype. */ oldtype = PyArray_DESCR(oparr); - if (PyArray_ViewableTypes(oldtype, dtype)) { + unsigned char viewable = PyArray_ViewableTypes(oldtype, dtype); + if (viewable < 0) { + goto finish; + } + if (viewable) { if (copy != NPY_COPY_ALWAYS && STRIDING_OK(oparr, order)) { if (oldtype == dtype) { Py_INCREF(op); diff --git a/numpy/_core/src/multiarray/stringdtype/casts.c b/numpy/_core/src/multiarray/stringdtype/casts.c index e3e7c5fde872..52e76b16c4f7 100644 --- a/numpy/_core/src/multiarray/stringdtype/casts.c +++ b/numpy/_core/src/multiarray/stringdtype/casts.c @@ -79,7 +79,8 @@ string_to_string_resolve_descriptors(PyObject *NPY_UNUSED(self), return NPY_UNSAFE_CASTING; } - *view_offset = 0; + // views are only legal between descriptors that share allocators (e.g. the same object) + *view_offset = descr0->allocator != descr1->allocator; return NPY_NO_CASTING; } From b7f6de962a26a17fe482901711f5e631dac49d8e Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 28 Mar 2024 13:59:55 -0600 Subject: [PATCH 3/7] MNT: Only assert if finalize_descr isn't defined --- numpy/_core/src/multiarray/convert_datatype.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/numpy/_core/src/multiarray/convert_datatype.c b/numpy/_core/src/multiarray/convert_datatype.c index 5f0a8138c10e..c35f899847a5 100644 --- a/numpy/_core/src/multiarray/convert_datatype.c +++ b/numpy/_core/src/multiarray/convert_datatype.c @@ -465,18 +465,18 @@ _get_cast_safety_from_castingimpl(PyArrayMethodObject *castingimpl, /* * Check for less harmful non-standard returns. The following two returns * should never happen: - * 1. No-casting must imply a view offset of 0. - * (with an exception for stringdtype) + * 1. No-casting must imply a view offset of 0 unless the DType + defines a finalization function, which implies it stores data + on the descriptor * 2. Equivalent-casting + 0 view offset is (usually) the definition * of a "no" cast. However, changing the order of fields can also * create descriptors that are not equivalent but views. * Note that unsafe casts can have a view offset. For example, in * principle, casting `kind != 'T' || to->kind != 'T') { - assert(casting != NPY_NO_CASTING); - } + if ((*view_offset != 0 && + NPY_DT_SLOTS(NPY_DTYPE(from))->finalize_descr == NULL)) { + assert(casting != NPY_NO_CASTING); } else { assert(casting != NPY_EQUIV_CASTING From a914726f3b68f4b0d5526a051c3ba6ab1b19f55b Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 29 Mar 2024 16:59:21 -0600 Subject: [PATCH 4/7] MAINT: refactor to pass casting safety in --- numpy/_core/src/multiarray/ctors.c | 10 ++- numpy/_core/src/multiarray/multiarraymodule.c | 72 +++++-------------- numpy/_core/src/multiarray/multiarraymodule.h | 6 +- .../_core/src/multiarray/stringdtype/casts.c | 4 +- numpy/_core/src/umath/ufunc_object.c | 11 +-- 5 files changed, 36 insertions(+), 67 deletions(-) diff --git a/numpy/_core/src/multiarray/ctors.c b/numpy/_core/src/multiarray/ctors.c index 2c8373a25aca..abe6a631ebbe 100644 --- a/numpy/_core/src/multiarray/ctors.c +++ b/numpy/_core/src/multiarray/ctors.c @@ -1908,11 +1908,9 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) } arrflags = PyArray_FLAGS(arr); - unsigned char viewable = PyArray_ViewableTypes(oldtype, newtype); - if (viewable < 0) { - Py_DECREF(newtype); - return NULL; - } + npy_intp view_offset; + npy_intp is_safe = PyArray_SafeCast(oldtype, newtype, &view_offset, NPY_NO_CASTING, 1); + npy_intp view_safe = (is_safe && (view_offset == 0)); /* If a guaranteed copy was requested */ copy = (flags & NPY_ARRAY_ENSURECOPY) || @@ -1928,7 +1926,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) /* If a writeable array was requested, and arr is not */ ((flags & NPY_ARRAY_WRITEABLE) && (!(arrflags & NPY_ARRAY_WRITEABLE))) || - !viewable; + !view_safe; if (copy) { if (flags & NPY_ARRAY_ENSURENOCOPY ) { diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index 9c1157cfb87b..dff272164398 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -1449,23 +1449,6 @@ PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) return 1; } - if (Py_TYPE(Py_TYPE(type1)) == &PyType_Type) { - /* - * 2021-12-17: This case is nonsense and should be removed eventually! - * - * boost::python has/had a bug effectively using EquivTypes with - * `type(arbitrary_obj)`. That is clearly wrong as that cannot be a - * `PyArray_Descr *`. We assume that `type(type(type(arbitrary_obj))` - * is always in practice `type` (this is the type of the metaclass), - * but for our descriptors, `type(type(descr))` is DTypeMeta. - * - * In that case, we just return False. There is a possibility that - * this actually _worked_ effectively (returning 1 sometimes). - * We ignore that possibility for simplicity; it really is not our bug. - */ - return 0; - } - /* * Do not use PyArray_CanCastTypeTo because it supports legacy flexible * dtypes as input. @@ -1482,45 +1465,29 @@ PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) /* - * This function returns true if a view can be safely created - * between the two types. This implies that PyArray_EquivTypes - * is true as well. + * This function returns true if the two types can be safely cast at + * *minimum_safety* casting level. Sets the view_offset if that is set + * for the cast. If ignore_error is 1, errors in cast setup are ignored. */ -NPY_NO_EXPORT unsigned char -PyArray_ViewableTypes(PyArray_Descr *type1, PyArray_Descr *type2) +NPY_NO_EXPORT npy_intp +PyArray_SafeCast(PyArray_Descr *type1, PyArray_Descr *type2, + npy_intp* view_offset, NPY_CASTING minimum_safety, + npy_intp ignore_error) { if (type1 == type2) { + *view_offset = 0; return 1; } - if (Py_TYPE(Py_TYPE(type1)) == &PyType_Type) { - /* - * 2021-12-17: This case is nonsense and should be removed eventually! - * - * boost::python has/had a bug effectively using EquivTypes with - * `type(arbitrary_obj)`. That is clearly wrong as that cannot be a - * `PyArray_Descr *`. We assume that `type(type(type(arbitrary_obj))` - * is always in practice `type` (this is the type of the metaclass), - * but for our descriptors, `type(type(descr))` is DTypeMeta. - * - * In that case, we just return False. There is a possibility that - * this actually _worked_ effectively (returning 1 sometimes). - * We ignore that possibility for simplicity; it really is not our bug. - */ - return 0; - } - - npy_intp view_offset; - NPY_CASTING safety = PyArray_GetCastInfo(type1, type2, NULL, &view_offset); + NPY_CASTING safety = PyArray_GetCastInfo(type1, type2, NULL, view_offset); if (safety < 0) { - PyErr_Clear(); - return 0; - } - if (view_offset != 0) { - return 0; + if (ignore_error) { + PyErr_Clear(); + return 0; + } + return -1; } - /* If casting is "no casting" this dtypes are considered equivalent. */ - return PyArray_MinCastSafety(safety, NPY_NO_CASTING) == NPY_NO_CASTING; + return PyArray_MinCastSafety(safety, minimum_safety) == minimum_safety; } @@ -1652,11 +1619,10 @@ _array_fromobject_generic( /* One more chance for faster exit if user specified the dtype. */ oldtype = PyArray_DESCR(oparr); - unsigned char viewable = PyArray_ViewableTypes(oldtype, dtype); - if (viewable < 0) { - goto finish; - } - if (viewable) { + npy_intp view_offset; + npy_intp is_safe = PyArray_SafeCast(oldtype, dtype, &view_offset, NPY_NO_CASTING, 1); + npy_intp view_safe = (is_safe && (view_offset == 0)); + if (view_safe) { if (copy != NPY_COPY_ALWAYS && STRIDING_OK(oparr, order)) { if (oldtype == dtype) { Py_INCREF(op); diff --git a/numpy/_core/src/multiarray/multiarraymodule.h b/numpy/_core/src/multiarray/multiarraymodule.h index 7e00e46ea2e3..4aac241523d7 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.h +++ b/numpy/_core/src/multiarray/multiarraymodule.h @@ -21,7 +21,9 @@ NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_convert_if_no_array; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_cpu; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_err_msg_substr; -NPY_NO_EXPORT unsigned char -PyArray_ViewableTypes(PyArray_Descr *type1, PyArray_Descr *type2); +NPY_NO_EXPORT npy_intp +PyArray_SafeCast(PyArray_Descr *type1, PyArray_Descr *type2, + npy_intp* view_offset, NPY_CASTING minimum_safety, + npy_intp ignore_errors); #endif /* NUMPY_CORE_SRC_MULTIARRAY_MULTIARRAYMODULE_H_ */ diff --git a/numpy/_core/src/multiarray/stringdtype/casts.c b/numpy/_core/src/multiarray/stringdtype/casts.c index 52e76b16c4f7..b896d09ace65 100644 --- a/numpy/_core/src/multiarray/stringdtype/casts.c +++ b/numpy/_core/src/multiarray/stringdtype/casts.c @@ -80,7 +80,9 @@ string_to_string_resolve_descriptors(PyObject *NPY_UNUSED(self), } // views are only legal between descriptors that share allocators (e.g. the same object) - *view_offset = descr0->allocator != descr1->allocator; + if (descr0->allocator == descr1->allocator) { + *view_offset = 0; + }; return NPY_NO_CASTING; } diff --git a/numpy/_core/src/umath/ufunc_object.c b/numpy/_core/src/umath/ufunc_object.c index 63ef0d9080ca..0fabc37b070f 100644 --- a/numpy/_core/src/umath/ufunc_object.c +++ b/numpy/_core/src/umath/ufunc_object.c @@ -62,6 +62,7 @@ #include "legacy_array_method.h" #include "abstractdtypes.h" #include "mapping.h" +#include "multiarraymodule.h" /* TODO: Only for `NpyIter_GetTransferFlags` until it is public */ #define NPY_ITERATOR_IMPLEMENTATION_CODE @@ -789,9 +790,9 @@ check_for_trivial_loop(PyArrayMethodObject *ufuncimpl, if (dtypes[i] != PyArray_DESCR(op[i])) { npy_intp view_offset; - NPY_CASTING safety = PyArray_GetCastInfo( - PyArray_DESCR(op[i]), dtypes[i], NULL, &view_offset); - if (safety < 0 && PyErr_Occurred()) { + npy_intp is_safe = PyArray_SafeCast( + PyArray_DESCR(op[i]), dtypes[i], &view_offset, casting, 0); + if (is_safe < 0 && PyErr_Occurred()) { /* A proper error during a cast check, should be rare */ return -1; } @@ -806,8 +807,8 @@ check_for_trivial_loop(PyArrayMethodObject *ufuncimpl, * can force cast to bool) */ } - else if (PyArray_MinCastSafety(safety, casting) != casting) { - return 0; /* the cast is not safe enough */ + else if (is_safe != 1) { + return 0; /* there was a cast error or cast is not safe enough */ } } if (must_copy) { From e16ad3b6f307522172cb6b513a430981d1854b05 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 1 Apr 2024 14:31:54 -0600 Subject: [PATCH 5/7] MNT: respond to review comments --- numpy/_core/src/multiarray/convert_datatype.c | 29 +++++++++++++++++++ numpy/_core/src/multiarray/convert_datatype.h | 5 ++++ numpy/_core/src/multiarray/ctors.c | 17 ++++++----- numpy/_core/src/multiarray/multiarraymodule.c | 27 ----------------- numpy/_core/src/multiarray/multiarraymodule.h | 5 ---- numpy/_core/src/umath/ufunc_object.c | 1 - numpy/_core/tests/test_stringdtype.py | 6 ++++ 7 files changed, 50 insertions(+), 40 deletions(-) diff --git a/numpy/_core/src/multiarray/convert_datatype.c b/numpy/_core/src/multiarray/convert_datatype.c index c35f899847a5..9765a4041e06 100644 --- a/numpy/_core/src/multiarray/convert_datatype.c +++ b/numpy/_core/src/multiarray/convert_datatype.c @@ -651,6 +651,35 @@ PyArray_CanCastTo(PyArray_Descr *from, PyArray_Descr *to) } +/* + * This function returns true if the two types can be safely cast at + * *minimum_safety* casting level. Sets the *view_offset* if that is set + * for the cast. If ignore_error is set, the error indicator is cleared + * if there are any errors in cast setup and returns false, otherwise + * the error indicator is left set and returns -1. + */ +NPY_NO_EXPORT npy_intp +PyArray_SafeCast(PyArray_Descr *type1, PyArray_Descr *type2, + npy_intp* view_offset, NPY_CASTING minimum_safety, + npy_intp ignore_error) +{ + if (type1 == type2) { + *view_offset = 0; + return 1; + } + + NPY_CASTING safety = PyArray_GetCastInfo(type1, type2, NULL, view_offset); + if (safety < 0) { + if (ignore_error) { + PyErr_Clear(); + return 0; + } + return -1; + } + return PyArray_MinCastSafety(safety, minimum_safety) == minimum_safety; +} + + /* Provides an ordering for the dtype 'kind' character codes */ NPY_NO_EXPORT int dtype_kind_to_ordering(char kind) diff --git a/numpy/_core/src/multiarray/convert_datatype.h b/numpy/_core/src/multiarray/convert_datatype.h index 8dccbab6f8c0..d1493e6997bd 100644 --- a/numpy/_core/src/multiarray/convert_datatype.h +++ b/numpy/_core/src/multiarray/convert_datatype.h @@ -102,6 +102,11 @@ PyArray_GetCastInfo( PyArray_Descr *from, PyArray_Descr *to, PyArray_DTypeMeta *to_dtype, npy_intp *view_offset); +NPY_NO_EXPORT npy_intp +PyArray_SafeCast(PyArray_Descr *type1, PyArray_Descr *type2, + npy_intp* view_offset, NPY_CASTING minimum_safety, + npy_intp ignore_errors); + NPY_NO_EXPORT int PyArray_CheckCastSafety(NPY_CASTING casting, PyArray_Descr *from, PyArray_Descr *to, PyArray_DTypeMeta *to_dtype); diff --git a/numpy/_core/src/multiarray/ctors.c b/numpy/_core/src/multiarray/ctors.c index abe6a631ebbe..f897afb5d4c0 100644 --- a/numpy/_core/src/multiarray/ctors.c +++ b/numpy/_core/src/multiarray/ctors.c @@ -1908,12 +1908,10 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) } arrflags = PyArray_FLAGS(arr); - npy_intp view_offset; - npy_intp is_safe = PyArray_SafeCast(oldtype, newtype, &view_offset, NPY_NO_CASTING, 1); - npy_intp view_safe = (is_safe && (view_offset == 0)); - /* If a guaranteed copy was requested */ - copy = (flags & NPY_ARRAY_ENSURECOPY) || + + copy = /* If a guaranteed copy was requested */ + (flags & NPY_ARRAY_ENSURECOPY) || /* If C contiguous was requested, and arr is not */ ((flags & NPY_ARRAY_C_CONTIGUOUS) && (!(arrflags & NPY_ARRAY_C_CONTIGUOUS))) || @@ -1925,8 +1923,13 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) (!(arrflags & NPY_ARRAY_F_CONTIGUOUS))) || /* If a writeable array was requested, and arr is not */ ((flags & NPY_ARRAY_WRITEABLE) && - (!(arrflags & NPY_ARRAY_WRITEABLE))) || - !view_safe; + (!(arrflags & NPY_ARRAY_WRITEABLE))); + + if (!copy) { + npy_intp view_offset; + npy_intp is_safe = PyArray_SafeCast(oldtype, newtype, &view_offset, NPY_NO_CASTING, 1); + copy = copy || !(is_safe && (view_offset == 0)); + } if (copy) { if (flags & NPY_ARRAY_ENSURENOCOPY ) { diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index dff272164398..0df294be14dc 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -1464,33 +1464,6 @@ PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) } -/* - * This function returns true if the two types can be safely cast at - * *minimum_safety* casting level. Sets the view_offset if that is set - * for the cast. If ignore_error is 1, errors in cast setup are ignored. - */ -NPY_NO_EXPORT npy_intp -PyArray_SafeCast(PyArray_Descr *type1, PyArray_Descr *type2, - npy_intp* view_offset, NPY_CASTING minimum_safety, - npy_intp ignore_error) -{ - if (type1 == type2) { - *view_offset = 0; - return 1; - } - - NPY_CASTING safety = PyArray_GetCastInfo(type1, type2, NULL, view_offset); - if (safety < 0) { - if (ignore_error) { - PyErr_Clear(); - return 0; - } - return -1; - } - return PyArray_MinCastSafety(safety, minimum_safety) == minimum_safety; -} - - /*NUMPY_API*/ NPY_NO_EXPORT unsigned char PyArray_EquivTypenums(int typenum1, int typenum2) diff --git a/numpy/_core/src/multiarray/multiarraymodule.h b/numpy/_core/src/multiarray/multiarraymodule.h index 4aac241523d7..ba03d367eeb8 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.h +++ b/numpy/_core/src/multiarray/multiarraymodule.h @@ -21,9 +21,4 @@ NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_convert_if_no_array; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_cpu; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_err_msg_substr; -NPY_NO_EXPORT npy_intp -PyArray_SafeCast(PyArray_Descr *type1, PyArray_Descr *type2, - npy_intp* view_offset, NPY_CASTING minimum_safety, - npy_intp ignore_errors); - #endif /* NUMPY_CORE_SRC_MULTIARRAY_MULTIARRAYMODULE_H_ */ diff --git a/numpy/_core/src/umath/ufunc_object.c b/numpy/_core/src/umath/ufunc_object.c index 0fabc37b070f..d1f4eb30b5a8 100644 --- a/numpy/_core/src/umath/ufunc_object.c +++ b/numpy/_core/src/umath/ufunc_object.c @@ -62,7 +62,6 @@ #include "legacy_array_method.h" #include "abstractdtypes.h" #include "mapping.h" -#include "multiarraymodule.h" /* TODO: Only for `NpyIter_GetTransferFlags` until it is public */ #define NPY_ITERATOR_IMPLEMENTATION_CODE diff --git a/numpy/_core/tests/test_stringdtype.py b/numpy/_core/tests/test_stringdtype.py index 8d72df6401f3..cc11ad237d47 100644 --- a/numpy/_core/tests/test_stringdtype.py +++ b/numpy/_core/tests/test_stringdtype.py @@ -460,8 +460,13 @@ def test_creation_functions(): def test_create_with_copy_none(string_list): arr = np.array(string_list, dtype=StringDType()) + # create another stringdtype array with an arena that has a different + # in-memory layout than the first array arr_rev = np.array(string_list[::-1], dtype=StringDType()) + # this should create a copy and the resulting array + # shouldn't share an allocator or arena with arr_rev, despite + # explicitly passing arr_rev.dtype arr_copy = np.array(arr, copy=None, dtype=arr_rev.dtype) np.testing.assert_array_equal(arr, arr_copy) assert arr_copy.base is None @@ -469,6 +474,7 @@ def test_create_with_copy_none(string_list): with pytest.raises(ValueError, match="Unable to avoid copy"): np.array(arr, copy=False, dtype=arr_rev.dtype) + # because we're using arr's dtype instance, the view is safe arr_view = np.array(arr, copy=None, dtype=arr.dtype) np.testing.assert_array_equal(arr, arr) np.testing.assert_array_equal(arr_view[::-1], arr_rev) From 54e3a83ff64cfe4691b7a2cd0156bd659cd1cb6d Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 1 Apr 2024 14:36:33 -0600 Subject: [PATCH 6/7] MNT: check for valid views by comparing with NPY_MIN_INTP --- numpy/_core/src/multiarray/ctors.c | 2 +- numpy/_core/src/multiarray/multiarraymodule.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/_core/src/multiarray/ctors.c b/numpy/_core/src/multiarray/ctors.c index f897afb5d4c0..3c0e11ddc749 100644 --- a/numpy/_core/src/multiarray/ctors.c +++ b/numpy/_core/src/multiarray/ctors.c @@ -1928,7 +1928,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) if (!copy) { npy_intp view_offset; npy_intp is_safe = PyArray_SafeCast(oldtype, newtype, &view_offset, NPY_NO_CASTING, 1); - copy = copy || !(is_safe && (view_offset == 0)); + copy = copy || !(is_safe && (view_offset != NPY_MIN_INTP)); } if (copy) { diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index 0df294be14dc..b8b777697fe3 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -1594,7 +1594,7 @@ _array_fromobject_generic( oldtype = PyArray_DESCR(oparr); npy_intp view_offset; npy_intp is_safe = PyArray_SafeCast(oldtype, dtype, &view_offset, NPY_NO_CASTING, 1); - npy_intp view_safe = (is_safe && (view_offset == 0)); + npy_intp view_safe = (is_safe && (view_offset != NPY_MIN_INTP)); if (view_safe) { if (copy != NPY_COPY_ALWAYS && STRIDING_OK(oparr, order)) { if (oldtype == dtype) { From 6373b850aa1baf676b14646e9cc76831eefee056 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 1 Apr 2024 15:18:24 -0600 Subject: [PATCH 7/7] MNT: tiny optimization --- numpy/_core/src/multiarray/ctors.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/_core/src/multiarray/ctors.c b/numpy/_core/src/multiarray/ctors.c index 3c0e11ddc749..836ed493e1e6 100644 --- a/numpy/_core/src/multiarray/ctors.c +++ b/numpy/_core/src/multiarray/ctors.c @@ -1928,7 +1928,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) if (!copy) { npy_intp view_offset; npy_intp is_safe = PyArray_SafeCast(oldtype, newtype, &view_offset, NPY_NO_CASTING, 1); - copy = copy || !(is_safe && (view_offset != NPY_MIN_INTP)); + copy = !(is_safe && (view_offset != NPY_MIN_INTP)); } if (copy) {