Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit e42bea5

Browse filesBrowse files
charrischarris
committed
Merge pull request #5943 from ahaldane/record_finalize
BUG: automatically convert recarray dtype to np.record
2 parents d033b6e + a93b862 commit e42bea5
Copy full SHA for e42bea5

File tree

7 files changed

+132
-72
lines changed
Filter options

7 files changed

+132
-72
lines changed

‎doc/release/1.10.0-notes.rst

Copy file name to clipboardExpand all lines: doc/release/1.10.0-notes.rst
+7Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,13 @@ Notably, this affect recarrays containing strings with whitespace, as trailing
8585
whitespace is trimmed from chararrays but kept in ndarrays of string type.
8686
Also, the dtype.type of nested structured fields is now inherited.
8787

88+
recarray views
89+
~~~~~~~~~~~~~~
90+
Viewing an ndarray as a recarray now automatically converts the dtype to
91+
np.record. See new record array documentation. Additionally, viewing a recarray
92+
with a non-structured dtype no longer converts the result's type to ndarray -
93+
the result will remain a recarray.
94+
8895
'out' keyword argument of ufuncs now accepts tuples of arrays
8996
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
9097
When using the 'out' keyword argument of a ufunc, a tuple of arrays, one per

‎doc/source/reference/arrays.dtypes.rst

Copy file name to clipboardExpand all lines: doc/source/reference/arrays.dtypes.rst
+10-8Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -424,16 +424,18 @@ Type strings
424424

425425
``(base_dtype, new_dtype)``
426426

427-
This usage is discouraged. In NumPy 1.7 and later, it is possible
428-
to specify struct dtypes with overlapping fields, functioning like
429-
the 'union' type in C. The union mechanism is preferred.
430-
431-
Both arguments must be convertible to data-type objects in this
432-
case. The *base_dtype* is the data-type object that the new
433-
data-type builds on. This is how you could assign named fields to
434-
any built-in data-type object, as done in
427+
In NumPy 1.7 and later, this form allows `base_dtype` to be interpreted as
428+
a structured dtype. Arrays created with this dtype will have underlying
429+
dtype `base_dtype` but will have fields and flags taken from `new_dtype`.
430+
This is useful for creating custom structured dtypes, as done in
435431
:ref:`record arrays <arrays.classes.rec>`.
436432

433+
This form also makes it possible to specify struct dtypes with overlapping
434+
fields, functioning like the 'union' type in C. This usage is discouraged,
435+
however, and the union mechanism is preferred.
436+
437+
Both arguments must be convertible to data-type objects with the same total
438+
size.
437439
.. admonition:: Example
438440

439441
32-bit integer, whose first two bytes are interpreted as an integer

‎numpy/core/records.py

Copy file name to clipboardExpand all lines: numpy/core/records.py
+18-26Lines changed: 18 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,12 @@ def __new__(subtype, shape, dtype=None, buf=None, offset=0, strides=None,
423423
strides=strides, order=order)
424424
return self
425425

426+
def __array_finalize__(self, obj):
427+
if self.dtype.type is not record:
428+
# if self.dtype is not np.record, invoke __setattr__ which will
429+
# convert it to a record if it is a void dtype.
430+
self.dtype = self.dtype
431+
426432
def __getattribute__(self, attr):
427433
# See if ndarray has this attr, and return it if so. (note that this
428434
# means a field with the same name as an ndarray attr cannot be
@@ -456,6 +462,11 @@ def __getattribute__(self, attr):
456462
# Undo any "setting" of the attribute and do a setfield
457463
# Thus, you can't create attributes on-the-fly that are field names.
458464
def __setattr__(self, attr, val):
465+
466+
# Automatically convert (void) dtypes to records.
467+
if attr == 'dtype' and issubclass(val.type, nt.void):
468+
val = sb.dtype((record, val))
469+
459470
newattr = attr not in self.__dict__
460471
try:
461472
ret = object.__setattr__(self, attr, val)
@@ -502,8 +513,10 @@ def __repr__(self):
502513
# show zero-length shape unless it is (0,)
503514
lst = "[], shape=%s" % (repr(self.shape),)
504515

505-
if self.dtype.type is record:
516+
if (self.dtype.type is record
517+
or (not issubclass(self.dtype.type, nt.void)) ):
506518
# If this is a full record array (has numpy.record dtype),
519+
# or if it has a scalar (non-void) dtype with no records,
507520
# represent it using the rec.array function. Since rec.array
508521
# converts dtype to a numpy.record for us, use only dtype.descr,
509522
# not repr(dtype).
@@ -512,7 +525,8 @@ def __repr__(self):
512525
(lst, lf, repr(self.dtype.descr)))
513526
else:
514527
# otherwise represent it using np.array plus a view
515-
# (There is currently (v1.10) no other easy way to create it)
528+
# This should only happen if the user is playing
529+
# strange games with dtypes.
516530
lf = '\n'+' '*len("array(")
517531
return ('array(%s, %sdtype=%s).view(numpy.recarray)' %
518532
(lst, lf, str(self.dtype)))
@@ -534,22 +548,6 @@ def field(self, attr, val=None):
534548
else:
535549
return self.setfield(val, *res)
536550

537-
def view(self, dtype=None, type=None):
538-
if dtype is None:
539-
return ndarray.view(self, type)
540-
elif type is None:
541-
try:
542-
if issubclass(dtype, ndarray):
543-
return ndarray.view(self, dtype)
544-
except TypeError:
545-
pass
546-
dtype = sb.dtype(dtype)
547-
if dtype.fields is None:
548-
return self.__array__().view(dtype)
549-
return ndarray.view(self, dtype)
550-
else:
551-
return ndarray.view(self, dtype, type)
552-
553551

554552
def fromarrays(arrayList, dtype=None, shape=None, formats=None,
555553
names=None, titles=None, aligned=False, byteorder=None):
@@ -837,10 +835,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
837835
new = obj
838836
if copy:
839837
new = new.copy()
840-
res = new.view(recarray)
841-
if issubclass(res.dtype.type, nt.void):
842-
res.dtype = sb.dtype((record, res.dtype))
843-
return res
838+
return new.view(recarray)
844839

845840
else:
846841
interface = getattr(obj, "__array_interface__", None)
@@ -849,7 +844,4 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
849844
obj = sb.array(obj)
850845
if dtype is not None and (obj.dtype != dtype):
851846
obj = obj.view(dtype)
852-
res = obj.view(recarray)
853-
if issubclass(res.dtype.type, nt.void):
854-
res.dtype = sb.dtype((record, res.dtype))
855-
return res
847+
return obj.view(recarray)

‎numpy/core/tests/test_records.py

Copy file name to clipboardExpand all lines: numpy/core/tests/test_records.py
+38-1Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,50 @@ def test_recarray_from_repr(self):
8888
assert_equal(recordarr, recordarr_r)
8989

9090
assert_equal(type(recarr_r), np.recarray)
91-
assert_equal(recarr_r.dtype.type, np.void)
91+
assert_equal(recarr_r.dtype.type, np.record)
9292
assert_equal(recarr, recarr_r)
9393

9494
assert_equal(type(recordview_r), np.ndarray)
9595
assert_equal(recordview.dtype.type, np.record)
9696
assert_equal(recordview, recordview_r)
9797

98+
def test_recarray_views(self):
99+
a = np.array([(1,'ABC'), (2, "DEF")],
100+
dtype=[('foo', int), ('bar', 'S4')])
101+
b = np.array([1,2,3,4,5], dtype=np.int64)
102+
103+
#check that np.rec.array gives right dtypes
104+
assert_equal(np.rec.array(a).dtype.type, np.record)
105+
assert_equal(type(np.rec.array(a)), np.recarray)
106+
assert_equal(np.rec.array(b).dtype.type, np.int64)
107+
assert_equal(type(np.rec.array(b)), np.recarray)
108+
109+
#check that viewing as recarray does the same
110+
assert_equal(a.view(np.recarray).dtype.type, np.record)
111+
assert_equal(type(a.view(np.recarray)), np.recarray)
112+
assert_equal(b.view(np.recarray).dtype.type, np.int64)
113+
assert_equal(type(b.view(np.recarray)), np.recarray)
114+
115+
#check that view to non-structured dtype preserves type=np.recarray
116+
r = np.rec.array(np.ones(4, dtype="f4,i4"))
117+
rv = r.view('f8').view('f4,i4')
118+
assert_equal(type(rv), np.recarray)
119+
assert_equal(rv.dtype.type, np.record)
120+
121+
#check that we can undo the view
122+
arrs = [np.ones(4, dtype='f4,i4'), np.ones(4, dtype='f8')]
123+
for arr in arrs:
124+
rec = np.rec.array(arr)
125+
# recommended way to view as an ndarray:
126+
arr2 = rec.view(rec.dtype.fields or rec.dtype, np.ndarray)
127+
assert_equal(arr2.dtype.type, arr.dtype.type)
128+
assert_equal(type(arr2), type(arr))
129+
130+
def test_recarray_repr(self):
131+
# make sure non-structured dtypes also show up as rec.array
132+
a = np.array(np.ones(4, dtype='f8'))
133+
assert_(repr(np.rec.array(a)).startswith('rec.array'))
134+
98135
def test_recarray_from_names(self):
99136
ra = np.rec.array([
100137
(1, 'abc', 3.7000002861022949, 0),

‎numpy/doc/structured_arrays.py

Copy file name to clipboardExpand all lines: numpy/doc/structured_arrays.py
+13-27Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,19 @@
258258
>>> recordarr = arr.view(dtype=dtype((np.record, arr.dtype)),
259259
... type=np.recarray)
260260
261+
For convenience, viewing an ndarray as type `np.recarray` will automatically
262+
convert to `np.record` datatype, so the dtype can be left out of the view: ::
263+
264+
>>> recordarr = arr.view(np.recarray)
265+
>>> recordarr.dtype
266+
dtype((numpy.record, [('foo', '<i4'), ('bar', '<f4'), ('baz', 'S10')]))
267+
268+
To get back to a plain ndarray both the dtype and type must be reset. The
269+
following view does so, taking into account the unusual case that the
270+
recordarr was not a structured type: ::
271+
272+
>>> arr2 = recordarr.view(recordarr.dtype.fields or recordarr.dtype, np.ndarray)
273+
261274
Record array fields accessed by index or by attribute are returned as a record
262275
array if the field has a structured type but as a plain ndarray otherwise. ::
263276
@@ -272,33 +285,6 @@
272285
attribute takes precedence. Such fields will be inaccessible by attribute but
273286
may still be accessed by index.
274287
275-
Partial Attribute Access
276-
------------------------
277-
278-
The differences between record arrays and plain structured arrays induce a
279-
small performance penalty. It is possible to apply one or the other view
280-
independently if desired. To allow field access by attribute only on the array
281-
object it is sufficient to view an array as a recarray: ::
282-
283-
>>> recarr = arr.view(np.recarray)
284-
285-
This type of view is commonly used, for example in np.npyio and
286-
np.recfunctions. Note that unlike full record arrays the individual elements of
287-
such a view do not have field attributes::
288-
289-
>>> recarr[0].foo
290-
AttributeError: 'numpy.void' object has no attribute 'foo'
291-
292-
To use the np.record dtype only, convert the dtype using the (base_class,
293-
dtype) form described in numpy.dtype. This type of view is rarely used. ::
294-
295-
>>> arr_records = arr.view(dtype((np.record, arr.dtype)))
296-
297-
In documentation, the term 'structured array' will refer to objects of type
298-
np.ndarray with structured dtype, 'record array' will refer to structured
299-
arrays subclassed as np.recarray and whose dtype is of type np.record, and
300-
'recarray' will refer to arrays subclassed as np.recarray but whose dtype is
301-
not of type np.record.
302288
303289
"""
304290
from __future__ import division, absolute_import, print_function

‎numpy/ma/core.py

Copy file name to clipboardExpand all lines: numpy/ma/core.py
+20-10Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2966,17 +2966,13 @@ def view(self, dtype=None, type=None, fill_value=None):
29662966
output = ndarray.view(self, dtype)
29672967
else:
29682968
output = ndarray.view(self, dtype, type)
2969-
# Should we update the mask ?
2969+
2970+
# also make the mask be a view (so attr changes to the view's
2971+
# mask do no affect original object's mask)
2972+
# (especially important to avoid affecting np.masked singleton)
29702973
if (getattr(output, '_mask', nomask) is not nomask):
2971-
if dtype is None:
2972-
dtype = output.dtype
2973-
mdtype = make_mask_descr(dtype)
2974-
output._mask = self._mask.view(mdtype, ndarray)
2975-
# Try to reset the shape of the mask (if we don't have a void)
2976-
try:
2977-
output._mask.shape = output.shape
2978-
except (AttributeError, TypeError):
2979-
pass
2974+
output._mask = output._mask.view()
2975+
29802976
# Make sure to reset the _fill_value if needed
29812977
if getattr(output, '_fill_value', None) is not None:
29822978
if fill_value is None:
@@ -3157,6 +3153,17 @@ def __setitem__(self, indx, value):
31573153
_mask[indx] = mindx
31583154
return
31593155

3156+
def __setattr__(self, attr, value):
3157+
super(MaskedArray, self).__setattr__(attr, value)
3158+
if attr == 'dtype' and self._mask is not nomask:
3159+
self._mask = self._mask.view(make_mask_descr(value), ndarray)
3160+
3161+
# Try to reset the shape of the mask (if we don't have a void)
3162+
# This raises a ValueError if the dtype change won't work
3163+
try:
3164+
self._mask.shape = self.shape
3165+
except (AttributeError, TypeError):
3166+
pass
31603167

31613168
def __getslice__(self, i, j):
31623169
"""x.__getslice__(i, j) <==> x[i:j]
@@ -4906,6 +4913,9 @@ def anom(self, axis=None, dtype=None):
49064913
49074914
"""
49084915
m = self.mean(axis, dtype)
4916+
if m is masked:
4917+
return m
4918+
49094919
if not axis:
49104920
return (self - m)
49114921
else:

‎numpy/ma/tests/test_core.py

Copy file name to clipboardExpand all lines: numpy/ma/tests/test_core.py
+26Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1401,6 +1401,32 @@ def test_flat(self):
14011401
assert_equal(b01.data, array([[1., 0.]]))
14021402
assert_equal(b01.mask, array([[False, False]]))
14031403

1404+
def test_assign_dtype(self):
1405+
# check that the mask's dtype is updated when dtype is changed
1406+
a = np.zeros(4, dtype='f4,i4')
1407+
1408+
m = np.ma.array(a)
1409+
m.dtype = np.dtype('f4')
1410+
repr(m) # raises?
1411+
assert_equal(m.dtype, np.dtype('f4'))
1412+
1413+
# check that dtype changes that change shape of mask too much
1414+
# are not allowed
1415+
def assign():
1416+
m = np.ma.array(a)
1417+
m.dtype = np.dtype('f8')
1418+
assert_raises(ValueError, assign)
1419+
1420+
b = a.view(dtype='f4', type=np.ma.MaskedArray) # raises?
1421+
assert_equal(b.dtype, np.dtype('f4'))
1422+
1423+
# check that nomask is preserved
1424+
a = np.zeros(4, dtype='f4')
1425+
m = np.ma.array(a)
1426+
m.dtype = np.dtype('f4,i4')
1427+
assert_equal(m.dtype, np.dtype('f4,i4'))
1428+
assert_equal(m._mask, np.ma.nomask)
1429+
14041430

14051431
#------------------------------------------------------------------------------
14061432
class TestFillingValues(TestCase):

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.