Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 397f1b2

Browse filesBrowse files
gpsheadgpshead
authored
bpo-33312: Fix clang ubsan out of bounds warnings in dict. (GH-6537)
Fix clang ubsan (undefined behavior sanitizer) warnings in dictobject.c by adjusting how the internal struct _dictkeysobject shared keys structure is declared. This remains ABI compatible. We get rid of the union at the end of the struct being used for conveinence to avoid typecasting in favor of char[] variable length array at the end of a struct. This is known to clang to be used for variable sized objects and will not cause an undefined behavior problem. Similarly, char arrays do not have strict aliasing undefined behavior when cast. PEP-007 does not currently list variable length arrays (VLAs) as allowed in our subset of C99. If this turns out to be a problem, the fix to this is to change the char `dk_indices[]` into `dk_indices[1]` and restore the three size computation subtractions this change removes: `- Py_MEMBER_SIZE(PyDictKeysObject, dk_indices)` If this works as is I'll make a separate PR to update PEP-007.
1 parent b87c1c9 commit 397f1b2
Copy full SHA for 397f1b2

File tree

Expand file treeCollapse file tree

3 files changed

+17
-24
lines changed
Filter options
Expand file treeCollapse file tree

3 files changed

+17
-24
lines changed

‎Misc/NEWS.d/next/Core and Builtins/2018-04-19-08-30-07.bpo-33312.mDe2iL.rst

Copy file name to clipboard
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fixed clang ubsan (undefined behavior sanitizer) warnings in dictobject.c by
2+
adjusting how the internal struct _dictkeysobject shared keys structure is
3+
declared.

‎Objects/dict-common.h

Copy file name to clipboardExpand all lines: Objects/dict-common.h
+2-9Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,15 +58,8 @@ struct _dictkeysobject {
5858
- 4 bytes if dk_size <= 0xffffffff (int32_t*)
5959
- 8 bytes otherwise (int64_t*)
6060
61-
Dynamically sized, 8 is minimum. */
62-
union {
63-
int8_t as_1[8];
64-
int16_t as_2[4];
65-
int32_t as_4[2];
66-
#if SIZEOF_VOID_P > 4
67-
int64_t as_8[1];
68-
#endif
69-
} dk_indices;
61+
Dynamically sized, SIZEOF_VOID_P is minimum. */
62+
char dk_indices[]; /* char is required to avoid strict aliasing. */
7063

7164
/* "PyDictKeyEntry dk_entries[dk_usable];" array follows:
7265
see the DK_ENTRIES() macro */

‎Objects/dictobject.c

Copy file name to clipboardExpand all lines: Objects/dictobject.c
+12-15Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@ PyDict_Fini(void)
298298
2 : sizeof(int32_t))
299299
#endif
300300
#define DK_ENTRIES(dk) \
301-
((PyDictKeyEntry*)(&(dk)->dk_indices.as_1[DK_SIZE(dk) * DK_IXSIZE(dk)]))
301+
((PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[DK_SIZE(dk) * DK_IXSIZE(dk)]))
302302

303303
#define DK_DEBUG_INCREF _Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA
304304
#define DK_DEBUG_DECREF _Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA
@@ -316,21 +316,21 @@ dk_get_index(PyDictKeysObject *keys, Py_ssize_t i)
316316
Py_ssize_t ix;
317317

318318
if (s <= 0xff) {
319-
int8_t *indices = keys->dk_indices.as_1;
319+
int8_t *indices = (int8_t*)(keys->dk_indices);
320320
ix = indices[i];
321321
}
322322
else if (s <= 0xffff) {
323-
int16_t *indices = keys->dk_indices.as_2;
323+
int16_t *indices = (int16_t*)(keys->dk_indices);
324324
ix = indices[i];
325325
}
326326
#if SIZEOF_VOID_P > 4
327327
else if (s > 0xffffffff) {
328-
int64_t *indices = keys->dk_indices.as_8;
328+
int64_t *indices = (int64_t*)(keys->dk_indices);
329329
ix = indices[i];
330330
}
331331
#endif
332332
else {
333-
int32_t *indices = keys->dk_indices.as_4;
333+
int32_t *indices = (int32_t*)(keys->dk_indices);
334334
ix = indices[i];
335335
}
336336
assert(ix >= DKIX_DUMMY);
@@ -346,23 +346,23 @@ dk_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix)
346346
assert(ix >= DKIX_DUMMY);
347347

348348
if (s <= 0xff) {
349-
int8_t *indices = keys->dk_indices.as_1;
349+
int8_t *indices = (int8_t*)(keys->dk_indices);
350350
assert(ix <= 0x7f);
351351
indices[i] = (char)ix;
352352
}
353353
else if (s <= 0xffff) {
354-
int16_t *indices = keys->dk_indices.as_2;
354+
int16_t *indices = (int16_t*)(keys->dk_indices);
355355
assert(ix <= 0x7fff);
356356
indices[i] = (int16_t)ix;
357357
}
358358
#if SIZEOF_VOID_P > 4
359359
else if (s > 0xffffffff) {
360-
int64_t *indices = keys->dk_indices.as_8;
360+
int64_t *indices = (int64_t*)(keys->dk_indices);
361361
indices[i] = ix;
362362
}
363363
#endif
364364
else {
365-
int32_t *indices = keys->dk_indices.as_4;
365+
int32_t *indices = (int32_t*)(keys->dk_indices);
366366
assert(ix <= 0x7fffffff);
367367
indices[i] = (int32_t)ix;
368368
}
@@ -421,8 +421,8 @@ static PyDictKeysObject empty_keys_struct = {
421421
lookdict_split, /* dk_lookup */
422422
0, /* dk_usable (immutable) */
423423
0, /* dk_nentries */
424-
.dk_indices = { .as_1 = {DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY,
425-
DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY}},
424+
{DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY,
425+
DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY}, /* dk_indices */
426426
};
427427

428428
static PyObject *empty_values[1] = { NULL };
@@ -530,7 +530,6 @@ static PyDictKeysObject *new_keys_object(Py_ssize_t size)
530530
}
531531
else {
532532
dk = PyObject_MALLOC(sizeof(PyDictKeysObject)
533-
- Py_MEMBER_SIZE(PyDictKeysObject, dk_indices)
534533
+ es * size
535534
+ sizeof(PyDictKeyEntry) * usable);
536535
if (dk == NULL) {
@@ -543,7 +542,7 @@ static PyDictKeysObject *new_keys_object(Py_ssize_t size)
543542
dk->dk_usable = usable;
544543
dk->dk_lookup = lookdict_unicode_nodummy;
545544
dk->dk_nentries = 0;
546-
memset(&dk->dk_indices.as_1[0], 0xff, es * size);
545+
memset(&dk->dk_indices[0], 0xff, es * size);
547546
memset(DK_ENTRIES(dk), 0, sizeof(PyDictKeyEntry) * usable);
548547
return dk;
549548
}
@@ -3007,7 +3006,6 @@ _PyDict_SizeOf(PyDictObject *mp)
30073006
in the type object. */
30083007
if (mp->ma_keys->dk_refcnt == 1)
30093008
res += (sizeof(PyDictKeysObject)
3010-
- Py_MEMBER_SIZE(PyDictKeysObject, dk_indices)
30113009
+ DK_IXSIZE(mp->ma_keys) * size
30123010
+ sizeof(PyDictKeyEntry) * usable);
30133011
return res;
@@ -3017,7 +3015,6 @@ Py_ssize_t
30173015
_PyDict_KeysSize(PyDictKeysObject *keys)
30183016
{
30193017
return (sizeof(PyDictKeysObject)
3020-
- Py_MEMBER_SIZE(PyDictKeysObject, dk_indices)
30213018
+ DK_IXSIZE(keys) * DK_SIZE(keys)
30223019
+ USABLE_FRACTION(DK_SIZE(keys)) * sizeof(PyDictKeyEntry));
30233020
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.