diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 060d4bb6997a4d..5b172f9c72e08f 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -874,6 +874,13 @@ All of the following opcodes use their arguments. Works as :opcode:`BUILD_TUPLE`, but creates a list. +.. opcode:: BUILD_LIST_PREALLOC (var_num) + + Creates a list with a pre-allocated size and pushes the resulting list onto the stack. + The size is the length-hint of the local ``co_varnames[var_num]``, if the length cannot + be established, it will default a length of 0. + + .. opcode:: BUILD_SET (count) Works as :opcode:`BUILD_TUPLE`, but creates a set. diff --git a/Include/listobject.h b/Include/listobject.h index 6057279d51c3a4..499bfd479384c1 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -62,7 +62,7 @@ PyAPI_FUNC(int) PyList_Reverse(PyObject *); PyAPI_FUNC(PyObject *) PyList_AsTuple(PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); - +PyAPI_FUNC(PyObject *) _PyList_NewPrealloc(Py_ssize_t size); PyAPI_FUNC(int) PyList_ClearFreeList(void); PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); #endif diff --git a/Include/opcode.h b/Include/opcode.h index 2a29e978857035..0311aae76a0b1e 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -124,6 +124,7 @@ extern "C" { #define BUILD_CONST_KEY_MAP 156 #define BUILD_STRING 157 #define BUILD_TUPLE_UNPACK_WITH_CALL 158 +#define BUILD_LIST_PREALLOC 159 #define LOAD_METHOD 160 #define CALL_METHOD 161 #define CALL_FINALLY 162 diff --git a/Lib/opcode.py b/Lib/opcode.py index 3fb716b5d96a29..2b0a74fc857bc3 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -207,6 +207,7 @@ def jabs_op(name, op): def_op('BUILD_CONST_KEY_MAP', 156) def_op('BUILD_STRING', 157) def_op('BUILD_TUPLE_UNPACK_WITH_CALL', 158) +def_op('BUILD_LIST_PREALLOC', 159) name_op('LOAD_METHOD', 160) def_op('CALL_METHOD', 161) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 8d1912b6ee5881..37dea9727a97dc 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -384,7 +384,7 @@ def foo(x): dis_nested_2 = """%s Disassembly of at 0x..., file "%s", line %d>: -%3d 0 BUILD_LIST 0 +%3d 0 BUILD_LIST_PREALLOC 0 2 LOAD_FAST 0 (.0) >> 4 FOR_ITER 12 (to 18) 6 STORE_FAST 1 (z) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index ddb169fe58957c..063afcfe31cd37 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -123,6 +123,13 @@ >>> test_func() [2, 2, 2, 2, 2] +Verify that an overflow error is raised for listcomps with very-large iterators + + >>> [y for y in range(2**256)] # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + OverflowError: ... + """ diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-04-08-03-42-24.bpo-36551.5Ocm2j.rst b/Misc/NEWS.d/next/Core and Builtins/2019-04-08-03-42-24.bpo-36551.5Ocm2j.rst new file mode 100644 index 00000000000000..118c1d8f679763 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-04-08-03-42-24.bpo-36551.5Ocm2j.rst @@ -0,0 +1,2 @@ +Change the compiled bytecode for list comprehensions where there is no if statement. Allocates the size of the list to the length of the iterator in order to avoid having to continuously resize the list. +Also fixes a bug for a list comprehension containing an iterator with a potential size > PY_SSIZE_MAX, e.g. range(2**256). \ No newline at end of file diff --git a/Objects/listobject.c b/Objects/listobject.c index c29954283c488e..3d780db1684cad 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -215,6 +215,12 @@ list_new_prealloc(Py_ssize_t size) return (PyObject *) op; } +PyObject * +_PyList_NewPrealloc(Py_ssize_t size) +{ + return list_new_prealloc(size); +} + Py_ssize_t PyList_Size(PyObject *op) { diff --git a/Python/ceval.c b/Python/ceval.c index 28e923219d389c..aa6537b699cab7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1168,6 +1168,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } case TARGET(LOAD_FAST): { + PREDICTED(LOAD_FAST); PyObject *value = GETLOCAL(oparg); if (value == NULL) { format_exc_check_arg(PyExc_UnboundLocalError, @@ -2484,6 +2485,20 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) DISPATCH(); } + case TARGET(BUILD_LIST_PREALLOC): { + PyObject *list, *target = GETLOCAL(oparg); + Py_INCREF(target); + Py_ssize_t size = PyObject_LengthHint(target, 0); + Py_DECREF(target); + if (size < 0) + goto error; + + list = _PyList_NewPrealloc(size); + PUSH(list); + PREDICT(LOAD_FAST); + DISPATCH(); + } + case TARGET(BUILD_LIST): { PyObject *list = PyList_New(oparg); if (list == NULL) diff --git a/Python/compile.c b/Python/compile.c index a992e4b4653cc1..3231c90b83e94c 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1006,6 +1006,8 @@ stack_effect(int opcode, int oparg, int jump) case BUILD_SET: case BUILD_STRING: return 1-oparg; + case BUILD_LIST_PREALLOC: + return 1; case BUILD_LIST_UNPACK: case BUILD_TUPLE_UNPACK: case BUILD_TUPLE_UNPACK_WITH_CALL: @@ -4368,7 +4370,10 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, int op; switch (type) { case COMP_LISTCOMP: - op = BUILD_LIST; + if (asdl_seq_LEN(outermost->ifs) > 0) + op = BUILD_LIST; + else + op = BUILD_LIST_PREALLOC; break; case COMP_SETCOMP: op = BUILD_SET; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index e82959be086455..3bbc54794bcc91 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -158,7 +158,7 @@ static void *opcode_targets[256] = { &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE_UNPACK_WITH_CALL, - &&_unknown_opcode, + &&TARGET_BUILD_LIST_PREALLOC, &&TARGET_LOAD_METHOD, &&TARGET_CALL_METHOD, &&TARGET_CALL_FINALLY,