Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 72fc3c0

Browse filesBrowse files
ShaharNavehShaharNaveh
authored
Update pickle{tools,}.py from 3.13.5 (#6064)
1 parent 566d9aa commit 72fc3c0
Copy full SHA for 72fc3c0

File tree

Expand file treeCollapse file tree

4 files changed

+106
-127
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

4 files changed

+106
-127
lines changed
Open diff view settings
Collapse file

‎Lib/pickle.py‎

Copy file name to clipboardExpand all lines: Lib/pickle.py
+99-49Lines changed: 99 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -314,16 +314,17 @@ def load_frame(self, frame_size):
314314
# Tools used for pickling.
315315

316316
def _getattribute(obj, name):
317+
top = obj
317318
for subpath in name.split('.'):
318319
if subpath == '<locals>':
319320
raise AttributeError("Can't get local attribute {!r} on {!r}"
320-
.format(name, obj))
321+
.format(name, top))
321322
try:
322323
parent = obj
323324
obj = getattr(obj, subpath)
324325
except AttributeError:
325326
raise AttributeError("Can't get attribute {!r} on {!r}"
326-
.format(name, obj)) from None
327+
.format(name, top)) from None
327328
return obj, parent
328329

329330
def whichmodule(obj, name):
@@ -396,6 +397,8 @@ def decode_long(data):
396397
return int.from_bytes(data, byteorder='little', signed=True)
397398

398399

400+
_NoValue = object()
401+
399402
# Pickling machinery
400403

401404
class _Pickler:
@@ -530,10 +533,11 @@ def save(self, obj, save_persistent_id=True):
530533
self.framer.commit_frame()
531534

532535
# Check for persistent id (defined by a subclass)
533-
pid = self.persistent_id(obj)
534-
if pid is not None and save_persistent_id:
535-
self.save_pers(pid)
536-
return
536+
if save_persistent_id:
537+
pid = self.persistent_id(obj)
538+
if pid is not None:
539+
self.save_pers(pid)
540+
return
537541

538542
# Check the memo
539543
x = self.memo.get(id(obj))
@@ -542,8 +546,8 @@ def save(self, obj, save_persistent_id=True):
542546
return
543547

544548
rv = NotImplemented
545-
reduce = getattr(self, "reducer_override", None)
546-
if reduce is not None:
549+
reduce = getattr(self, "reducer_override", _NoValue)
550+
if reduce is not _NoValue:
547551
rv = reduce(obj)
548552

549553
if rv is NotImplemented:
@@ -556,8 +560,8 @@ def save(self, obj, save_persistent_id=True):
556560

557561
# Check private dispatch table if any, or else
558562
# copyreg.dispatch_table
559-
reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
560-
if reduce is not None:
563+
reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue)
564+
if reduce is not _NoValue:
561565
rv = reduce(obj)
562566
else:
563567
# Check for a class with a custom metaclass; treat as regular
@@ -567,12 +571,12 @@ def save(self, obj, save_persistent_id=True):
567571
return
568572

569573
# Check for a __reduce_ex__ method, fall back to __reduce__
570-
reduce = getattr(obj, "__reduce_ex__", None)
571-
if reduce is not None:
574+
reduce = getattr(obj, "__reduce_ex__", _NoValue)
575+
if reduce is not _NoValue:
572576
rv = reduce(self.proto)
573577
else:
574-
reduce = getattr(obj, "__reduce__", None)
575-
if reduce is not None:
578+
reduce = getattr(obj, "__reduce__", _NoValue)
579+
if reduce is not _NoValue:
576580
rv = reduce()
577581
else:
578582
raise PicklingError("Can't pickle %r object: %r" %
@@ -780,14 +784,10 @@ def save_float(self, obj):
780784
self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
781785
dispatch[float] = save_float
782786

783-
def save_bytes(self, obj):
784-
if self.proto < 3:
785-
if not obj: # bytes object is empty
786-
self.save_reduce(bytes, (), obj=obj)
787-
else:
788-
self.save_reduce(codecs.encode,
789-
(str(obj, 'latin1'), 'latin1'), obj=obj)
790-
return
787+
def _save_bytes_no_memo(self, obj):
788+
# helper for writing bytes objects for protocol >= 3
789+
# without memoizing them
790+
assert self.proto >= 3
791791
n = len(obj)
792792
if n <= 0xff:
793793
self.write(SHORT_BINBYTES + pack("<B", n) + obj)
@@ -797,28 +797,44 @@ def save_bytes(self, obj):
797797
self._write_large_bytes(BINBYTES + pack("<I", n), obj)
798798
else:
799799
self.write(BINBYTES + pack("<I", n) + obj)
800+
801+
def save_bytes(self, obj):
802+
if self.proto < 3:
803+
if not obj: # bytes object is empty
804+
self.save_reduce(bytes, (), obj=obj)
805+
else:
806+
self.save_reduce(codecs.encode,
807+
(str(obj, 'latin1'), 'latin1'), obj=obj)
808+
return
809+
self._save_bytes_no_memo(obj)
800810
self.memoize(obj)
801811
dispatch[bytes] = save_bytes
802812

813+
def _save_bytearray_no_memo(self, obj):
814+
# helper for writing bytearray objects for protocol >= 5
815+
# without memoizing them
816+
assert self.proto >= 5
817+
n = len(obj)
818+
if n >= self.framer._FRAME_SIZE_TARGET:
819+
self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
820+
else:
821+
self.write(BYTEARRAY8 + pack("<Q", n) + obj)
822+
803823
def save_bytearray(self, obj):
804824
if self.proto < 5:
805825
if not obj: # bytearray is empty
806826
self.save_reduce(bytearray, (), obj=obj)
807827
else:
808828
self.save_reduce(bytearray, (bytes(obj),), obj=obj)
809829
return
810-
n = len(obj)
811-
if n >= self.framer._FRAME_SIZE_TARGET:
812-
self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
813-
else:
814-
self.write(BYTEARRAY8 + pack("<Q", n) + obj)
830+
self._save_bytearray_no_memo(obj)
815831
self.memoize(obj)
816832
dispatch[bytearray] = save_bytearray
817833

818834
if _HAVE_PICKLE_BUFFER:
819835
def save_picklebuffer(self, obj):
820836
if self.proto < 5:
821-
raise PicklingError("PickleBuffer can only pickled with "
837+
raise PicklingError("PickleBuffer can only be pickled with "
822838
"protocol >= 5")
823839
with obj.raw() as m:
824840
if not m.contiguous:
@@ -830,10 +846,18 @@ def save_picklebuffer(self, obj):
830846
if in_band:
831847
# Write data in-band
832848
# XXX The C implementation avoids a copy here
849+
buf = m.tobytes()
850+
in_memo = id(buf) in self.memo
833851
if m.readonly:
834-
self.save_bytes(m.tobytes())
852+
if in_memo:
853+
self._save_bytes_no_memo(buf)
854+
else:
855+
self.save_bytes(buf)
835856
else:
836-
self.save_bytearray(m.tobytes())
857+
if in_memo:
858+
self._save_bytearray_no_memo(buf)
859+
else:
860+
self.save_bytearray(buf)
837861
else:
838862
# Write data out-of-band
839863
self.write(NEXT_BUFFER)
@@ -1070,11 +1094,16 @@ def save_global(self, obj, name=None):
10701094
(obj, module_name, name))
10711095

10721096
if self.proto >= 2:
1073-
code = _extension_registry.get((module_name, name))
1074-
if code:
1075-
assert code > 0
1097+
code = _extension_registry.get((module_name, name), _NoValue)
1098+
if code is not _NoValue:
10761099
if code <= 0xff:
1077-
write(EXT1 + pack("<B", code))
1100+
data = pack("<B", code)
1101+
if data == b'\0':
1102+
# Should never happen in normal circumstances,
1103+
# since the type and the value of the code are
1104+
# checked in copyreg.add_extension().
1105+
raise RuntimeError("extension code 0 is out of range")
1106+
write(EXT1 + data)
10781107
elif code <= 0xffff:
10791108
write(EXT2 + pack("<H", code))
10801109
else:
@@ -1088,11 +1117,35 @@ def save_global(self, obj, name=None):
10881117
self.save(module_name)
10891118
self.save(name)
10901119
write(STACK_GLOBAL)
1091-
elif parent is not module:
1092-
self.save_reduce(getattr, (parent, lastname))
1093-
elif self.proto >= 3:
1094-
write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
1095-
bytes(name, "utf-8") + b'\n')
1120+
elif '.' in name:
1121+
# In protocol < 4, objects with multi-part __qualname__
1122+
# are represented as
1123+
# getattr(getattr(..., attrname1), attrname2).
1124+
dotted_path = name.split('.')
1125+
name = dotted_path.pop(0)
1126+
save = self.save
1127+
for attrname in dotted_path:
1128+
save(getattr)
1129+
if self.proto < 2:
1130+
write(MARK)
1131+
self._save_toplevel_by_name(module_name, name)
1132+
for attrname in dotted_path:
1133+
save(attrname)
1134+
if self.proto < 2:
1135+
write(TUPLE)
1136+
else:
1137+
write(TUPLE2)
1138+
write(REDUCE)
1139+
else:
1140+
self._save_toplevel_by_name(module_name, name)
1141+
1142+
self.memoize(obj)
1143+
1144+
def _save_toplevel_by_name(self, module_name, name):
1145+
if self.proto >= 3:
1146+
# Non-ASCII identifiers are supported only with protocols >= 3.
1147+
self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
1148+
bytes(name, "utf-8") + b'\n')
10961149
else:
10971150
if self.fix_imports:
10981151
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
@@ -1102,14 +1155,12 @@ def save_global(self, obj, name=None):
11021155
elif module_name in r_import_mapping:
11031156
module_name = r_import_mapping[module_name]
11041157
try:
1105-
write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
1106-
bytes(name, "ascii") + b'\n')
1158+
self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
1159+
bytes(name, "ascii") + b'\n')
11071160
except UnicodeEncodeError:
11081161
raise PicklingError(
11091162
"can't pickle global identifier '%s.%s' using "
1110-
"pickle protocol %i" % (module, name, self.proto)) from None
1111-
1112-
self.memoize(obj)
1163+
"pickle protocol %i" % (module_name, name, self.proto)) from None
11131164

11141165
def save_type(self, obj):
11151166
if obj is type(None):
@@ -1546,9 +1597,8 @@ def load_ext4(self):
15461597
dispatch[EXT4[0]] = load_ext4
15471598

15481599
def get_extension(self, code):
1549-
nil = []
1550-
obj = _extension_cache.get(code, nil)
1551-
if obj is not nil:
1600+
obj = _extension_cache.get(code, _NoValue)
1601+
if obj is not _NoValue:
15521602
self.append(obj)
15531603
return
15541604
key = _inverted_registry.get(code)
@@ -1705,8 +1755,8 @@ def load_build(self):
17051755
stack = self.stack
17061756
state = stack.pop()
17071757
inst = stack[-1]
1708-
setstate = getattr(inst, "__setstate__", None)
1709-
if setstate is not None:
1758+
setstate = getattr(inst, "__setstate__", _NoValue)
1759+
if setstate is not _NoValue:
17101760
setstate(state)
17111761
return
17121762
slotstate = None
Collapse file

‎Lib/pickletools.py‎

Copy file name to clipboardExpand all lines: Lib/pickletools.py
+7-4Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ def read_uint8(f):
312312
doc="Eight-byte unsigned integer, little-endian.")
313313

314314

315-
def read_stringnl(f, decode=True, stripquotes=True):
315+
def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'):
316316
r"""
317317
>>> import io
318318
>>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n"))
@@ -356,7 +356,7 @@ def read_stringnl(f, decode=True, stripquotes=True):
356356
raise ValueError("no string quotes around %r" % data)
357357

358358
if decode:
359-
data = codecs.escape_decode(data)[0].decode("ascii")
359+
data = codecs.escape_decode(data)[0].decode(encoding)
360360
return data
361361

362362
stringnl = ArgumentDescriptor(
@@ -370,7 +370,7 @@ def read_stringnl(f, decode=True, stripquotes=True):
370370
""")
371371

372372
def read_stringnl_noescape(f):
373-
return read_stringnl(f, stripquotes=False)
373+
return read_stringnl(f, stripquotes=False, encoding='utf-8')
374374

375375
stringnl_noescape = ArgumentDescriptor(
376376
name='stringnl_noescape',
@@ -2513,7 +2513,10 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0):
25132513
# make a mild effort to align arguments
25142514
line += ' ' * (10 - len(opcode.name))
25152515
if arg is not None:
2516-
line += ' ' + repr(arg)
2516+
if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"):
2517+
line += ' ' + ascii(arg)
2518+
else:
2519+
line += ' ' + repr(arg)
25172520
if markmsg:
25182521
line += ' ' + markmsg
25192522
if annotate:

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.