From bcf89c7efd4083e013ac305030b96252251c8aa5 Mon Sep 17 00:00:00 2001 From: Valentin Valls Date: Sun, 26 Apr 2026 18:39:18 +0200 Subject: [PATCH 01/35] Fixed README example (#668) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1f063242..223742dd 100644 --- a/README.md +++ b/README.md @@ -119,15 +119,15 @@ It is also possible to pack/unpack custom data types using the **ext** type. >>> import array >>> def default(obj): ... if isinstance(obj, array.array) and obj.typecode == 'd': -... return msgpack.ExtType(42, obj.tostring()) +... return msgpack.ExtType(42, obj.tobytes()) ... raise TypeError("Unknown type: %r" % (obj,)) ... >>> def ext_hook(code, data): ... if code == 42: ... a = array.array('d') -... a.fromstring(data) +... a.frombytes(data) ... return a -... return ExtType(code, data) +... return msgpack.ExtType(code, data) ... >>> data = array.array('d', [1.2, 3.4]) >>> packed = msgpack.packb(data, default=default) From a54fad13c10d3a555d6bd71b06dccf980d516a18 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Apr 2026 15:06:20 +0900 Subject: [PATCH 02/35] pinact --- .github/workflows/docs.yaml | 4 ++-- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yml | 6 +++--- .github/workflows/wheel.yml | 18 +++++++++--------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b696b926..917e3027 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' cache: "pip" diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 198cf7b5..3ea2a87f 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: ruff check run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6b1664ae..9d447843 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,10 +19,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.py }} allow-prereleases: true @@ -55,7 +55,7 @@ jobs: python -m build -nv - name: upload packages - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: dist-${{ matrix.os }}-${{ matrix.py }} path: dist diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index f3fbe29e..e9304670 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -27,8 +27,8 @@ jobs: name: Build wheels on ${{ matrix.os }}${{ matrix.name_suffix || '' }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" cache: "pip" @@ -40,12 +40,12 @@ jobs: - name: Set up QEMU for emulation if: matrix.cibw_archs == 'riscv64' - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 with: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@v3.3.0 + uses: pypa/cibuildwheel@63fd63b352a9a8bdcc24791c9dbee952ee9a8abc # v3.3.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -59,7 +59,7 @@ jobs: python -m build -s -o wheelhouse - name: Upload Wheels to artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: wheels-${{ matrix.os }}${{ matrix.name_suffix || '' }} path: wheelhouse @@ -69,7 +69,7 @@ jobs: needs: [build_wheels] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: # unpacks all CIBW artifacts into dist/ pattern: wheels-* @@ -77,7 +77,7 @@ jobs: merge-multiple: true - name: Upload Wheels to artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: wheels-all path: dist @@ -93,13 +93,13 @@ jobs: # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: # unpacks all CIBW artifacts into dist/ pattern: wheels-* path: dist merge-multiple: true - - uses: pypa/gh-action-pypi-publish@release/v1 + - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 #with: # To test: repository-url: https://test.pypi.org/legacy/ From 94242eb0e9c2292b51b76400cb6523eee6d8d576 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Apr 2026 15:07:06 +0900 Subject: [PATCH 03/35] dependabot --- .github/dependabot.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..c2e9b9c9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + all-dependencies: + patterns: + - "*" From b7ebb87b9d48969959c79644e225d90d08b3e47c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:16:05 +0900 Subject: [PATCH 04/35] action: Bump the all-dependencies group with 6 updates (#669) --- .github/workflows/docs.yaml | 4 ++-- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/wheel.yml | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 917e3027..4467ca8b 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' cache: "pip" diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 3ea2a87f..93b64947 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: ruff check run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d447843..729a45f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -55,7 +55,7 @@ jobs: python -m build -nv - name: upload packages - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dist-${{ matrix.os }}-${{ matrix.py }} path: dist diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index e9304670..eb457277 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -40,12 +40,12 @@ jobs: - name: Set up QEMU for emulation if: matrix.cibw_archs == 'riscv64' - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 with: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@63fd63b352a9a8bdcc24791c9dbee952ee9a8abc # v3.3.0 + uses: pypa/cibuildwheel@8d2b08b68458a16aeb24b64e68a09ab1c8e82084 # v3.4.1 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -59,7 +59,7 @@ jobs: python -m build -s -o wheelhouse - name: Upload Wheels to artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-${{ matrix.os }}${{ matrix.name_suffix || '' }} path: wheelhouse @@ -69,7 +69,7 @@ jobs: needs: [build_wheels] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: # unpacks all CIBW artifacts into dist/ pattern: wheels-* @@ -77,7 +77,7 @@ jobs: merge-multiple: true - name: Upload Wheels to artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-all path: dist @@ -93,7 +93,7 @@ jobs: # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: # unpacks all CIBW artifacts into dist/ pattern: wheels-* From 0d600a33285c8ef968a4926840ccc6a096ae3e09 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Apr 2026 17:02:52 +0900 Subject: [PATCH 05/35] ci: use ubuntu-slim for lint (#670) --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 93b64947..e2631ffa 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -8,7 +8,7 @@ jobs: # by the push to the branch. if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From cd8137093ac0abcfa9acc9d9c2ad6abeeee8ad30 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 12 May 2026 06:59:40 +0200 Subject: [PATCH 06/35] fix: enforce `strict_map_key` with `object_pairs_hook` (#673) This PR makes it so that `self._strict_map_key` is enforced even when using `_object_pairs_hook` which didn't use to be the case. --- msgpack/fallback.py | 12 +++++++++--- test/test_except.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b02e47cf..d71bbfc7 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -518,9 +518,15 @@ def _unpack(self, execute=EX_CONSTRUCT): self._unpack(EX_SKIP) return if self._object_pairs_hook is not None: - ret = self._object_pairs_hook( - (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) for _ in range(n) - ) + + def _gen(): + for _ in range(n): + key = self._unpack(EX_CONSTRUCT) + if self._strict_map_key and type(key) not in (str, bytes): + raise ValueError("%s is not allowed for map key" % str(type(key))) + yield key, self._unpack(EX_CONSTRUCT) + + ret = self._object_pairs_hook(_gen()) else: ret = {} for _ in range(n): diff --git a/test/test_except.py b/test/test_except.py index c56a6a30..be8ca4cf 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -89,3 +89,17 @@ def test_strict_map_key(): packed = packb(invalid, use_bin_type=True) with raises(ValueError): unpackb(packed, raw=False, strict_map_key=True) + + +def test_strict_map_key_with_object_pairs_hook(): + # strict_map_key should be enforced even when object_pairs_hook is set + invalid = {42: "value"} + packed = packb(invalid, use_bin_type=True) + with raises(ValueError): + unpackb(packed, raw=False, strict_map_key=True, object_pairs_hook=list) + + # valid keys (str/bytes) should still work with object_pairs_hook + valid = {"key": "value"} + packed = packb(valid, use_bin_type=True) + result = unpackb(packed, raw=False, strict_map_key=True, object_pairs_hook=list) + assert result == [("key", "value")] From 4b2749cdff9326ec8bf2b7006530ac43d263b992 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 16:08:23 +0900 Subject: [PATCH 07/35] Raise DEFAULT_RECURSE_LIMIT from 511 to 1024 (#676) --- msgpack/_packer.pyx | 2 +- msgpack/fallback.py | 2 +- test/test_limits.py | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 94d1462c..4d0d2d21 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -38,7 +38,7 @@ cdef extern from "pack.h": int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds) except -1 -cdef int DEFAULT_RECURSE_LIMIT=511 +cdef int DEFAULT_RECURSE_LIMIT=1024 cdef long long ITEM_LIMIT = (2**32)-1 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d71bbfc7..1f2daf7b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -52,7 +52,7 @@ def newlist_hint(size): TYPE_BIN = 4 TYPE_EXT = 5 -DEFAULT_RECURSE_LIMIT = 511 +DEFAULT_RECURSE_LIMIT = 1024 def _check_type_strict(obj, t, type=type, tuple=tuple): diff --git a/test/test_limits.py b/test/test_limits.py index 9b92b4d9..468c1c36 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -153,6 +153,30 @@ def test_auto_max_array_len(): unpacker.unpack() +def test_nest_limit_1024(): + import sys + + # Build a list nested 1024 levels deep + d = None + for _ in range(1024): + d = [d] + + # Temporarily raise Python's recursion limit so packing 1024 levels succeeds + old_limit = sys.getrecursionlimit() + sys.setrecursionlimit(max(old_limit, 10000)) + try: + packed = packb(d) + result = unpackb(packed) + finally: + sys.setrecursionlimit(old_limit) + + # Verify structure iteratively to avoid hitting C-level recursion limit in == + for _ in range(1024): + assert isinstance(result, list) and len(result) == 1 + result = result[0] + assert result is None + + def test_auto_max_map_len(): # len(packed) == 6 -> max_map_len == 3 packed = b"\xde\x00\x04zzz" From 378edc60f15c425231d1db99b1886b93c0dcf784 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Sat, 23 May 2026 18:48:01 +0200 Subject: [PATCH 08/35] fix: properly handle return codes in `pack_timestamp` (#672) --- msgpack/pack_template.h | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index b8959f02..e02ec324 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -551,6 +551,7 @@ static inline int msgpack_pack_ext(msgpack_packer* x, char typecode, size_t l) */ static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uint32_t nanoseconds) { + int ret; if ((seconds >> 34) == 0) { /* seconds is unsigned and fits in 34 bits */ uint64_t data64 = ((uint64_t)nanoseconds << 34) | (uint64_t)seconds; @@ -558,26 +559,33 @@ static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uin /* no nanoseconds and seconds is 32bits or smaller. timestamp32. */ unsigned char buf[4]; uint32_t data32 = (uint32_t)data64; - msgpack_pack_ext(x, -1, 4); + ret = msgpack_pack_ext(x, -1, 4); + if (ret != 0) + return ret; + _msgpack_store32(buf, data32); - msgpack_pack_raw_body(x, buf, 4); + return msgpack_pack_raw_body(x, buf, 4); } else { /* timestamp64 */ unsigned char buf[8]; - msgpack_pack_ext(x, -1, 8); - _msgpack_store64(buf, data64); - msgpack_pack_raw_body(x, buf, 8); + ret = msgpack_pack_ext(x, -1, 8); + if (ret != 0) + return ret; + _msgpack_store64(buf, data64); + return msgpack_pack_raw_body(x, buf, 8); } } else { - /* seconds is signed or >34bits */ - unsigned char buf[12]; - _msgpack_store32(&buf[0], nanoseconds); - _msgpack_store64(&buf[4], seconds); - msgpack_pack_ext(x, -1, 12); - msgpack_pack_raw_body(x, buf, 12); + /* seconds is signed or >34bits */ + unsigned char buf[12]; + _msgpack_store32(&buf[0], nanoseconds); + _msgpack_store64(&buf[4], seconds); + ret = msgpack_pack_ext(x, -1, 12); + if (ret != 0) + return ret; + + return msgpack_pack_raw_body(x, buf, 12); } - return 0; } From 4a077456754a060d2bb14578e03bcb1153d952cc Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Wed, 27 May 2026 11:04:24 +0200 Subject: [PATCH 09/35] fix: avoid memory leak when decoding invalid nested arrays (#671) --- msgpack/_unpacker.pyx | 1 + msgpack/unpack_template.h | 8 ++++++++ test/test_except.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e25986ee..29cdec4a 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -322,6 +322,7 @@ cdef class Unpacker: self.buf = NULL def __dealloc__(self): + unpack_clear(&self.ctx) PyMem_Free(self.buf) self.buf = NULL diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index cce29e7a..42306618 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -72,6 +72,14 @@ static inline PyObject* unpack_data(unpack_context* ctx) static inline void unpack_clear(unpack_context *ctx) { + unsigned int i; + for (i = 1; i < ctx->top; i++) { + Py_CLEAR(ctx->stack[i].obj); + /* map_key holds a live reference only while waiting for the value */ + if (ctx->stack[i].ct == CT_MAP_VALUE) { + Py_CLEAR(ctx->stack[i].map_key); + } + } Py_CLEAR(ctx->stack[0].obj); } diff --git a/test/test_except.py b/test/test_except.py index be8ca4cf..c04b110d 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import datetime +import gc +import tracemalloc from pytest import raises @@ -80,6 +82,39 @@ def test_invalidvalue(): unpackb(b"\x91" * 3000) # nested fixarray(len=1) +def test_no_memory_leak_on_nested_invalid_tag() -> None: + """Regression test: unpacking nested arrays containing an invalid tag must not leak objects.""" + + kwargs: dict = { + "raw": False, + "strict_map_key": False, + "max_array_len": 1 << 20, + "max_map_len": 1 << 20, + } + n = 1000 + + for depth in range(1, 15): + data = bytes([0x91] * depth + [0xC1]) + + gc.collect() + tracemalloc.start() + s1 = tracemalloc.take_snapshot() + + for _ in range(n): + try: + unpackb(data, **kwargs) + except Exception: + pass + + gc.collect() + s2 = tracemalloc.take_snapshot() + tracemalloc.stop() + + leaked = sum(s.count_diff for s in s2.compare_to(s1, "lineno") if s.count_diff > 0) + per_call = leaked / n + assert per_call < 1.0, f"depth={depth}: {per_call:.2f} leaked objects/call (expected < 1)" + + def test_strict_map_key(): valid = {"unicode": 1, b"bytes": 2} packed = packb(valid, use_bin_type=True) From 284782d647815ed46b07c6bd2d5c620a8a46869e Mon Sep 17 00:00:00 2001 From: Kadir Can Ozden <101993364+bysiber@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:27:05 +0300 Subject: [PATCH 10/35] Add missing autoreset in Packer.pack_ext_type (#663) --- msgpack/_packer.pyx | 4 ++++ msgpack/fallback.py | 4 ++++ pyproject.toml | 7 +++++++ test/test_extension.py | 11 ++++++++++- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 4d0d2d21..2d92e8a4 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -291,6 +291,10 @@ cdef class Packer: raise ValueError("ext data too large") msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) + if self.autoreset: + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf @cython.critical_section def pack_array_header(self, long long size): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 1f2daf7b..61d77db0 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -867,6 +867,10 @@ def pack_ext_type(self, typecode, data): self._buffer.write(b"\xc9" + struct.pack(">I", L)) self._buffer.write(struct.pack("B", typecode)) self._buffer.write(data) + if self._autoreset: + ret = self._buffer.getvalue() + self._buffer = BytesIO() + return ret def _pack_array_header(self, n): if n <= 0x0F: diff --git a/pyproject.toml b/pyproject.toml index c69d5a7c..4e4510b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] +dependencies = [] [project.urls] Homepage = "https://msgpack.org/" @@ -43,3 +44,9 @@ lint.select = [ "I", # isort #"UP", pyupgrade ] + +[dependency-groups] +dev = [ + "cython>=3.2.5", + "pytest>=9.0.3", +] diff --git a/test/test_extension.py b/test/test_extension.py index aaf0fd92..61852f15 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -6,7 +6,7 @@ def test_pack_ext_type(): def p(s): - packer = msgpack.Packer() + packer = msgpack.Packer(autoreset=False) packer.pack_ext_type(0x42, s) return packer.bytes() @@ -20,6 +20,15 @@ def p(s): assert p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 # ext 32 +def test_pack_ext_type_autoreset(): + packer = msgpack.Packer() + + assert packer.pack_ext_type(0x42, b"A") == b"\xd4\x42A" + assert packer.bytes() == b"" + assert packer.pack_ext_type(0x42, b"ABC") == b"\xc7\x03\x42ABC" + assert packer.bytes() == b"" + + def test_unpack_ext_type(): def check(b, expected): assert msgpack.unpackb(b) == expected From 9de2fd9b24b2026ce5921fc45417893d98f534bf Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Tue, 2 Jun 2026 03:21:05 -0400 Subject: [PATCH 11/35] Add no-GIL interpreter support (#641) Co-authored-by: Lysandros Nikolaou Co-authored-by: Frank Dana Co-authored-by: Inada Naoki --- .github/workflows/test.yml | 20 +++++++++++ test/uneeded_test_multithreading.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 test/uneeded_test_multithreading.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 729a45f1..a8873d51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,7 @@ jobs: exclude: - os: windows-11-arm py: "3.10" + runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -33,6 +34,11 @@ jobs: run: | python -m pip install -r requirements.txt pytest + - name: Install pytest-run-parallel under free-threading + if: contains(matrix.py, 't') + run: | + pip install pytest-run-parallel + - name: Build shell: bash run: | @@ -40,15 +46,29 @@ jobs: pip install . - name: Test (C extension) + if: ${{ ! contains(matrix.py, 't') }} shell: bash run: | pytest -v test - name: Test (pure Python fallback) + if: ${{ ! contains(matrix.py, 't') }} shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test + - name: Test (C extension) in parallel under free-threading + if: contains(matrix.py, 't') + shell: bash + run: | + pytest -v --parallel-threads=auto --iterations=20 test + + - name: Test (pure Python fallback) in parallel under free-threading + if: contains(matrix.py, 't') + shell: bash + run: | + MSGPACK_PUREPYTHON=1 pytest -v --parallel-threads=auto --iterations=20 test + - name: build packages shell: bash run: | diff --git a/test/uneeded_test_multithreading.py b/test/uneeded_test_multithreading.py new file mode 100644 index 00000000..6694fdc6 --- /dev/null +++ b/test/uneeded_test_multithreading.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +import threading +from concurrent.futures import ThreadPoolExecutor + +from msgpack import Packer + + +def run_threaded( + func, + num_threads=8, + pass_count=False, + pass_barrier=False, + outer_iterations=1, + prepare_args=None, +): + """Runs a function many times in parallel""" + for _ in range(outer_iterations): + with ThreadPoolExecutor(max_workers=num_threads) as tpe: + if prepare_args is None: + args = [] + else: + args = prepare_args() + if pass_barrier: + barrier = threading.Barrier(num_threads) + args.append(barrier) + if pass_count: + all_args = [(func, i, *args) for i in range(num_threads)] + else: + all_args = [(func, *args) for i in range(num_threads)] + try: + futures = [] + for arg in all_args: + futures.append(tpe.submit(*arg)) + finally: + if len(futures) < num_threads and pass_barrier: + barrier.abort() + for f in futures: + f.result() + + +def test_multithread_packing(): + output = [] + test_data = "abcd" * 10_000_000 + packer = Packer() + + def closure(b): + data = packer.pack(test_data) + output.append(data) + b.wait() + + run_threaded(closure, num_threads=10, pass_barrier=True, pass_count=False) From b83a0aa62b7288f50ae0181f03f573cb0ef83919 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 2 Jun 2026 20:56:05 +0900 Subject: [PATCH 12/35] skip recursion limit test on free-threaded CPython builds (#679) For some reason, it seems that the recursion limit cannot be avoided in free-threaded CPython on GitHub Actions, so I will skip the test. --- test/test_limits.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_limits.py b/test/test_limits.py index 468c1c36..e3bab146 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +import sysconfig + import pytest from msgpack import ( @@ -153,6 +155,11 @@ def test_auto_max_array_len(): unpacker.unpack() +# Skip on free-threaded CPython builds because this test depends on recursion behavior. +IS_FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + + +@pytest.mark.skipif(IS_FREE_THREADED_BUILD, reason="Skipped on free-threaded build") def test_nest_limit_1024(): import sys From 98d2c7907ddf5a6a810aaece9a7394e20bc490bf Mon Sep 17 00:00:00 2001 From: Kadir Can Ozden <101993364+bysiber@users.noreply.github.com> Date: Tue, 2 Jun 2026 15:41:52 +0300 Subject: [PATCH 13/35] Fix Timestamp.from_datetime returning wrong value for pre-epoch datetimes (#662) --- msgpack/ext.py | 2 +- test/test_timestamp.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 9694819a..92ea4530 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -167,4 +167,4 @@ def from_datetime(dt): :rtype: Timestamp """ - return Timestamp(seconds=int(dt.timestamp()), nanoseconds=dt.microsecond * 1000) + return Timestamp(seconds=int(dt.timestamp() // 1), nanoseconds=dt.microsecond * 1000) diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 831141a1..6dfe3689 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -103,6 +103,13 @@ def test_timestamp_datetime(): assert Timestamp.from_datetime(ts).to_datetime() == ts + # Regression test: pre-epoch fractional seconds must floor toward -inf. + pre_epoch = datetime.datetime(1969, 12, 31, 23, 59, 59, 500000, tzinfo=utc) + ts_pre_epoch = Timestamp.from_datetime(pre_epoch) + assert ts_pre_epoch.seconds == -1 + assert ts_pre_epoch.nanoseconds == 500000000 + assert ts_pre_epoch.to_datetime() == pre_epoch + def test_unpack_datetime(): t = Timestamp(42, 14) From 975a2a4c84095c3a16b32bfaf5d6dea0a9593d79 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Tue, 2 Jun 2026 08:42:57 -0400 Subject: [PATCH 14/35] Add 3.15 to CI (#678) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8873d51..8b36670b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] - py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] + py: ["3.15", "3.15t", "3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] exclude: - os: windows-11-arm py: "3.10" From e861f75374b023c9a6b35451757d23e713ec4068 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 2 Jun 2026 17:21:59 +0200 Subject: [PATCH 15/35] fix: use-after-free in `get_data_from_buffer` (#677) --- msgpack/_unpacker.pyx | 7 +++---- msgpack/fallback.py | 2 +- test/test_memoryview.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 29cdec4a..40d12291 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -129,10 +129,9 @@ cdef inline int get_data_from_buffer(object obj, PyBuffer_Release(view) # create a contiguous copy and get buffer contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') - PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) - # view must hold the only reference to contiguous, - # so memory is freed when view is released - Py_DECREF(contiguous) + if PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) == -1: + raise + buffer_len[0] = view.len buf[0] = view.buf return 1 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 61d77db0..860d94ac 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -328,7 +328,7 @@ def feed(self, next_bytes): self._buf_checkpoint = 0 # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython - self._buffer.extend(view) + self._buffer.extend(view if view.contiguous else view.tobytes()) view.release() def _consume(self): diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 0a2a6f53..3f6a39d4 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -97,3 +97,15 @@ def test_multidim_memoryview(): data = view.cast(view.format, (3, 2)) packed = packb(data) assert packed == b"\xc4\x06\x00\x00\x00\x00\x00\x00" + + +def test_unpack_noncontiguous_memoryview(): + # Use a multi-byte value so the padded stride-2 view is non-contiguous. + packed = packb(2**32) + padded = bytearray() + for byte in packed: + padded.append(byte) + padded.append(0) + noncont = memoryview(bytes(padded))[::2] + assert not noncont.c_contiguous + assert unpackb(noncont) == 2**32 From 6cd75741833c38a482c5007c9113e3a263cdcc3d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 3 Jun 2026 00:44:43 +0900 Subject: [PATCH 16/35] change changelog format to markdown (#680) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 555 +++++++++++++++++++++++++++++++++++++++++++ ChangeLog.rst | 625 ------------------------------------------------- pyproject.toml | 2 +- 3 files changed, 556 insertions(+), 626 deletions(-) create mode 100644 CHANGELOG.md delete mode 100644 ChangeLog.rst diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..74e6eb47 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,555 @@ +# 1.1.2 + +Release Date: 2025-10-08 + +This release does not change source code. It updates only building +wheels: + +- Update Cython to v3.1.4 +- Update cibuildwheel to v3.2.0 +- Drop Python 3.8 +- Add Python 3.14 +- Add windows-arm + +# 1.1.1 + +Release Date: 2025-06-13 + +- No change from 1.1.1rc1. + +# 1.1.1rc1 + +Release Date: 2025-06-06 + +- Update Cython to 3.1.1 and cibuildwheel to 2.23.3. + +# 1.1.0 + +Release Date: 2024-09-10 + +- use `PyLong_*` instead of `PyInt_*` for compatibility with future + Cython. (#620) + +# 1.1.0rc2 + +Release Date: 2024-08-19 + +- Update Cython to 3.0.11 for better Python 3.13 support. +- Update cibuildwheel to 2.20.0 to build Python 3.13 wheels. + +# 1.1.0rc1 + +Release Date: 2024-05-07 + +- Update Cython to 3.0.10 to reduce C warnings and future support for + Python 3.13. +- Stop using C++ mode in Cython to reduce compile error on some + compilers. +- `Packer()` has `buf_size` option to specify initial size of internal + buffer to reduce reallocation. +- The default internal buffer size of `Packer()` is reduced from 1MiB to + 256KiB to optimize for common use cases. Use `buf_size` if you are + packing large data. +- `Timestamp.to_datetime()` and `Timestamp.from_datetime()` become more + accurate by avoiding floating point calculations. (#591) +- The Cython code for `Unpacker` has been slightly rewritten for + maintainability. +- The fallback implementation of `Packer()` and `Unpacker()` now uses + keyword-only arguments to improve compatibility with the Cython + implementation. + +# 1.0.8 + +Release Date: 2024-03-01 + +- Update Cython to 3.0.8. This fixes memory leak when iterating + `Unpacker` object on Python 3.12. +- Do not include C/Cython files in binary wheels. + +# 1.0.7 + +Release Date: 2023-09-28 + +- Fix build error of extension module on Windows. (#567) +- `setup.py` doesn't skip build error of extension module. (#568) + +# 1.0.6 + +Release Date: 2023-09-21 + +> [!NOTE] +> v1.0.6 Wheels for Windows don't contain extension module. Please +> upgrade to v1.0.7 or newer. + +- Add Python 3.12 wheels (#517) +- Remove Python 2.7, 3.6, and 3.7 support + +# 1.0.5 + +Release Date: 2023-03-08 + +- Use `__BYTE_ORDER__` instead of `__BYTE_ORDER` for portability. (#513, + \#514) +- Add Python 3.11 wheels (#517) +- fallback: Fix packing multidimensional memoryview (#527) + +# 1.0.4 + +Release Date: 2022-06-03 + +- Support Python 3.11 (beta). +- Don't define `__*_ENDIAN__` macro + on Unix. by @methane in + +- Use PyFloat_Pack8() on Python 3.11a7 by @vstinner in + +- Fix Unpacker max_buffer_length handling by @methane in + + +# 1.0.3 + +Release Date: 2021-11-24 JST + +- Fix Docstring (#459) +- Fix error formatting (#463) +- Improve error message about strict_map_key (#485) + +# 1.0.2 + +- Fix year 2038 problem regression in 1.0.1. (#451) + +# 1.0.1 + +- Add Python 3.9 and linux/arm64 wheels. (#439) +- Fixed Unpacker.tell() after read_bytes() (#426) +- Fixed unpacking datetime before epoch on Windows (#433) +- Fixed fallback Packer didn't check DateTime.tzinfo (#434) + +# 1.0.0 + +Release Date: 2020-02-17 + +- Remove Python 2 support from the `msgpack/_cmsgpack`. + `msgpack/fallback` still supports Python 2. +- Remove `encoding` option from the Packer and Unpacker. +- Unpacker: The default value of `max_buffer_size` is changed to 100MiB. +- Unpacker: `strict_map_key` is True by default now. +- Unpacker: String map keys are interned. +- Drop old buffer protocol support. +- Support Timestamp type. +- Support serializing and decerializing `datetime` object with tzinfo. +- Unpacker: `Fix Unpacker.read_bytes()` in fallback implementation. + (#352) + +# 0.6.2 + +Release Date: 2019-09-20 + +- Support Python 3.8. +- Update Cython to 0.29.13 for support Python 3.8. +- Some small optimizations. + +# 0.6.1 + +Release Date: 2019-01-25 + +This release is for mitigating pain caused by v0.6.0 reduced max input +limits for security reason. + +- `unpackb(data)` configures `max_*_len` options from `len(data)`, + instead of static default sizes. +- `Unpacker(max_buffer_len=N)` configures `max_*_len` options from `N`, + instead of static default sizes. +- `max_bin_len`, `max_str_len`, and `max_ext_len` are deprecated. Since + this is minor release, it's document only deprecation. + +# 0.6.0 + +Release Date: 2018-11-30 + +This release contains some backward incompatible changes for security +reason (DoS). + +## Important changes + +- unpacker: Default value of input limits are smaller than before to + avoid DoS attack. If you need to handle large data, you need to + specify limits manually. (#319) +- Unpacker doesn't wrap underlying `ValueError` (including + `UnicodeError`) into `UnpackValueError`. If you want to catch all + exception during unpack, you need to use `try ... except Exception` + with minimum try code block. (#323, \#233) +- `PackValueError` and `PackOverflowError` are also removed. You need to + catch normal `ValueError` and `OverflowError`. (#323, \#233) +- Unpacker has `strict_map_key` option now. When it is true, only bytes + and str (unicode in Python 2) are allowed for map keys. It is + recommended to avoid hashdos. Default value of this option is False + for backward compatibility reason. But it will be changed True in 1.0. + (#296, \#334) + +## Other changes + +- Extension modules are merged. There is `msgpack._cmsgpack` instead of + `msgpack._packer` and `msgpack._unpacker`. (#314, \#328) +- Add `Unpacker.getbuffer()` method. (#320) +- unpacker: `msgpack.StackError` is raised when input data contains too + nested data. (#331) +- unpacker: `msgpack.FormatError` is raised when input data is not valid + msgpack format. (#331) + +# 0.5.6 + +- Fix fallback.Unpacker.feed() dropped unused data from buffer (#287) +- Resurrect fallback.unpack() and `unpacker.unpack()`. They were removed + at 0.5.5 but it breaks backward compatibility. (#288, #290) + +# 0.5.5 + +- Fix memory leak in pure Python Unpacker.feed() (#283) +- Fix unpack() didn't support `raw` option + (#285) + +# 0.5.4 + +- Undeprecate `unicode_errors` option. (#278) + +# 0.5.3 + +- Fixed regression when passing `unicode_errors` to Packer but not + `encoding`. (#277) + +# 0.5.2 + +- Add `raw` option to Unpacker. It is preferred way than `encoding` + option. +- Packer.pack() reset buffer on exception (#274) + +# 0.5.1 + +- Remove FutureWarning about use_bin_type option (#271) + +# 0.5.0 + +There are some deprecations. Please read changes carefully. + +## Changes + +- Drop Python 2.6 and ~3.4 support. Python 2.7 and 3.5+ are supported. +- Deprecate useless custom exceptions. Use ValueError instead of + PackValueError, Exception instead of PackException and + UnpackException, etc... See msgpack/exceptions.py +- Add *strict_types* option to packer. It can be used to serialize + subclass of builtin types. For example, when packing object which type + is subclass of dict, `default()` is called. `default()` is called for + tuple too. +- Pure Python implementation supports packing memoryview object. +- Support packing bytearray. +- Add `Unpacker.tell()`. And `write_bytes` option is deprecated. + +## Bugs fixed + +- Fixed zero length raw can't be decoded when encoding is specified. + (#236) + +# 0.4.8 + +Release Date: 2016-07-29 + +## Bugs fixed + +- Calling ext_hook with wrong length. (Only on Windows, maybe. \#203) + +# 0.4.7 + +Release Date: 2016-01-25 + +## Bugs fixed + +- Memory leak when unpack is failed + +## Changes + +- Reduce compiler warnings while building extension module +- unpack() now accepts ext_hook argument like Unpacker and unpackb() +- Update Cython version to 0.23.4 +- default function is called when integer overflow + +# 0.4.6 + +Release Date: 2015-03-13 + +## Bugs fixed + +- fallback.Unpacker: Fix Data corruption when OutOfData. This bug only + affects "Streaming unpacking." + +# 0.4.5 + +Release Date: 2015-01-25 + +## Incompatible Changes + +## Changes + +## Bugs fixed + +- Fix test failure on pytest 2.3. (by @ktdreyer) +- Fix typos in ChangeLog. (Thanks to @dmick) +- Improve README.rst (by @msabramo) + +# 0.4.4 + +Release Date: 2015-01-09 + +## Incompatible Changes + +## Changes + +## Bugs fixed + +- Fix compile error. + +# 0.4.3 + +Release Date: 2015-01-07 + +## Incompatible Changes + +## Changes + +## Bugs fixed + +- Unpacker may unpack wrong uint32 value on 32bit or LLP64 environment. + (#101) +- Build failed on Windows Python 2.7. + +# 0.4.2 + +Release Date: 2014-03-26 + +## Incompatible Changes + +## Changes + +## Bugs fixed + +- Unpacker doesn't increment refcount of ExtType hook. +- Packer raises no exception for inputs doesn't fit to msgpack format. + +# 0.4.1 + +Release Date: 2014-02-17 + +## Incompatible Changes + +## Changes + +- fallback.Unpacker.feed() supports bytearray. + +## Bugs fixed + +- Unpacker doesn't increment refcount of hooks. Hooks may be GCed while + unpacking. +- Unpacker may read unfilled internal buffer. + +# 0.4.0 + +Release Date: 2013-10-21 + +## Incompatible Changes + +- Raises TypeError instead of ValueError when packer receives + unsupported type. + +## Changes + +- Support New msgpack spec. + +# 0.3.0 + +## Incompatible Changes + +- Default value of `use_list` is `True` for now. (It was `False` for + 0.2.x) You should pass it explicitly for compatibility to 0.2.x. +- `Unpacker.unpack()` and some unpack methods now raise `OutOfData` instead of + `StopIteration`. `StopIteration` is used for iterator protocol only. + +## Changes + +- Pure Python fallback module is added. (thanks to bwesterb) +- Add `.skip()` method to `Unpacker` (thanks to jnothman) +- Add capturing feature. You can pass the writable object to + `Unpacker.unpack()` as a second parameter. +- Add `Packer.pack_array_header` and `Packer.pack_map_header`. These + methods only pack header of each type. +- Add `autoreset` option to `Packer` (default: True). Packer doesn't + return packed bytes and clear internal buffer. +- Add `Packer.pack_map_pairs`. It packs sequence of pair to map type. + +# 0.2.4 + +Release Date: 2012-12-22 + +## Bugs fixed + +- Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39) + +# 0.2.3 + +Release Date: 2012-12-11 + +## Changes + +- Warn when use_list is not specified. It's default value will be + changed in 0.3. + +## Bugs fixed + +- Can't pack subclass of dict. + +# 0.2.2 + +Release Date: 2012-09-21 + +## Changes + +- Add `use_single_float` option to `Packer`. When it is true, packs + float object in single precision format. + +## Bugs fixed + +- `unpack()` didn't restores gc state when it called with gc disabled. + `unpack()` doesn't control gc now instead of restoring gc state + collectly. User can control gc state when gc cause performance issue. +- `Unpacker`'s `read_size` option didn't used. + +# 0.2.1 + +Release Date: 2012-08-20 + +## Changes + +- Add `max_buffer_size` parameter to Unpacker. It limits internal buffer + size and allows unpack data from untrusted source safely. +- Unpacker's buffer reallocation algorithm is less greedy now. It cause + performance decrease in rare case but memory efficient and don't + allocate than `max_buffer_size`. + +## Bugs fixed + +- Fix msgpack didn't work on SPARC Solaris. It was because choosing + wrong byteorder on compilation time. Use `sys.byteorder` to get + correct byte order. Very thanks to Chris Casey for giving test + environment to me. + +# 0.2.0 + +Release Date: 2012-06-27 + +## Changes + +- Drop supporting Python 2.5 and unify tests for Py2 and Py3. +- Use new version of msgpack-c. It packs correctly on big endian + platforms. +- Remove deprecated packs and unpacks API. + +## Bugs fixed + +- \#8 Packing subclass of dict raises TypeError. (Thanks to Steeve + Morin.) + +# 0.1.13 + +Release Date: 2012-04-21 + +## New + +- Don't accept subtype of list and tuple as msgpack list. (Steeve Morin) + It allows customize how it serialized with `default` argument. + +## Bugs fixed + +- Fix wrong error message. (David Wolever) +- Fix memory leak while unpacking when `object_hook` or `list_hook` is + used. (Steeve Morin) + +## Other changes + +- setup.py works on Python 2.5 (Steffen Siering) +- Optimization for serializing dict. + +# 0.1.12 + +Release Date: 2011-12-27 + +## Bugs fixed + +- Re-enable packs/unpacks removed at 0.1.11. It will be removed when 0.2 + is released. + +# 0.1.11 + +Release Date: 2011-12-26 + +## Bugs fixed + +- Include test code for Python3 to sdist. (Johan Bergström) +- Fix compilation error on MSVC. (davidgaleano) + +# 0.1.10 + +Release Date: 2011-08-22 + +## New feature + +- Add `encoding` and `unicode_errors` option to packer and unpacker. + When this option is specified, (un)packs unicode object instead of + bytes. This enables using msgpack as a replacement of json. (tailhook) + +# 0.1.9 + +Release Date: 2011-01-29 + +## New feature + +- `use_list` option is added to unpack(b) like Unpacker. (Use keyword + argument because order of parameters are different) + +## Bugs fixed + +- Fix typo. +- Add MemoryError check. + +# 0.1.8 + +Release Date: 2011-01-10 + +## New feature + +- Support `loads` and `dumps` aliases for API compatibility with + simplejson and pickle. +- Add *object_hook* and *list_hook* option to unpacker. It allows you to + hook unpacking mapping type and array type. +- Add *default* option to packer. It allows you to pack unsupported + types. +- unpacker accepts (old) buffer types. + +## Bugs fixed + +- Fix segv around `Unpacker.feed` or `Unpacker(file)`. + +# 0.1.7 + +Release Date: 2010-11-02 + +## New feature + +- Add *object_hook* and *list_hook* option to unpacker. It allows you to + hook unpacking mapping type and array type. +- Add *default* option to packer. It allows you to pack unsupported + types. +- unpacker accepts (old) buffer types. + +## Bugs fixed + +- Compilation error on win32. diff --git a/ChangeLog.rst b/ChangeLog.rst deleted file mode 100644 index beeab15c..00000000 --- a/ChangeLog.rst +++ /dev/null @@ -1,625 +0,0 @@ -1.1.2 -===== - -Release Date: 2025-10-08 - -This release does not change source code. It updates only building wheels: - -* Update Cython to v3.1.4 -* Update cibuildwheel to v3.2.0 -* Drop Python 3.8 -* Add Python 3.14 -* Add windows-arm - -1.1.1 -===== - -Release Date: 2025-06-13 - -* No change from 1.1.1rc1. - -1.1.1rc1 -======== - -Release Date: 2025-06-06 - -* Update Cython to 3.1.1 and cibuildwheel to 2.23.3. - -1.1.0 -===== - -Release Date: 2024-09-10 - -* use ``PyLong_*`` instead of ``PyInt_*`` for compatibility with - future Cython. (#620) - -1.1.0rc2 -======== - -Release Date: 2024-08-19 - -* Update Cython to 3.0.11 for better Python 3.13 support. -* Update cibuildwheel to 2.20.0 to build Python 3.13 wheels. - -1.1.0rc1 -======== - -Release Date: 2024-05-07 - -* Update Cython to 3.0.10 to reduce C warnings and future support for Python 3.13. -* Stop using C++ mode in Cython to reduce compile error on some compilers. -* ``Packer()`` has ``buf_size`` option to specify initial size of - internal buffer to reduce reallocation. -* The default internal buffer size of ``Packer()`` is reduced from - 1MiB to 256KiB to optimize for common use cases. Use ``buf_size`` - if you are packing large data. -* ``Timestamp.to_datetime()`` and ``Timestamp.from_datetime()`` become - more accurate by avoiding floating point calculations. (#591) -* The Cython code for ``Unpacker`` has been slightly rewritten for maintainability. -* The fallback implementation of ``Packer()`` and ``Unpacker()`` now uses keyword-only - arguments to improve compatibility with the Cython implementation. - -1.0.8 -===== - -Release Date: 2024-03-01 - -* Update Cython to 3.0.8. This fixes memory leak when iterating - ``Unpacker`` object on Python 3.12. -* Do not include C/Cython files in binary wheels. - - -1.0.7 -===== - -Release Date: 2023-09-28 - -* Fix build error of extension module on Windows. (#567) -* ``setup.py`` doesn't skip build error of extension module. (#568) - - -1.0.6 -===== - -Release Date: 2023-09-21 - -.. note:: - v1.0.6 Wheels for Windows don't contain extension module. - Please upgrade to v1.0.7 or newer. - -* Add Python 3.12 wheels (#517) -* Remove Python 2.7, 3.6, and 3.7 support - - -1.0.5 -===== - -Release Date: 2023-03-08 - -* Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514) -* Add Python 3.11 wheels (#517) -* fallback: Fix packing multidimensional memoryview (#527) - -1.0.4 -===== - -Release Date: 2022-06-03 - -* Support Python 3.11 (beta). -* Don't define `__*_ENDIAN__` macro on Unix. by @methane in https://github.com/msgpack/msgpack-python/pull/495 -* Use PyFloat_Pack8() on Python 3.11a7 by @vstinner in https://github.com/msgpack/msgpack-python/pull/499 -* Fix Unpacker max_buffer_length handling by @methane in https://github.com/msgpack/msgpack-python/pull/506 - -1.0.3 -===== - -Release Date: 2021-11-24 JST - -* Fix Docstring (#459) -* Fix error formatting (#463) -* Improve error message about strict_map_key (#485) - -1.0.2 -===== - -* Fix year 2038 problem regression in 1.0.1. (#451) - -1.0.1 -===== - -* Add Python 3.9 and linux/arm64 wheels. (#439) -* Fixed Unpacker.tell() after read_bytes() (#426) -* Fixed unpacking datetime before epoch on Windows (#433) -* Fixed fallback Packer didn't check DateTime.tzinfo (#434) - -1.0.0 -===== - -Release Date: 2020-02-17 - -* Remove Python 2 support from the ``msgpack/_cmsgpack``. - ``msgpack/fallback`` still supports Python 2. -* Remove ``encoding`` option from the Packer and Unpacker. -* Unpacker: The default value of ``max_buffer_size`` is changed to 100MiB. -* Unpacker: ``strict_map_key`` is True by default now. -* Unpacker: String map keys are interned. -* Drop old buffer protocol support. -* Support Timestamp type. -* Support serializing and decerializing ``datetime`` object - with tzinfo. -* Unpacker: ``Fix Unpacker.read_bytes()`` in fallback implementation. (#352) - - -0.6.2 -===== - -Release Date: 2019-09-20 - -* Support Python 3.8. -* Update Cython to 0.29.13 for support Python 3.8. -* Some small optimizations. - - -0.6.1 -====== - -Release Date: 2019-01-25 - -This release is for mitigating pain caused by v0.6.0 reduced max input limits -for security reason. - -* ``unpackb(data)`` configures ``max_*_len`` options from ``len(data)``, - instead of static default sizes. - -* ``Unpacker(max_buffer_len=N)`` configures ``max_*_len`` options from ``N``, - instead of static default sizes. - -* ``max_bin_len``, ``max_str_len``, and ``max_ext_len`` are deprecated. - Since this is minor release, it's document only deprecation. - - -0.6.0 -====== - -Release Date: 2018-11-30 - -This release contains some backward incompatible changes for security reason (DoS). - -Important changes ------------------ - -* unpacker: Default value of input limits are smaller than before to avoid DoS attack. - If you need to handle large data, you need to specify limits manually. (#319) - -* Unpacker doesn't wrap underlying ``ValueError`` (including ``UnicodeError``) into - ``UnpackValueError``. If you want to catch all exception during unpack, you need - to use ``try ... except Exception`` with minimum try code block. (#323, #233) - -* ``PackValueError`` and ``PackOverflowError`` are also removed. You need to catch - normal ``ValueError`` and ``OverflowError``. (#323, #233) - -* Unpacker has ``strict_map_key`` option now. When it is true, only bytes and str - (unicode in Python 2) are allowed for map keys. It is recommended to avoid - hashdos. Default value of this option is False for backward compatibility reason. - But it will be changed True in 1.0. (#296, #334) - -Other changes -------------- - -* Extension modules are merged. There is ``msgpack._cmsgpack`` instead of - ``msgpack._packer`` and ``msgpack._unpacker``. (#314, #328) - -* Add ``Unpacker.getbuffer()`` method. (#320) - -* unpacker: ``msgpack.StackError`` is raised when input data contains too - nested data. (#331) - -* unpacker: ``msgpack.FormatError`` is raised when input data is not valid - msgpack format. (#331) - - -0.5.6 -====== - -* Fix fallback.Unpacker.feed() dropped unused data from buffer (#287) -* Resurrect fallback.unpack() and _unpacker.unpack(). - They were removed at 0.5.5 but it breaks backward compatibility. (#288, #290) - -0.5.5 -====== - -* Fix memory leak in pure Python Unpacker.feed() (#283) -* Fix unpack() didn't support `raw` option (#285) - -0.5.4 -====== - -* Undeprecate ``unicode_errors`` option. (#278) - -0.5.3 -====== - -* Fixed regression when passing ``unicode_errors`` to Packer but not ``encoding``. (#277) - -0.5.2 -====== - -* Add ``raw`` option to Unpacker. It is preferred way than ``encoding`` option. - -* Packer.pack() reset buffer on exception (#274) - - -0.5.1 -====== - -* Remove FutureWarning about use_bin_type option (#271) - -0.5.0 -====== - -There are some deprecations. Please read changes carefully. - -Changes -------- - -* Drop Python 2.6 and ~3.4 support. Python 2.7 and 3.5+ are supported. - -* Deprecate useless custom exceptions. Use ValueError instead of PackValueError, - Exception instead of PackException and UnpackException, etc... - See msgpack/exceptions.py - -* Add *strict_types* option to packer. It can be used to serialize subclass of - builtin types. For example, when packing object which type is subclass of dict, - ``default()`` is called. ``default()`` is called for tuple too. - -* Pure Python implementation supports packing memoryview object. - -* Support packing bytearray. - -* Add ``Unpacker.tell()``. And ``write_bytes`` option is deprecated. - - -Bugs fixed ----------- - -* Fixed zero length raw can't be decoded when encoding is specified. (#236) - - -0.4.8 -===== -:release date: 2016-07-29 - -Bugs fixed ----------- - -* Calling ext_hook with wrong length. (Only on Windows, maybe. #203) - - -0.4.7 -===== -:release date: 2016-01-25 - -Bugs fixed ----------- - -* Memory leak when unpack is failed - -Changes -------- - -* Reduce compiler warnings while building extension module -* unpack() now accepts ext_hook argument like Unpacker and unpackb() -* Update Cython version to 0.23.4 -* default function is called when integer overflow - - -0.4.6 -===== -:release date: 2015-03-13 - -Bugs fixed ----------- - -* fallback.Unpacker: Fix Data corruption when OutOfData. - This bug only affects "Streaming unpacking." - - -0.4.5 -===== -:release date: 2015-01-25 - -Incompatible Changes --------------------- - -Changes -------- - -Bugs fixed ----------- - -* Fix test failure on pytest 2.3. (by @ktdreyer) -* Fix typos in ChangeLog. (Thanks to @dmick) -* Improve README.rst (by @msabramo) - - -0.4.4 -===== -:release date: 2015-01-09 - -Incompatible Changes --------------------- - -Changes -------- - -Bugs fixed ----------- - -* Fix compile error. - -0.4.3 -===== -:release date: 2015-01-07 - -Incompatible Changes --------------------- - -Changes -------- - -Bugs fixed ----------- - -* Unpacker may unpack wrong uint32 value on 32bit or LLP64 environment. (#101) -* Build failed on Windows Python 2.7. - -0.4.2 -===== -:release date: 2014-03-26 - -Incompatible Changes --------------------- - -Changes -------- - -Bugs fixed ----------- - -* Unpacker doesn't increment refcount of ExtType hook. -* Packer raises no exception for inputs doesn't fit to msgpack format. - -0.4.1 -===== -:release date: 2014-02-17 - -Incompatible Changes --------------------- - -Changes -------- - -* fallback.Unpacker.feed() supports bytearray. - -Bugs fixed ----------- - -* Unpacker doesn't increment refcount of hooks. Hooks may be GCed while unpacking. -* Unpacker may read unfilled internal buffer. - -0.4.0 -===== -:release date: 2013-10-21 - -Incompatible Changes --------------------- - -* Raises TypeError instead of ValueError when packer receives unsupported type. - -Changes -------- - -* Support New msgpack spec. - - -0.3.0 -===== - -Incompatible Changes --------------------- - -* Default value of ``use_list`` is ``True`` for now. (It was ``False`` for 0.2.x) - You should pass it explicitly for compatibility to 0.2.x. -* `Unpacker.unpack()` and some unpack methods now raise `OutOfData` instead of - `StopIteration`. `StopIteration` is used for iterator protocol only. - -Changes -------- -* Pure Python fallback module is added. (thanks to bwesterb) -* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) -* Add capturing feature. You can pass the writable object to - ``Unpacker.unpack()`` as a second parameter. -* Add ``Packer.pack_array_header`` and ``Packer.pack_map_header``. - These methods only pack header of each type. -* Add ``autoreset`` option to ``Packer`` (default: True). - Packer doesn't return packed bytes and clear internal buffer. -* Add ``Packer.pack_map_pairs``. It packs sequence of pair to map type. - - - -0.2.4 -===== -:release date: 2012-12-22 - -Bugs fixed ----------- - -* Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39) - -0.2.3 -===== -:release date: 2012-12-11 - -Changes -------- -* Warn when use_list is not specified. It's default value will be changed in 0.3. - -Bugs fixed ----------- -* Can't pack subclass of dict. - -0.2.2 -===== -:release date: 2012-09-21 - -Changes -------- -* Add ``use_single_float`` option to ``Packer``. When it is true, packs float - object in single precision format. - -Bugs fixed ----------- -* ``unpack()`` didn't restores gc state when it called with gc disabled. - ``unpack()`` doesn't control gc now instead of restoring gc state collectly. - User can control gc state when gc cause performance issue. - -* ``Unpacker``'s ``read_size`` option didn't used. - -0.2.1 -===== -:release date: 2012-08-20 - -Changes -------- -* Add ``max_buffer_size`` parameter to Unpacker. It limits internal buffer size - and allows unpack data from untrusted source safely. - -* Unpacker's buffer reallocation algorithm is less greedy now. It cause performance - decrease in rare case but memory efficient and don't allocate than ``max_buffer_size``. - -Bugs fixed ----------- -* Fix msgpack didn't work on SPARC Solaris. It was because choosing wrong byteorder - on compilation time. Use ``sys.byteorder`` to get correct byte order. - Very thanks to Chris Casey for giving test environment to me. - - -0.2.0 -===== -:release date: 2012-06-27 - -Changes -------- -* Drop supporting Python 2.5 and unify tests for Py2 and Py3. -* Use new version of msgpack-c. It packs correctly on big endian platforms. -* Remove deprecated packs and unpacks API. - -Bugs fixed ----------- -* #8 Packing subclass of dict raises TypeError. (Thanks to Steeve Morin.) - - -0.1.13 -====== -:release date: 2012-04-21 - -New ---- -* Don't accept subtype of list and tuple as msgpack list. (Steeve Morin) - It allows customize how it serialized with ``default`` argument. - -Bugs fixed ----------- -* Fix wrong error message. (David Wolever) -* Fix memory leak while unpacking when ``object_hook`` or ``list_hook`` is used. - (Steeve Morin) - -Other changes -------------- -* setup.py works on Python 2.5 (Steffen Siering) -* Optimization for serializing dict. - - -0.1.12 -====== -:release date: 2011-12-27 - -Bugs fixed ----------- - -* Re-enable packs/unpacks removed at 0.1.11. It will be removed when 0.2 is released. - - -0.1.11 -====== -:release date: 2011-12-26 - -Bugs fixed ----------- - -* Include test code for Python3 to sdist. (Johan Bergström) -* Fix compilation error on MSVC. (davidgaleano) - - -0.1.10 -====== -:release date: 2011-08-22 - -New feature ------------ -* Add ``encoding`` and ``unicode_errors`` option to packer and unpacker. - When this option is specified, (un)packs unicode object instead of bytes. - This enables using msgpack as a replacement of json. (tailhook) - - -0.1.9 -===== -:release date: 2011-01-29 - -New feature ------------ -* ``use_list`` option is added to unpack(b) like Unpacker. - (Use keyword argument because order of parameters are different) - -Bugs fixed ----------- -* Fix typo. -* Add MemoryError check. - -0.1.8 -===== -:release date: 2011-01-10 - -New feature ------------ -* Support ``loads`` and ``dumps`` aliases for API compatibility with - simplejson and pickle. - -* Add *object_hook* and *list_hook* option to unpacker. It allows you to - hook unpacking mapping type and array type. - -* Add *default* option to packer. It allows you to pack unsupported types. - -* unpacker accepts (old) buffer types. - -Bugs fixed ----------- -* Fix segv around ``Unpacker.feed`` or ``Unpacker(file)``. - - -0.1.7 -===== -:release date: 2010-11-02 - -New feature ------------ -* Add *object_hook* and *list_hook* option to unpacker. It allows you to - hook unpacking mapping type and array type. - -* Add *default* option to packer. It allows you to pack unsupported types. - -* unpacker accepts (old) buffer types. - -Bugs fixed ----------- -* Compilation error on win32. diff --git a/pyproject.toml b/pyproject.toml index 4e4510b4..ebc6b50d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ Homepage = "https://msgpack.org/" Documentation = "https://msgpack-python.readthedocs.io/" Repository = "https://github.com/msgpack/msgpack-python/" Tracker = "https://github.com/msgpack/msgpack-python/issues" -Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" +Changelog = "https://github.com/msgpack/msgpack-python/blob/main/CHANGELOG.md" [tool.setuptools] # Do not install C/C++/Cython source files From e327a0e93bc742b19f6a407c1dbbc732a822ce17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 10:36:10 +0900 Subject: [PATCH 17/35] Bump the all-dependencies group with 2 updates (#684) --- .github/workflows/docs.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 4467ca8b..221f5d42 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index e2631ffa..6ed5a1c1 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-slim steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: ruff check run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b36670b..169f1fda 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index eb457277..a9c4b09e 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -27,7 +27,7 @@ jobs: name: Build wheels on ${{ matrix.os }}${{ matrix.name_suffix || '' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -40,7 +40,7 @@ jobs: - name: Set up QEMU for emulation if: matrix.cibw_archs == 'riscv64' - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 with: platforms: ${{ matrix.cibw_archs }} From d627d308ef6308dd86911e5f6de1a4bc56183aa3 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Tue, 2 Jun 2026 23:30:00 -0400 Subject: [PATCH 18/35] Bump sys.setrecursionlimit within test_nest_limit_1024 (#682) Fixes test failures with no-GIL interpreters --- test/test_limits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_limits.py b/test/test_limits.py index e3bab146..bb554d68 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -170,7 +170,7 @@ def test_nest_limit_1024(): # Temporarily raise Python's recursion limit so packing 1024 levels succeeds old_limit = sys.getrecursionlimit() - sys.setrecursionlimit(max(old_limit, 10000)) + sys.setrecursionlimit(max(old_limit, 30000)) try: packed = packb(d) result = unpackb(packed) From 7df7136e20933fd3a498bd58bff7975a24790516 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:22:40 +0900 Subject: [PATCH 19/35] Guard `Packer` buffer protocol hooks with Cython critical sections (#686) --- msgpack/_packer.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 2d92e8a4..277239d8 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -360,9 +360,11 @@ cdef class Packer: """ return memoryview(self) + @cython.critical_section def __getbuffer__(self, Py_buffer *buffer, int flags): PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags) self.exports += 1 + @cython.critical_section def __releasebuffer__(self, Py_buffer *buffer): self.exports -= 1 From 77395c19a50061238306df5ea10e67c5312ceff0 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:40:16 +0900 Subject: [PATCH 20/35] Harden `Unpacker.__init__` re-entry cleanup to prevent buffer/context leaks (#687) --- msgpack/_unpacker.pyx | 9 ++++++--- test/test_unpack.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 40d12291..4bfbe064 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -317,9 +317,6 @@ cdef class Unpacker: cdef Py_ssize_t max_buffer_size cdef uint64_t stream_offset - def __cinit__(self): - self.buf = NULL - def __dealloc__(self): unpack_clear(&self.ctx) PyMem_Free(self.buf) @@ -338,6 +335,12 @@ cdef class Unpacker: Py_ssize_t max_ext_len=-1): cdef const char *cerr=NULL + unpack_clear(&self.ctx) + unpack_init(&self.ctx) + if self.buf != NULL: + PyMem_Free(self.buf) + self.buf = NULL + self.object_hook = object_hook self.object_pairs_hook = object_pairs_hook self.list_hook = list_hook diff --git a/test/test_unpack.py b/test/test_unpack.py index b17c3c53..705c16a6 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -1,4 +1,6 @@ +import gc import sys +import weakref from io import BytesIO from pytest import mark, raises @@ -87,3 +89,37 @@ def test_unpacker_tell_read_bytes(): assert obj == unp assert pos == unpacker.tell() assert unpacker.read_bytes(n) == raw + + +@mark.skipif( + Unpacker.__module__ == "msgpack.fallback", + reason="specific to C extension reinit leak", +) +def test_unpacker_reinit_clears_partial_state(): + refs = [] + + class Marker: + pass + + def hook(code, data): + obj = Marker() + refs.append(weakref.ref(obj)) + return obj + + unpacker = Unpacker(ext_hook=hook, strict_map_key=False) + # Keep parser state mid-map with a live key object from ext_hook. + # Encodes: [ {ExtType(1, b"a"): } ]. + unpacker.feed(b"\x91\x81\xd4\x01a") + with raises(OutOfData): + unpacker.unpack() + assert len(refs) == 1 + assert refs[0]() is not None + + unpacker.__init__() + gc.collect() + assert refs[0]() is None + with raises(OutOfData): + unpacker.unpack() + + unpacker.feed(packb({"a": 1})) + assert unpacker.unpack() == {"a": 1} From 5eb57e15455f5dd25001e97a7ef1170911dee104 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 3 Jun 2026 17:50:51 +0900 Subject: [PATCH 21/35] release v1.2.0rc1 (#681) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 16 ++++++++++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74e6eb47..63cfcce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +# 1.2.0 + +Release Date: TBD + +- Support free threaded Python. #654, #686 +- Dropped support for Python 3.9. #656 +- Fix missing error checks in C code. #665, #666, #667, #672 +- Fix `strict_map_key` option didn't work for `object_pairs_hook`. #673 +- Increase DEFAULT_RECURSE_LIMIT of Unpacker to 1024. #676 +- Fix memory leak when Unpacker returns error for invalid input. #671 +- Fix `Packer.pack_ext_type()` ignored `autoreset` option. #663 +- Fix `Timestamp.from_datetime()` returning wrong value for pre-epoch datetimes. #662 +- Fix use-after-free in `unpackb()` and `Unpacker.unpack()` for non-contiguous input. #677 +- Fix possible memory leak when calling `Unpacker.__init__()` several times. #687 + + # 1.1.2 Release Date: 2025-10-08 diff --git a/msgpack/__init__.py b/msgpack/__init__.py index f3266b70..f7346d0f 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 2) -__version__ = "1.1.2" +version = (1, 2, 0) +__version__ = "1.2.0rc1" if os.environ.get("MSGPACK_PUREPYTHON"): From cdde1b0d6336caff4e82310c3c7d2b4c73f22d59 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 3 Jun 2026 13:20:15 +0100 Subject: [PATCH 22/35] Wheels CI hangs for MacOS Intel (#689) - Fix wheels generation for MacOS Intel, which is hanging due to gha VM brownout: https://github.com/msgpack/msgpack-python/actions/runs/26874183762/job/79271522595 - Run tests on MacOS Intel and on Ubuntu ARM --- .github/workflows/test.yml | 8 +++++++- .github/workflows/wheel.yml | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 169f1fda..bc101aa7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,13 @@ jobs: test: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] + os: + - ubuntu-latest + - ubuntu-24.04-arm + - windows-latest + - windows-11-arm + - macos-15-intel + - macos-latest py: ["3.15", "3.15t", "3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] exclude: - os: windows-11-arm diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index a9c4b09e..fb786883 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -11,13 +11,12 @@ jobs: build_wheels: strategy: matrix: - # macos-13 is for intel include: - os: ubuntu-24.04 - os: ubuntu-24.04-arm - os: windows-latest - os: windows-11-arm - - os: macos-13 + - os: macos-15-intel - os: macos-latest - os: ubuntu-24.04 cibw_archs: riscv64 From 97ba6ca0d29c1e8a5b30112f13f37a4d6b3c4492 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 3 Jun 2026 22:17:26 +0900 Subject: [PATCH 23/35] skip ci: remove unneeded CIBW_SKIP option --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index fb786883..56b31f03 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -48,7 +48,7 @@ jobs: env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64" + CIBW_SKIP: "cp38-* cp39-* cp310-win_arm64" CIBW_ARCHS: ${{ matrix.cibw_archs || 'auto' }} - name: Build sdist From c410a388c58d55091736e510cad064977a12cb8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:54:05 +0900 Subject: [PATCH 24/35] Bump pypa/cibuildwheel from 3.4.1 to 4.0.0 (#691) --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 56b31f03..a3d80b5c 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -44,7 +44,7 @@ jobs: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@8d2b08b68458a16aeb24b64e68a09ab1c8e82084 # v3.4.1 + uses: pypa/cibuildwheel@f03ac7617d6cff873ccf24cc0d567ef5ba5a9e6d # v4.0.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From 11ed0a5110c1920b85b7d24d9414f480983e0f16 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 11 Jun 2026 12:29:41 +0900 Subject: [PATCH 25/35] release v1.2.0 (#692) --- CHANGELOG.md | 2 +- msgpack/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63cfcce1..a29b4b2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # 1.2.0 -Release Date: TBD +Release Date: 2026-06-11 - Support free threaded Python. #654, #686 - Dropped support for Python 3.9. #656 diff --git a/msgpack/__init__.py b/msgpack/__init__.py index f7346d0f..f053abd0 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -5,7 +5,7 @@ from .ext import ExtType, Timestamp version = (1, 2, 0) -__version__ = "1.2.0rc1" +__version__ = "1.2.0" if os.environ.get("MSGPACK_PUREPYTHON"): From 0f4f350b6f3e80f24750422673424cce5d96e15d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 13:22:58 +0900 Subject: [PATCH 26/35] Bump pypa/cibuildwheel from 4.0.0 to 4.1.0 in the all-dependencies group (#694) --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index a3d80b5c..92fce016 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -44,7 +44,7 @@ jobs: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@f03ac7617d6cff873ccf24cc0d567ef5ba5a9e6d # v4.0.0 + uses: pypa/cibuildwheel@294735312765b09d24a2fbec22660ce817587d55 # v4.1.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From 2c56ddb5d0025ed481d962c0f5d62d19dec7476d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 00:13:13 +0900 Subject: [PATCH 27/35] Merge commit from fork * fix Unpacker crash after unpack failure. * fixup --- msgpack/_unpacker.pyx | 8 ++++++-- msgpack/unpack_template.h | 22 +++++++++++++++------- requirements.txt | 3 ++- test/test_except.py | 26 ++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 4bfbe064..e0cdde43 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -51,6 +51,7 @@ cdef extern from "unpack.h": execute_fn unpack_skip execute_fn read_array_header execute_fn read_map_header + void unpack_init(unpack_context* ctx) object unpack_data(unpack_context* ctx) void unpack_clear(unpack_context* ctx) @@ -197,6 +198,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, if off < buf_len: raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj + unpack_clear(&ctx) if ret == 0: raise ValueError("Unpack failed: incomplete input") @@ -475,7 +477,7 @@ cdef class Unpacker: obj = unpack_data(&self.ctx) unpack_init(&self.ctx) return obj - elif ret == 0: + if ret == 0: if self.file_like is not None: self.read_from_file() continue @@ -483,7 +485,9 @@ cdef class Unpacker: raise StopIteration("No more data to unpack.") else: raise OutOfData("No more data to unpack.") - elif ret == -2: + + unpack_clear(&self.ctx) + if ret == -2: raise FormatError elif ret == -3: raise StackError diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 42306618..ab5887ae 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -72,15 +72,14 @@ static inline PyObject* unpack_data(unpack_context* ctx) static inline void unpack_clear(unpack_context *ctx) { - unsigned int i; - for (i = 1; i < ctx->top; i++) { - Py_CLEAR(ctx->stack[i].obj); + for (unsigned int i = 0; i < ctx->top; i++) { /* map_key holds a live reference only while waiting for the value */ if (ctx->stack[i].ct == CT_MAP_VALUE) { Py_CLEAR(ctx->stack[i].map_key); } + Py_CLEAR(ctx->stack[i].obj); } - Py_CLEAR(ctx->stack[0].obj); + unpack_init(ctx); } static inline int unpack_execute(bool construct, unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) @@ -200,7 +199,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char case 0xd5: // fixext 2 case 0xd6: // fixext 4 case 0xd7: // fixext 8 - again_fixed_trail_if_zero(ACS_EXT_VALUE, + again_fixed_trail_if_zero(ACS_EXT_VALUE, (1 << (((unsigned int)*p) & 0x03))+1, _ext_zero); case 0xd8: // fixext 16 @@ -344,6 +343,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char goto _header_again; case CT_MAP_VALUE: if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } + c->map_key = NULL; if(++c->count == c->size) { obj = c->obj; if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } @@ -406,10 +406,18 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char #undef start_container static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { - return unpack_execute(1, ctx, data, len, off); + int ret = unpack_execute(1, ctx, data, len, off); + if (ret == -1) { + unpack_clear(ctx); + } + return ret; } static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { - return unpack_execute(0, ctx, data, len, off); + int ret = unpack_execute(0, ctx, data, len, off); + if (ret == -1) { + unpack_clear(ctx); + } + return ret; } #define unpack_container_header read_array_header diff --git a/requirements.txt b/requirements.txt index 9e4643b6..7991e3f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -Cython==3.2.1 +cython==3.2.5 setuptools==78.1.1 +pytest build diff --git a/test/test_except.py b/test/test_except.py index c04b110d..a3bf4675 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -33,6 +33,22 @@ def hook(obj): object_pairs_hook=hook, ) + up = Unpacker(object_hook=hook) + + def up_unpack(x): + up.feed(x) + return up.unpack() + + raises(DummyException, up_unpack, packb({})) + raises(DummyException, up_unpack, packb({"fizz": "buzz"})) + raises(DummyException, up_unpack, packb({"fizz": "buzz"})) + raises(DummyException, up_unpack, packb({"fizz": {"buzz": "spam"}})) + raises( + DummyException, + up_unpack, + packb({"fizz": {"buzz": "spam"}}), + ) + def test_raise_from_list_hook(): def hook(lst: list) -> list: @@ -138,3 +154,13 @@ def test_strict_map_key_with_object_pairs_hook(): packed = packb(valid, use_bin_type=True) result = unpackb(packed, raw=False, strict_map_key=True, object_pairs_hook=list) assert result == [("key", "value")] + + +def test_unpacker_should_not_crash_after_exception(): + up = Unpacker() # default: strict_map_key=True + up.feed(b"\x83\x73\xc4\x00") # fixmap(3): int key (rejected) + empty bin8 + try: + up.unpack() # ValueError: int is not allowed for map key ... + except Exception: + pass + up.skip() # SIGSEGV (resumes from a corrupt parser context) From 448d43f5dcca7b3a2b0810e161af0a6431d90071 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 00:33:29 +0900 Subject: [PATCH 28/35] release v1.2.1 (#698) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 9 +++++++++ README.md | 6 +++++- msgpack/__init__.py | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a29b4b2f..b8fbd8ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# 1.2.1 + +Release Date: 2026-06-19 + +Fix a segfault when calling `Unpacker.unpack()` or `Unpacker.skip()` after an unpacking failure. +But note that reusing the same `Unpacker` instance after an unpacking failure is not supported. +Please create a new `Unpacker` instance instead. GHSA-6v7p-g79w-8964 + + # 1.2.0 Release Date: 2026-06-11 diff --git a/README.md b/README.md index 223742dd..e66d4549 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,10 @@ for unpacked in unpacker: print(unpacked) ``` +> [!IMPORTANT] +> If `Unpacker.unpack()` stops with an exception other than `OutOfData`, that `Unpacker` cannot be reused. +> Create a new `Unpacker` when reading another stream. + ### Packing/unpacking of custom data types @@ -220,7 +224,7 @@ When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` be * The extension module no longer supports Python 2. The pure Python implementation (`msgpack.fallback`) is used for Python 2. - + * msgpack 1.0.6 drops official support of Python 2.7, as pip and GitHub Action "setup-python" no longer supports Python 2.7. diff --git a/msgpack/__init__.py b/msgpack/__init__.py index f053abd0..eeb85afa 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 2, 0) -__version__ = "1.2.0" +version = (1, 2, 1) +__version__ = "1.2.1" if os.environ.get("MSGPACK_PUREPYTHON"): From f1a170234a95dd64d4c0dee398d16d4a011e90ea Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 18:49:02 +0900 Subject: [PATCH 29/35] fix docstring for read_size (#700) fix #697 --- .gitignore | 1 + msgpack/_unpacker.pyx | 2 +- msgpack/fallback.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 341be631..31d7d6e1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ msgpack/*.cpp /tags /docs/_build .cache +uv.lock diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e0cdde43..26fb377e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -222,7 +222,7 @@ cdef class Unpacker: If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) + Used as `file_like.read(read_size)`. Must be equal to or smaller than *max_buffer_size*. :param bool use_list: If true, unpack msgpack array to Python list. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 860d94ac..824f59d5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -137,7 +137,7 @@ class Unpacker: If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) + Used as `file_like.read(read_size)`. Must be equal to or smaller than *max_buffer_size*. :param bool use_list: If true, unpack msgpack array to Python list. From d13418061d00e10964394bda555247727f85ce28 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 20:48:27 +0900 Subject: [PATCH 30/35] unpacker: fix silent datetime truncation with datetime=3 (#701) --- msgpack/unpack.h | 9 ++++++++- test/test_timestamp.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 0f9ffc05..55cfdab9 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -283,6 +283,7 @@ static int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timest ts->tv_sec = _msgpack_load64(int64_t, buf + 4); return 0; default: + PyErr_Format(PyExc_ValueError, "invalid timestamp data (length %u)", buflen); return -1; } } @@ -336,12 +337,18 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos else if (u->timestamp == 3) { // datetime // Calculate datetime using epoch + delta // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps + int64_t days = ts.tv_sec / (24*3600); + if (days < INT_MIN || days > INT_MAX) { + PyErr_Format(PyExc_OverflowError, + "days=%lld; too large to convert to C int", days); + return -1; + } PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType); if (epoch == NULL) { return -1; } - PyObject* d = PyDelta_FromDSU(ts.tv_sec/(24*3600), ts.tv_sec%(24*3600), ts.tv_nsec / 1000); + PyObject* d = PyDelta_FromDSU((int)days, ts.tv_sec%(24*3600), ts.tv_nsec / 1000); if (d == NULL) { Py_DECREF(epoch); return -1; diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 6dfe3689..7c8e3e83 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -176,3 +176,9 @@ def test_pack_datetime_without_tzinfo(): packed = msgpack.packb(dt, datetime=True) unpacked = msgpack.unpackb(packed, timestamp=3) assert unpacked == dt + + +def test_too_large_timestamp(): + # When timestamp64 is too large, conversion to datetime fails due to int64 -> int32 conversion. + # https://github.com/msgpack/msgpack-python/issues/696 + print(msgpack.unpackb(b"\xd7\xff" + b"\x00" * 8, timestamp=3)) From 7082130739d640134e961a384d33ca9939e1f969 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 21:45:06 +0900 Subject: [PATCH 31/35] run test with debug build (#702) --- .github/workflows/test_debug.yml | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/test_debug.yml diff --git a/.github/workflows/test_debug.yml b/.github/workflows/test_debug.yml new file mode 100644 index 00000000..8aeec640 --- /dev/null +++ b/.github/workflows/test_debug.yml @@ -0,0 +1,62 @@ +name: Run tests with debug Python +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + env: + PYTHON_VERSION: 3.14.6 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: /opt/python-debug + key: python-debug-${{ runner.os }}-${{ env.PYTHON_VERSION }} + + - name: Install build dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -y build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev curl libncursesw5-dev \ + xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev + + - name: Build Python with Py_DEBUG + if: steps.cache.outputs.cache-hit != 'true' + run: | + wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz + tar -xzf Python-${PYTHON_VERSION}.tgz + cd Python-${PYTHON_VERSION} + + ./configure --with-pydebug --prefix=/opt/python-debug + make -j4 + sudo make install + + - name: Create venv from debug Python + run: | + PYDBG_BIN="/opt/python-debug/bin/python3" + "$PYDBG_BIN" -m venv .venv + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV" + + - name: Prepare + run: | + python -m pip install -r requirements.txt pytest + + - name: Build + run: | + make cython + pip install . + + - name: Test (C extension) + run: | + pytest -v test + + - name: Test (pure Python fallback) + run: | + MSGPACK_PUREPYTHON=1 pytest -v test From cf3fd2b0617d50f60d331490f91110721cebaef6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 Jun 2026 15:13:45 +0900 Subject: [PATCH 32/35] fix: add reentrant guard to Unpacker.feed() (#704) fix ##695 --- msgpack/_unpacker.pyx | 71 +++++++++++++++++++++++++------------------ test/test_unpack.py | 19 ++++++++++++ 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 26fb377e..f758c0c4 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -305,6 +305,8 @@ cdef class Unpacker: Raises ``OutOfData`` when *packed* is incomplete. Raises ``FormatError`` when *packed* is not valid msgpack. Raises ``StackError`` when *packed* contains too nested. + Raises ``RuntimeError`` when ``feed()`` is called while unpacking + is in progress (e.g. from a hook). Other exceptions can be raised during unpacking. """ cdef unpack_context ctx @@ -318,6 +320,7 @@ cdef class Unpacker: cdef object unicode_errors cdef Py_ssize_t max_buffer_size cdef uint64_t stream_offset + cdef bint _unpacking def __dealloc__(self): unpack_clear(&self.ctx) @@ -381,6 +384,7 @@ cdef class Unpacker: self.buf_head = 0 self.buf_tail = 0 self.stream_offset = 0 + self._unpacking = False if unicode_errors is not None: self.unicode_errors = unicode_errors @@ -398,6 +402,11 @@ cdef class Unpacker: cdef char* buf cdef Py_ssize_t buf_len + if self._unpacking: + raise RuntimeError( + "Unpacker.feed() cannot be called while unpacking is in progress" + ) + if self.file_like is not None: raise AssertionError( "unpacker.feed() is not be able to use with `file_like`.") @@ -465,36 +474,40 @@ cdef class Unpacker: cdef object obj cdef Py_ssize_t prev_head - while 1: - prev_head = self.buf_head - if prev_head < self.buf_tail: - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - self.stream_offset += self.buf_head - prev_head - else: - ret = 0 - - if ret == 1: - obj = unpack_data(&self.ctx) - unpack_init(&self.ctx) - return obj - if ret == 0: - if self.file_like is not None: - self.read_from_file() - continue - if iter: - raise StopIteration("No more data to unpack.") + self._unpacking = True + try: + while 1: + prev_head = self.buf_head + if prev_head < self.buf_tail: + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + self.stream_offset += self.buf_head - prev_head else: - raise OutOfData("No more data to unpack.") - - unpack_clear(&self.ctx) - if ret == -2: - raise FormatError - elif ret == -3: - raise StackError - elif PyErr_Occurred(): - raise - else: - raise ValueError("Unpack failed: error = %d" % (ret,)) + ret = 0 + + if ret == 1: + obj = unpack_data(&self.ctx) + unpack_init(&self.ctx) + return obj + if ret == 0: + if self.file_like is not None: + self.read_from_file() + continue + if iter: + raise StopIteration("No more data to unpack.") + else: + raise OutOfData("No more data to unpack.") + + unpack_clear(&self.ctx) + if ret == -2: + raise FormatError + elif ret == -3: + raise StackError + elif PyErr_Occurred(): + raise + else: + raise ValueError("Unpack failed: error = %d" % (ret,)) + finally: + self._unpacking = False @cython.critical_section def read_bytes(self, Py_ssize_t nbytes): diff --git a/test/test_unpack.py b/test/test_unpack.py index 705c16a6..81bc976b 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -123,3 +123,22 @@ def hook(code, data): unpacker.feed(packb({"a": 1})) assert unpacker.unpack() == {"a": 1} + + +@mark.skipif( + Unpacker.__module__ == "msgpack.fallback", + reason="reentrant guard is implemented in C extension only", +) +def test_unpacker_reentrant_feed(): + import struct + + def ext_hook(code, data): + # re-entrant feed on the SAME unpacker, large enough to force a buffer realloc + up.feed(b"\xc0" * 100) + return 0 + + up = Unpacker(ext_hook=ext_hook, max_buffer_size=64 * 1024 * 1024) + # array(11): [ ExtType(code=5, data=b'A') (fires the re-entrant hook), then 10 more elements ] + up.feed(b"\xdc" + struct.pack(">H", 11) + b"\xd4\x05A" + b"\x2a" * 10) + with raises(RuntimeError): + up.unpack() From 33fec11eb4b72133f10d0640cc7209b5cc8ed660 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:14:03 +0900 Subject: [PATCH 33/35] Bump actions/checkout from 6.0.3 to 7.0.0 in the all-dependencies group (#705) --- .github/workflows/docs.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/test_debug.yml | 2 +- .github/workflows/wheel.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 221f5d42..7ae7943e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 6ed5a1c1..1ad0a1e6 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-slim steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: ruff check run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc101aa7..67c5e52e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/test_debug.yml b/.github/workflows/test_debug.yml index 8aeec640..0cfb2006 100644 --- a/.github/workflows/test_debug.yml +++ b/.github/workflows/test_debug.yml @@ -11,7 +11,7 @@ jobs: PYTHON_VERSION: 3.14.6 steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 92fce016..8234ab4e 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -26,7 +26,7 @@ jobs: name: Build wheels on ${{ matrix.os }}${{ matrix.name_suffix || '' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" From 951995ecca1f4428967fbbab36f147412d0a5214 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 Jun 2026 17:14:55 +0900 Subject: [PATCH 34/35] Improve error handling and reporting in unpacking functions (#707) --- msgpack/_unpacker.pyx | 6 +----- msgpack/unpack.h | 5 ----- msgpack/unpack_template.h | 25 +++---------------------- 3 files changed, 4 insertions(+), 32 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index f758c0c4..e0463617 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -46,7 +46,7 @@ cdef extern from "unpack.h": Py_ssize_t count ctypedef int (*execute_fn)(unpack_context* ctx, const char* data, - Py_ssize_t len, Py_ssize_t* off) except? -1 + Py_ssize_t len, Py_ssize_t* off) except -1 execute_fn unpack_construct execute_fn unpack_skip execute_fn read_array_header @@ -206,8 +206,6 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, raise FormatError elif ret == -3: raise StackError - elif PyErr_Occurred(): - raise else: raise ValueError("Unpack failed: error = %d" % (ret,)) @@ -502,8 +500,6 @@ cdef class Unpacker: raise FormatError elif ret == -3: raise StackError - elif PyErr_Occurred(): - raise else: raise ValueError("Unpack failed: error = %d" % (ret,)) finally: diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 55cfdab9..eb4330ac 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -40,11 +40,6 @@ struct unpack_context; typedef struct unpack_context unpack_context; typedef int (*execute_fn)(unpack_context *ctx, const char* data, Py_ssize_t len, Py_ssize_t* off); -static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) -{ - return NULL; -} - static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) { PyObject *p = PyLong_FromLong((long)d); diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index ab5887ae..797a2f91 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -35,11 +35,6 @@ struct unpack_context { unsigned int cs; unsigned int trail; unsigned int top; - /* - unpack_stack* stack; - unsigned int stack_size; - unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE]; - */ unpack_stack stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -49,22 +44,9 @@ static inline void unpack_init(unpack_context* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; - /* - ctx->stack = ctx->embed_stack; - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; - */ - ctx->stack[0].obj = unpack_callback_root(&ctx->user); + ctx->stack[0].obj = NULL; } -/* -static inline void unpack_destroy(unpack_context* ctx) -{ - if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { - free(ctx->stack); - } -} -*/ - static inline PyObject* unpack_data(unpack_context* ctx) { return (ctx)->stack[0].obj; @@ -94,9 +76,6 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char unsigned int cs = ctx->cs; unsigned int top = ctx->top; unpack_stack* stack = ctx->stack; - /* - unsigned int stack_size = ctx->stack_size; - */ unpack_user* user = &ctx->user; PyObject* obj = NULL; @@ -319,6 +298,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); default: + PyErr_Format(PyExc_RuntimeError, "Invalid state: %d", cs); goto _failed; } } @@ -355,6 +335,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char goto _header_again; default: + PyErr_Format(PyExc_RuntimeError, "Invalid container type: %u", c->ct); goto _failed; } From 2de627311fb17b1a942d052447c3777cd58b162d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 Jun 2026 18:17:22 +0900 Subject: [PATCH 35/35] test_unpack.py: add tests for ExtraData (#708) --- test/test_unpack.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/test/test_unpack.py b/test/test_unpack.py index 81bc976b..ae6ea10c 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -5,7 +5,15 @@ from pytest import mark, raises -from msgpack import ExtType, OutOfData, Unpacker, packb +from msgpack import ( + ExtraData, + ExtType, + OutOfData, + Unpacker, + packb, + unpack, + unpackb, +) def test_unpack_array_header_from_file(): @@ -142,3 +150,20 @@ def ext_hook(code, data): up.feed(b"\xdc" + struct.pack(">H", 11) + b"\xd4\x05A" + b"\x2a" * 10) with raises(RuntimeError): up.unpack() + + +def test_unpackb_raises_extra_data_with_trailing_bytes(): + packed = packb(42) + packb("trailing") + with raises(ExtraData) as exc_info: + unpackb(packed) + err = exc_info.value + assert err.unpacked == 42 + assert err.extra == packb("trailing") + + +def test_unpack_raises_extra_data_on_stream_with_trailing_bytes(): + stream = BytesIO(packb(100) + packb(200)) + with raises(ExtraData) as exc_info: + unpack(stream) + assert exc_info.value.unpacked == 100 + assert exc_info.value.extra == packb(200)