diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index c5459539..0a0a737d 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -11,15 +11,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '3.x' architecture: 'x64' - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Black Code Formatter run: | - pip install black - black --diff --check msgpack/ test/ setup.py + pip install black==22.3.0 + black -S --diff --check msgpack/ test/ setup.py diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml deleted file mode 100644 index 95c672b2..00000000 --- a/.github/workflows/linux.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Build Linux Wheels -on: - push: - pull_request: - create: - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Cythonize - shell: bash - run: | - pip install -U pip - pip -V - pip install -r requirements.txt - make cython - #python setup.py sdist - - - name: Build wheels - shell: bash - run: | - make linux-wheel - - - name: Install qemu-user-static for docker - shell: bash - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build arm64 wheels - shell: bash - run: | - make linux-arm64-wheel - - - name: Run test (3.8) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - - name: Set up Python 3.10 - uses: actions/setup-python@v1 - with: - python-version: "3.10" - - - name: Run test (3.10) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.9 - uses: actions/setup-python@v1 - with: - python-version: 3.9 - - - name: Run test (3.9) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Run test (3.7) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.6 - uses: actions/setup-python@v1 - with: - python-version: 3.6 - - - name: Run test (3.6) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: linux-wheels - path: ./dist/wheelhouse/ diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml deleted file mode 100644 index 85844e7a..00000000 --- a/.github/workflows/mac.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Build macOS Wheels -on: - push: - create: - -jobs: - build: - runs-on: macos-latest - - steps: - - name: Checkout - uses: actions/checkout@v1 - - # Python 3.9 - - name: Set up Python 3.9 - uses: actions/setup-python@v1 - with: - python-version: "3.9" - - - name: Cythonize - run: | - pip install -U pip - pip install -r requirements.txt - make cython - - - name: Build wheels - uses: pypa/cibuildwheel@v2.2.2 - env: - CIBW_ARCHS_MACOS: x86_64 universal2 - CIBW_SKIP: pp* - - - name: Run test - run: | - ls wheelhouse/ - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.10 - - name: Set up Python 3.10 - uses: actions/setup-python@v1 - with: - python-version: "3.10" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.8 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: "3.8" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.7 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: "3.7" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: macos-wheels - path: ./wheelhouse/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..d01d74cb --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,49 @@ +name: Run tests +on: + push: + branches: [main] + pull_request: + create: + +jobs: + test: + strategy: + matrix: + os: [ubuntu-20.04, windows-2022, macos-10.15] + py: ["3.11-dev", "3.10", "3.9", "3.8", "3.7", "3.6"] + + runs-on: ${{ matrix.os }} + name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.py }} + cache: "pip" + + - name: Build + shell: bash + run: | + pip install -r requirements.txt pytest + make cython + pip install . + + - name: Test (C extension) + shell: bash + run: | + pytest -v test + + - name: Test (pure Python fallback) + shell: bash + run: | + MSGPACK_PUREPYTHON=1 pytest -v test + + - name: Publish Wheels to TestPyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml new file mode 100644 index 00000000..d73898c2 --- /dev/null +++ b/.github/workflows/wheel.yml @@ -0,0 +1,57 @@ +name: Build Wheels +on: + push: + branches: [main] + create: + +jobs: + build_wheels: + strategy: + matrix: + os: [ubuntu-20.04, windows-2022, macos-10.15] + runs-on: ${{ matrix.os }} + name: Build wheels on ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + + - name: Set up Python 3.9 + uses: actions/setup-python@v3 + with: + python-version: 3.9 + cache: "pip" + + - name: Prepare + shell: bash + run: | + pip install -r requirements.txt + make cython + + - name: Build + uses: pypa/cibuildwheel@v2.6.0 + env: + CIBW_TEST_REQUIRES: "pytest" + CIBW_TEST_COMMAND: "pytest {package}/test" + CIBW_ARCHS_LINUX: auto aarch64 + CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_SKIP: pp* + + - name: Upload Wheels to artifact + uses: actions/upload-artifact@v1 + with: + name: Wheels + path: wheelhouse + + - name: Publish Wheels to TestPyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages_dir: wheelhouse + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml deleted file mode 100644 index 0ce50f58..00000000 --- a/.github/workflows/windows.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: Build and test windows wheels -on: - push: - pull_request: - create: - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - - runs-on: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Cythonize - shell: bash - run: | - pip install -U Cython - make cython - #python setup.py sdist - - - name: Python 3.6 (amd64) - env: - PYTHON: "py -3.6-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.6 (x86) - env: - PYTHON: "py -3.6-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.7 (amd64) - env: - PYTHON: "py -3.7-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.7 (x86) - env: - PYTHON: "py -3.7-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.8 (amd64) - env: - PYTHON: "py -3.8-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.8 (x86) - env: - PYTHON: "py -3.8-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.9 (amd64) - env: - PYTHON: "py -3.9-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.9 (x86) - env: - PYTHON: "py -3.9-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.10 (amd64) - env: - PYTHON: "py -3.10-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.10 (x86) - env: - PYTHON: "py -3.10-32" - shell: bash - run: | - ci/runtests.sh - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: win-wheels - path: ./dist diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4974d26c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,89 +0,0 @@ -version: ~> 1.0 -dist: xenial -language: python -cache: pip -arch: - - arm64 - -python: - # Available Python (PyPy) can be listed by: - # - # $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/ - - "3.6" - - "3.7" - - "3.8" - - "3.9-dev" - -_pure: &pure - install: - - pip install -U pip - - pip install -U pytest pytest-cov codecov - - pip install . - script: - - pytest --cov=msgpack -v test - -matrix: - include: - - name: 32bit build - arch: amd64 - language: python - services: - - docker - env: - - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 - install: - - pip install -U pip - - pip install -r requirements.txt - - make cython - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - - arch: arm64 - name: arm64 32bit build - language: python - services: - - docker - env: - - DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64 - install: - - pip install -U pip - - pip install -r requirements.txt - - make cython - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - - name: "Python 2 (fallback)" - python: "2.7" - <<: *pure - - - name: "pypy2.7" - arch: amd64 - python: "pypy2.7-7.1.1" - <<: *pure - - - name: "pypy3" - arch: amd64 - python: "pypy3.6-7.1.1" - <<: *pure - -install: - - pip install -U pip - - pip install -U pytest pytest-cov codecov - - pip install -r requirements.txt # Cython - - make cython - - pip install -e . - -script: - - python -c 'import sys; print(hex(sys.maxsize))' - - python -c 'from msgpack import _cmsgpack' - - pytest --cov=msgpack -v test - - MSGPACK_PUREPYTHON=x pytest --cov=msgpack -v test - -after_success: - - if [ -f .coverage ]; then - codecov; - fi - -# vim: sw=2 ts=2 diff --git a/ChangeLog.rst b/ChangeLog.rst index fc6df680..a11c8144 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +1.0.4 +===== + +Release Date: 2022-06-03 + +* Support Python 3.11 (beta). +* Don't define `__*_ENDIAN__` macro on Unix. by @methane in https://github.com/msgpack/msgpack-python/pull/495 +* Use PyFloat_Pack8() on Python 3.11a7 by @vstinner in https://github.com/msgpack/msgpack-python/pull/499 +* Fix Unpacker max_buffer_length handling by @methane in https://github.com/msgpack/msgpack-python/pull/506 + 1.0.3 ===== diff --git a/Makefile b/Makefile index b50fa80c..415dcfdd 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: cython .PHONY: black black: - black msgpack/ test/ setup.py + black -S msgpack/ test/ setup.py .PHONY: cython cython: @@ -31,8 +31,9 @@ clean: .PHONY: update-docker update-docker: - docker pull quay.io/pypa/manylinux1_i686 - docker pull quay.io/pypa/manylinux1_x86_64 + docker pull quay.io/pypa/manylinux2014_i686 + docker pull quay.io/pypa/manylinux2014_x86_64 + docker pull quay.io/pypa/manylinux2014_aarch64 .PHONY: linux-wheel linux-wheel: diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index f338e177..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,50 +0,0 @@ -environment: - matrix: - # For Python versions available on Appveyor, see - # http://www.appveyor.com/docs/installed-software#python - - PYTHON: "C:\\Python36" - -install: - # We need wheel installed to build wheels - - "%PYTHON%\\python.exe -m pip install -U pip" - - "%PYTHON%\\python.exe -m pip install -U cython" - - "%PYTHON%\\Scripts\\cython --cplus msgpack/_cmsgpack.pyx" - -build: off - -test_script: - # Put your test command here. - # Note that you must use the environment variable %PYTHON% to refer to - # the interpreter you're using - Appveyor does not do anything special - # to put the Python version you want to use on PATH. - - set PYTHON="C:\\Python27" - - ci\\runtests.bat - - set PYTHON="C:\\Python27-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python36" - - ci\\runtests.bat - - set PYTHON="C:\\Python36-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python37" - - ci\\runtests.bat - - set PYTHON="C:\\Python37-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python38" - - ci\\runtests.bat - - set PYTHON="C:\\Python38-x64" - - ci\\runtests.bat - -after_test: - # This step builds your wheels. - # Again, you need to use %PYTHON% to get the correct interpreter - -artifacts: - # bdist_wheel puts your built wheel in the dist directory - - path: dist\*.whl - -#on_success: -# You can use this step to upload your artifacts to a public website. -# See Appveyor's documentation for more details. Or you can simply -# access your wheels from the Appveyor "artifacts" tab for your build. - -# vim: set shiftwidth=2 diff --git a/ci/runtests.bat b/ci/runtests.bat deleted file mode 100644 index 4ae2f708..00000000 --- a/ci/runtests.bat +++ /dev/null @@ -1,9 +0,0 @@ -%PYTHON%\python.exe -m pip install -U pip wheel pytest -%PYTHON%\python.exe setup.py build_ext -i -%PYTHON%\python.exe setup.py install -%PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))" -%PYTHON%\python.exe -c "from msgpack import _cmsgpack" -%PYTHON%\python.exe setup.py bdist_wheel -%PYTHON%\python.exe -m pytest -v test -SET EL=%ERRORLEVEL% -exit /b %EL% diff --git a/ci/runtests.sh b/ci/runtests.sh deleted file mode 100644 index 5d87f696..00000000 --- a/ci/runtests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -ex -${PYTHON} -VV -${PYTHON} -m pip install setuptools wheel pytest -${PYTHON} setup.py build_ext -if -${PYTHON} -c "from msgpack import _cmsgpack" -${PYTHON} setup.py bdist_wheel -${PYTHON} -m pytest -v test diff --git a/msgpack/__init__.py b/msgpack/__init__.py index d6705e22..50710218 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,5 +1,4 @@ # coding: utf-8 -from ._version import version from .exceptions import * from .ext import ExtType, Timestamp @@ -7,6 +6,10 @@ import sys +version = (1, 0, 4) +__version__ = "1.0.4" + + if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: from .fallback import Packer, unpackb, Unpacker else: diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 27facc0a..8b06661e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -440,34 +440,30 @@ cdef class Unpacker(object): self.buf_size = buf_size self.buf_tail = tail + _buf_len - cdef read_from_file(self): - next_bytes = self.file_like_read( - min(self.read_size, - self.max_buffer_size - (self.buf_tail - self.buf_head) - )) + cdef int read_from_file(self) except -1: + cdef Py_ssize_t remains = self.max_buffer_size - (self.buf_tail - self.buf_head) + if remains <= 0: + raise BufferFull + + next_bytes = self.file_like_read(min(self.read_size, remains)) if next_bytes: self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) else: self.file_like = None + return 0 cdef object _unpack(self, execute_fn execute, bint iter=0): cdef int ret cdef object obj cdef Py_ssize_t prev_head - if self.buf_head >= self.buf_tail and self.file_like is not None: - self.read_from_file() - while 1: prev_head = self.buf_head - if prev_head >= self.buf_tail: - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") - - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - self.stream_offset += self.buf_head - prev_head + if prev_head < self.buf_tail: + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + self.stream_offset += self.buf_head - prev_head + else: + ret = 0 if ret == 1: obj = unpack_data(&self.ctx) diff --git a/msgpack/_version.py b/msgpack/_version.py deleted file mode 100644 index fb878b35..00000000 --- a/msgpack/_version.py +++ /dev/null @@ -1 +0,0 @@ -version = (1, 0, 3) diff --git a/msgpack/ext.py b/msgpack/ext.py index 4eb9dd65..25544c55 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -59,7 +59,7 @@ def __init__(self, seconds, nanoseconds=0): raise TypeError("seconds must be an interger") if not isinstance(nanoseconds, int_types): raise TypeError("nanoseconds must be an integer") - if not (0 <= nanoseconds < 10 ** 9): + if not (0 <= nanoseconds < 10**9): raise ValueError( "nanoseconds must be a non-negative integer less than 999999999." ) @@ -143,7 +143,7 @@ def from_unix(unix_sec): :type unix_float: int or float. """ seconds = int(unix_sec // 1) - nanoseconds = int((unix_sec % 1) * 10 ** 9) + nanoseconds = int((unix_sec % 1) * 10**9) return Timestamp(seconds, nanoseconds) def to_unix(self): @@ -161,7 +161,7 @@ def from_unix_nano(unix_ns): :param int unix_ns: Posix timestamp in nanoseconds. :rtype: Timestamp """ - return Timestamp(*divmod(unix_ns, 10 ** 9)) + return Timestamp(*divmod(unix_ns, 10**9)) def to_unix_nano(self): """Get the timestamp as a unixtime in nanoseconds. @@ -169,7 +169,7 @@ def to_unix_nano(self): :returns: posix timestamp in nanoseconds :rtype: int """ - return self.seconds * 10 ** 9 + self.nanoseconds + return self.seconds * 10**9 + self.nanoseconds def to_datetime(self): """Get the timestamp as a UTC datetime. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b27acb29..f560c7b5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -11,7 +11,6 @@ def dict_iteritems(d): return d.iteritems() - else: int_types = int unicode = str @@ -32,7 +31,6 @@ def _is_recursionerror(e): and e.args[0].startswith("maximum recursion depth exceeded") ) - else: def _is_recursionerror(e): @@ -68,7 +66,6 @@ def write(self, s): def getvalue(self): return self.builder.build() - else: USING_STRINGBUILDER = False from io import BytesIO as StringIO @@ -143,7 +140,6 @@ def _unpack_from(f, b, o=0): """Explicit type cast for legacy struct.unpack_from""" return struct.unpack_from(f, bytes(b), o) - else: _unpack_from = struct.unpack_from @@ -322,7 +318,7 @@ def __init__( self._buf_checkpoint = 0 if not max_buffer_size: - max_buffer_size = 2 ** 31 - 1 + max_buffer_size = 2**31 - 1 if max_str_len == -1: max_str_len = max_buffer_size if max_bin_len == -1: @@ -427,6 +423,8 @@ def _reserve(self, n, raise_outofdata=True): # Read from file remain_bytes = -remain_bytes + if remain_bytes + len(self._buffer) > self._max_buffer_size: + raise BufferFull while remain_bytes > 0: to_read_bytes = max(self._read_size, remain_bytes) read_data = self.file_like.read(to_read_bytes) @@ -804,20 +802,20 @@ def _pack( raise OverflowError("Integer value out of range") if check(obj, (bytes, bytearray)): n = len(obj) - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, unicode): obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("String is too large") self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("Memoryview is too large") self._pack_bin_header(n) return self._buffer.write(obj) @@ -899,7 +897,7 @@ def pack_map_pairs(self, pairs): return ret def pack_array_header(self, n): - if n >= 2 ** 32: + if n >= 2**32: raise ValueError self._pack_array_header(n) if self._autoreset: @@ -908,7 +906,7 @@ def pack_array_header(self, n): return ret def pack_map_header(self, n): - if n >= 2 ** 32: + if n >= 2**32: raise ValueError self._pack_map_header(n) if self._autoreset: diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 0e940b84..7d479b6d 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -568,7 +568,12 @@ static inline int msgpack_pack_float(msgpack_packer* x, float d) { unsigned char buf[5]; buf[0] = 0xca; + +#if PY_VERSION_HEX >= 0x030B00A7 + PyFloat_Pack4(d, (char *)&buf[1], 0); +#else _PyFloat_Pack4(d, &buf[1], 0); +#endif msgpack_pack_append_buffer(x, buf, 5); } @@ -576,7 +581,11 @@ static inline int msgpack_pack_double(msgpack_packer* x, double d) { unsigned char buf[9]; buf[0] = 0xcb; +#if PY_VERSION_HEX >= 0x030B00A7 + PyFloat_Pack8(d, (char *)&buf[1], 0); +#else _PyFloat_Pack8(d, &buf[1], 0); +#endif msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 9924b9c6..8b9fcc19 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -243,10 +243,20 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize _msgpack_load32(uint32_t,n)+1, _ext_zero); case CS_FLOAT: { - double f = _PyFloat_Unpack4((unsigned char*)n, 0); + double f; +#if PY_VERSION_HEX >= 0x030B00A7 + f = PyFloat_Unpack4((const char*)n, 0); +#else + f = _PyFloat_Unpack4((unsigned char*)n, 0); +#endif push_fixed_value(_float, f); } case CS_DOUBLE: { - double f = _PyFloat_Unpack8((unsigned char*)n, 0); + double f; +#if PY_VERSION_HEX >= 0x030B00A7 + f = PyFloat_Unpack8((const char*)n, 0); +#else + f = _PyFloat_Unpack8((unsigned char*)n, 0); +#endif push_fixed_value(_double, f); } case CS_UINT_8: push_fixed_value(_uint8, *(uint8_t*)n); diff --git a/pyproject.toml b/pyproject.toml index a9eb8aa9..195795f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=0.29.13", + "Cython~=0.29.30", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 180fe851..9f3c1a0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there -Cython~=0.29.13 +Cython~=0.29.30 + +# dev only tools. no need to add pyproject +black==22.3.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..1cb6ce36 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,32 @@ +[metadata] +name = msgpack +#version = attr: msgpack.__version__ +version = attr: msgpack.version +license = Apache 2.0 +author = Inada Naoki +author_email = songofacandy@gmail.com +description = MessagePack serializer +long_description = file: README.md +long_description_content_type = text/markdown +url = https://msgpack.org/ + +project_urls = + Documentation = https://msgpack-python.readthedocs.io/ + Source = https://github.com/msgpack/msgpack-python + Tracker = https://github.com/msgpack/msgpack-python/issues + +classifiers = + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + +[flake8] +max_line_length = 100 + diff --git a/setup.py b/setup.py index 01f125fb..9630cda0 100755 --- a/setup.py +++ b/setup.py @@ -4,10 +4,9 @@ import os import sys from glob import glob -from distutils.command.sdist import sdist from setuptools import setup, Extension - -from distutils.command.build_ext import build_ext +from setuptools.command.build_ext import build_ext +from setuptools.command.sdist import sdist PYPY = hasattr(sys, "pypy_version_info") @@ -65,12 +64,6 @@ def build_extension(self, ext): print(e) -exec(open("msgpack/_version.py").read()) - -version_str = ".".join(str(x) for x in version[:3]) -if len(version) > 3 and version[3] != "final": - version_str += version[3] - # Cython is required for sdist class Sdist(sdist): def __init__(self, *args, **kwargs): @@ -79,12 +72,10 @@ def __init__(self, *args, **kwargs): libraries = [] +macros = [] + if sys.platform == "win32": libraries.append("ws2_32") - -if sys.byteorder == "big": - macros = [("__BIG_ENDIAN__", "1")] -else: macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] @@ -101,39 +92,8 @@ def __init__(self, *args, **kwargs): del libraries, macros -desc = "MessagePack (de)serializer." -with io.open("README.md", encoding="utf-8") as f: - long_desc = f.read() -del f - setup( - name="msgpack", - author="Inada Naoki", - author_email="songofacandy@gmail.com", - version=version_str, cmdclass={"build_ext": BuildExt, "sdist": Sdist}, ext_modules=ext_modules, packages=["msgpack"], - description=desc, - long_description=long_desc, - long_description_content_type="text/markdown", - url="https://msgpack.org/", - project_urls={ - "Documentation": "https://msgpack-python.readthedocs.io/", - "Source": "https://github.com/msgpack/msgpack-python", - "Tracker": "https://github.com/msgpack/msgpack-python/issues", - }, - license="Apache 2.0", - classifiers=[ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - ], ) diff --git a/test/test_limits.py b/test/test_limits.py index 65e6bcc7..4314c2c0 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -16,12 +16,12 @@ def test_integer(): - x = -(2 ** 63) + x = -(2**63) assert unpackb(packb(x)) == x with pytest.raises(PackOverflowError): packb(x - 1) - x = 2 ** 64 - 1 + x = 2**64 - 1 assert unpackb(packb(x)) == x with pytest.raises(PackOverflowError): packb(x + 1) @@ -29,16 +29,16 @@ def test_integer(): def test_array_header(): packer = Packer() - packer.pack_array_header(2 ** 32 - 1) + packer.pack_array_header(2**32 - 1) with pytest.raises(PackValueError): - packer.pack_array_header(2 ** 32) + packer.pack_array_header(2**32) def test_map_header(): packer = Packer() - packer.pack_map_header(2 ** 32 - 1) + packer.pack_map_header(2**32 - 1) with pytest.raises(PackValueError): - packer.pack_array_header(2 ** 32) + packer.pack_array_header(2**32) def test_max_str_len(): diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 86b2c1f7..84941db8 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -53,46 +53,46 @@ def test_fixstr_from_float(): def test_str16_from_byte(): - _runtest("B", 2 ** 8, b"\xda", b"\x01\x00", False) - _runtest("B", 2 ** 16 - 1, b"\xda", b"\xff\xff", False) + _runtest("B", 2**8, b"\xda", b"\x01\x00", False) + _runtest("B", 2**16 - 1, b"\xda", b"\xff\xff", False) def test_str16_from_float(): - _runtest("f", 2 ** 8, b"\xda", b"\x01\x00", False) - _runtest("f", 2 ** 16 - 4, b"\xda", b"\xff\xfc", False) + _runtest("f", 2**8, b"\xda", b"\x01\x00", False) + _runtest("f", 2**16 - 4, b"\xda", b"\xff\xfc", False) def test_str32_from_byte(): - _runtest("B", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False) + _runtest("B", 2**16, b"\xdb", b"\x00\x01\x00\x00", False) def test_str32_from_float(): - _runtest("f", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False) + _runtest("f", 2**16, b"\xdb", b"\x00\x01\x00\x00", False) def test_bin8_from_byte(): _runtest("B", 1, b"\xc4", b"\x01", True) - _runtest("B", 2 ** 8 - 1, b"\xc4", b"\xff", True) + _runtest("B", 2**8 - 1, b"\xc4", b"\xff", True) def test_bin8_from_float(): _runtest("f", 4, b"\xc4", b"\x04", True) - _runtest("f", 2 ** 8 - 4, b"\xc4", b"\xfc", True) + _runtest("f", 2**8 - 4, b"\xc4", b"\xfc", True) def test_bin16_from_byte(): - _runtest("B", 2 ** 8, b"\xc5", b"\x01\x00", True) - _runtest("B", 2 ** 16 - 1, b"\xc5", b"\xff\xff", True) + _runtest("B", 2**8, b"\xc5", b"\x01\x00", True) + _runtest("B", 2**16 - 1, b"\xc5", b"\xff\xff", True) def test_bin16_from_float(): - _runtest("f", 2 ** 8, b"\xc5", b"\x01\x00", True) - _runtest("f", 2 ** 16 - 4, b"\xc5", b"\xff\xfc", True) + _runtest("f", 2**8, b"\xc5", b"\x01\x00", True) + _runtest("f", 2**16 - 4, b"\xc5", b"\xff\xfc", True) def test_bin32_from_byte(): - _runtest("B", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True) + _runtest("B", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) def test_bin32_from_float(): - _runtest("f", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True) + _runtest("f", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 6293a453..c091076b 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -2,7 +2,7 @@ # coding: utf-8 import io from msgpack import Unpacker, BufferFull -from msgpack import pack +from msgpack import pack, packb from msgpack.exceptions import OutOfData from pytest import raises @@ -78,6 +78,15 @@ def test_maxbuffersize(): assert ord("b") == next(unpacker) +def test_maxbuffersize_file(): + buff = io.BytesIO(packb(b"a" * 10) + packb([b"a" * 20] * 2)) + unpacker = Unpacker(buff, read_size=1, max_buffer_size=19, max_bin_len=20) + assert unpacker.unpack() == b"a" * 10 + # assert unpacker.unpack() == [b"a" * 20]*2 + with raises(BufferFull): + print(unpacker.unpack()) + + def test_readbytes(): unpacker = Unpacker(read_size=3) unpacker.feed(b"foobar") @@ -118,8 +127,8 @@ def test_issue124(): def test_unpack_tell(): stream = io.BytesIO() - messages = [2 ** i - 1 for i in range(65)] - messages += [-(2 ** i) for i in range(1, 64)] + messages = [2**i - 1 for i in range(65)] + messages += [-(2**i) for i in range(1, 64)] messages += [ b"hello", b"hello" * 1000, diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 4e26489b..253228e7 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -10,31 +10,31 @@ def test_timestamp(): # timestamp32 - ts = Timestamp(2 ** 32 - 1) + ts = Timestamp(2**32 - 1) assert ts.to_bytes() == b"\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xd6\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 32 - 1 and ts.nanoseconds == 0 + assert ts.seconds == 2**32 - 1 and ts.nanoseconds == 0 # timestamp64 - ts = Timestamp(2 ** 34 - 1, 999999999) + ts = Timestamp(2**34 - 1, 999999999) assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xd7\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 34 - 1 and ts.nanoseconds == 999999999 + assert ts.seconds == 2**34 - 1 and ts.nanoseconds == 999999999 # timestamp96 - ts = Timestamp(2 ** 63 - 1, 999999999) + ts = Timestamp(2**63 - 1, 999999999) assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xc7\x0c\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999 + assert ts.seconds == 2**63 - 1 and ts.nanoseconds == 999999999 # negative fractional ts = Timestamp.from_unix(-2.3) # s: -3, ns: 700000000