From ff9befb8300f31b9410913c3b561ab9f13acf29d Mon Sep 17 00:00:00 2001 From: Dan Tillberg Date: Mon, 11 Jan 2016 16:35:10 -0500 Subject: [PATCH 1/2] Add `force_str_type` option to force using fixstr/str* in place of bin* --- msgpack/_packer.pyx | 6 +++++- msgpack/fallback.py | 9 ++++++--- msgpack/pack.h | 1 + msgpack/pack_template.h | 4 ++-- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 0a6513c0..d65cfbe9 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -16,6 +16,7 @@ cdef extern from "pack.h": size_t length size_t buf_size bint use_bin_type + bint force_str_type int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) @@ -63,6 +64,8 @@ cdef class Packer(object): :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. It also enable str8 type for unicode. + :param bool force_str_type: + Use str* encoding in place of bin* encoding. Also enable str8. """ cdef msgpack_packer pk cdef object _default @@ -82,12 +85,13 @@ cdef class Packer(object): self.pk.length = 0 def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, bint autoreset=1, bint use_bin_type=0): + use_single_float=False, bint autoreset=1, bint use_bin_type=0, bint force_str_type=0): """ """ self.use_float = use_single_float self.autoreset = autoreset self.pk.use_bin_type = use_bin_type + self.pk.force_str_type = force_str_type if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f682611d..0bd25601 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -609,12 +609,15 @@ class Packer(object): :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. It also enable str8 type for unicode. + :param bool force_str_type: + Use str* encoding in place of bin* encoding. Also enable str8. """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, autoreset=True, use_bin_type=False): + use_single_float=False, autoreset=True, use_bin_type=False, force_str_type=False): self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type + self._force_str_type = force_str_type self._encoding = encoding self._unicode_errors = unicode_errors self._buffer = StringIO() @@ -660,7 +663,7 @@ def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance): default_used = True continue raise PackValueError("Integer value out of range") - if self._use_bin_type and isinstance(obj, bytes): + if (not self._force_str_type) and self._use_bin_type and isinstance(obj, bytes): n = len(obj) if n <= 0xff: self._buffer.write(struct.pack('>BB', 0xc4, n)) @@ -681,7 +684,7 @@ def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance): n = len(obj) if n <= 0x1f: self._buffer.write(struct.pack('B', 0xa0 + n)) - elif self._use_bin_type and n <= 0xff: + elif (self._force_str_type or self._use_bin_type) and n <= 0xff: self._buffer.write(struct.pack('>BB', 0xd9, n)) elif n <= 0xffff: self._buffer.write(struct.pack(">BH", 0xda, n)) diff --git a/msgpack/pack.h b/msgpack/pack.h index a75bdb04..14e55d1a 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -35,6 +35,7 @@ typedef struct msgpack_packer { size_t length; size_t buf_size; bool use_bin_type; + bool force_str_type; } msgpack_packer; typedef struct Packer Packer; diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 5d1088f4..6ddc0dbe 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -667,7 +667,7 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) if (l < 32) { unsigned char d = 0xa0 | (uint8_t)l; msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); - } else if (x->use_bin_type && l < 256) { // str8 is new format introduced with bin. + } else if ((x->force_str_type || x->use_bin_type) && l < 256) { // str8 is new format introduced with bin. unsigned char buf[2] = {0xd9, (uint8_t)l}; msgpack_pack_append_buffer(x, buf, 2); } else if (l < 65536) { @@ -686,7 +686,7 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) */ static inline int msgpack_pack_bin(msgpack_packer *x, size_t l) { - if (!x->use_bin_type) { + if (x->force_str_type || !x->use_bin_type) { return msgpack_pack_raw(x, l); } if (l < 256) { From 28ec0470cf3979ccc05ac6c28f47d3b97f8a37c6 Mon Sep 17 00:00:00 2001 From: Dan Tillberg Date: Sun, 31 Jan 2016 18:34:54 -0500 Subject: [PATCH 2/2] tmp --- msgpack/fallback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 76c370f5..5eec5144 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -698,7 +698,7 @@ def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, else: raise PackValueError("Bytes is too large") return self._buffer.write(obj) - if check(obj, (Unicode, bytes, memoryview)): + if check(obj, (Unicode, bytes, memoryview, str)): if check(obj, Unicode): if self._encoding is None: raise TypeError( @@ -759,7 +759,7 @@ def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, obj = self._default(obj) default_used = 1 continue - raise TypeError("Cannot serialize %r" % obj) + raise TypeError("Cannot serialize %r, type %s" % (obj, type(obj))) def pack(self, obj): self._pack(obj)