Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit 70f4777

Browse filesBrowse files
authored
Fix SELECT CAST(... AS JSON) cause UnicodeError (#496)
* Fix SELECT CAST(... AS JSON) cause UnicodeError fixes #488
1 parent 634c1c3 commit 70f4777
Copy full SHA for 70f4777

File tree

Expand file treeCollapse file tree

4 files changed

+24
-14
lines changed
Filter options
Expand file treeCollapse file tree

4 files changed

+24
-14
lines changed

‎pymysql/charset.py

Copy file name to clipboardExpand all lines: pymysql/charset.py
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ def __init__(self, id, name, collation, is_default):
1111
self.id, self.name, self.collation = id, name, collation
1212
self.is_default = is_default == 'Yes'
1313

14+
def __repr__(self):
15+
return "Charset(id=%s, name=%r, collation=%r)" % (
16+
self.id, self.name, self.collation)
17+
1418
@property
1519
def encoding(self):
1620
name = self.name

‎pymysql/connections.py

Copy file name to clipboardExpand all lines: pymysql/connections.py
+15-7Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,6 @@ def _makefile(sock, mode):
8888
FIELD_TYPE.BLOB,
8989
FIELD_TYPE.LONG_BLOB,
9090
FIELD_TYPE.MEDIUM_BLOB,
91-
FIELD_TYPE.JSON,
9291
FIELD_TYPE.STRING,
9392
FIELD_TYPE.TINY_BLOB,
9493
FIELD_TYPE.VAR_STRING,
@@ -407,9 +406,9 @@ class FieldDescriptorPacket(MysqlPacket):
407406

408407
def __init__(self, data, encoding):
409408
MysqlPacket.__init__(self, data, encoding)
410-
self.__parse_field_descriptor(encoding)
409+
self._parse_field_descriptor(encoding)
411410

412-
def __parse_field_descriptor(self, encoding):
411+
def _parse_field_descriptor(self, encoding):
413412
"""Parse the 'Field Descriptor' (Metadata) packet.
414413
415414
This is compatible with MySQL 4.1+ (not compatible with MySQL 4.0).
@@ -1433,21 +1432,30 @@ def _get_descriptions(self):
14331432
self.fields = []
14341433
self.converters = []
14351434
use_unicode = self.connection.use_unicode
1435+
conn_encoding = self.connection.encoding
14361436
description = []
1437+
14371438
for i in range_type(self.field_count):
14381439
field = self.connection._read_packet(FieldDescriptorPacket)
14391440
self.fields.append(field)
14401441
description.append(field.description())
14411442
field_type = field.type_code
14421443
if use_unicode:
1443-
if field_type in TEXT_TYPES:
1444-
charset = charset_by_id(field.charsetnr)
1445-
if charset.is_binary:
1444+
if field_type == FIELD_TYPE.JSON:
1445+
# When SELECT from JSON column: charset = binary
1446+
# When SELECT CAST(... AS JSON): charset = connection encoding
1447+
# This behavior is different from TEXT / BLOB.
1448+
# We should decode result by connection encoding regardless charsetnr.
1449+
# See https://github.com/PyMySQL/PyMySQL/issues/488
1450+
encoding = conn_encoding # SELECT CAST(... AS JSON)
1451+
elif field_type in TEXT_TYPES:
1452+
if field.charsetnr == 63: # binary
14461453
# TEXTs with charset=binary means BINARY types.
14471454
encoding = None
14481455
else:
1449-
encoding = charset.encoding
1456+
encoding = conn_encoding
14501457
else:
1458+
# Integers, Dates and Times, and other basic data is encoded in ascii
14511459
encoding = 'ascii'
14521460
else:
14531461
encoding = None

‎pymysql/converters.py

Copy file name to clipboardExpand all lines: pymysql/converters.py
-7Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -333,12 +333,6 @@ def convert_set(s):
333333
return set(s.split(","))
334334

335335

336-
def convert_json(b):
337-
# JSON is returned as binary data.
338-
# Decode with utf-8 regardless connection encoding.
339-
return b.decode('utf-8')
340-
341-
342336
def through(x):
343337
return x
344338

@@ -416,7 +410,6 @@ def convert_characters(connection, field, data):
416410
FIELD_TYPE.VARCHAR: through,
417411
FIELD_TYPE.DECIMAL: Decimal,
418412
FIELD_TYPE.NEWDECIMAL: Decimal,
419-
FIELD_TYPE.JSON: convert_json,
420413
}
421414

422415

‎pymysql/tests/test_basic.py

Copy file name to clipboardExpand all lines: pymysql/tests/test_basic.py
+5Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,12 +253,17 @@ def test_json(self):
253253
primary key (id)
254254
);""")
255255
cur = conn.cursor()
256+
256257
json_str = u'{"hello": "こんにちは"}'
257258
cur.execute("INSERT INTO test_json (id, `json`) values (42, %s)", (json_str,))
258259
cur.execute("SELECT `json` from `test_json` WHERE `id`=42")
259260
res = cur.fetchone()[0]
260261
self.assertEqual(json.loads(res), json.loads(json_str))
261262

263+
cur.execute("SELECT CAST(%s AS JSON) AS x", (json_str,))
264+
res = cur.fetchone()[0]
265+
self.assertEqual(json.loads(res), json.loads(json_str))
266+
262267

263268
class TestBulkInserts(base.PyMySQLTestCase):
264269

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.