diff --git a/MySQLdb/_mysql.c b/MySQLdb/_mysql.c index 2d236982..53e57f85 100644 --- a/MySQLdb/_mysql.c +++ b/MySQLdb/_mysql.c @@ -37,6 +37,8 @@ PERFORMANCE OF THIS SOFTWARE. #include "Python.h" #if PY_MAJOR_VERSION >= 3 #define IS_PY3K +#define PyInt_Type PyLong_Type +#define PyInt_FromString PyLong_FromString #define PyInt_FromLong(n) PyLong_FromLong(n) #define PyInt_Check(n) PyLong_Check(n) #define PyInt_AS_LONG(n) PyLong_AS_LONG(n) @@ -87,6 +89,7 @@ typedef struct { int use; char has_next; PyObject *converter; + const char *encoding; } _mysql_ResultObject; extern PyTypeObject _mysql_ResultObject_Type; @@ -203,6 +206,25 @@ _mysql_Exception(_mysql_ConnectionObject *c) return NULL; } +static const char *utf8 = "utf8"; + +static const char* +_get_encoding(MYSQL *mysql) +{ + MY_CHARSET_INFO cs; + mysql_get_character_set_info(mysql, &cs); + if (strncmp(utf8, cs.csname, 4) == 0) { // utf8, utf8mb3, utf8mb4 + return utf8; + } + else if (strncmp("koi8r", cs.csname, 5) == 0) { + return "koi8_r"; + } + else if (strncmp("koi8u", cs.csname, 5) == 0) { + return "koi8_u"; + } + return cs.csname; +} + static char _mysql_thread_safe__doc__[] = "Indicates whether the client is compiled as thread-safe."; @@ -251,6 +273,9 @@ _mysql_ResultObject_Initialize( self->result = result; self->has_next = (char)mysql_more_results(&(conn->connection)); Py_END_ALLOW_THREADS ; + + self->encoding = _get_encoding(&(conn->connection)); + //fprintf(stderr, "encoding=%s\n", self->encoding); if (!result) { if (mysql_errno(&(conn->connection))) { _mysql_Exception(conn); @@ -289,6 +314,8 @@ _mysql_ResultObject_Initialize( int j, n2=PySequence_Size(fun); if (fields[i].charsetnr != 63) { /* maaagic */ flags &= ~BINARY_FLAG; + } else { + flags |= BINARY_FLAG; } for (j=0; jconnection, &cs); - if (strncmp("utf8", cs.name, 4) == 0) { // utf8, utf8mb3, utf8mb4 - isutf8 = 1; - } - n = mysql_num_fields(self->result); fields = mysql_fetch_fields(self->result); if (!(d = PyTuple_New(n))) return NULL; @@ -1029,10 +1047,10 @@ _mysql_ResultObject_describe( PyObject *t; #ifdef IS_PY3K PyObject *name; - if (isutf8) { + if (self->encoding == utf8) { name = PyUnicode_DecodeUTF8(fields[i].name, fields[i].name_length, "replace"); } else { - name = PyUnicode_Decode(fields[i].name, fields[i].name_length, cs.name, "replace"); + name = PyUnicode_Decode(fields[i].name, fields[i].name_length, self->encoding, "replace"); } if (name == NULL) { goto error; @@ -1089,9 +1107,10 @@ _mysql_ResultObject_field_flags( static PyObject * _mysql_field_to_python( PyObject *converter, - char *rowitem, + const char *rowitem, unsigned long length, - MYSQL_FIELD *field) + MYSQL_FIELD *field, + const char *encoding) { PyObject *v; #ifdef IS_PY3K @@ -1110,7 +1129,27 @@ _mysql_field_to_python( } #endif if (rowitem) { - if (converter != Py_None) { + if (converter == (PyObject*)&PyUnicode_Type) { + if (encoding == utf8) { + //fprintf(stderr, "decoding with utf8!\n"); + v = PyUnicode_DecodeUTF8(rowitem, length, NULL); + } else { + //fprintf(stderr, "decoding with %s\n", encoding); + v = PyUnicode_Decode(rowitem, length, encoding, NULL); + } + } + else if (converter == (PyObject*)&PyBytes_Type) { + //fprintf(stderr, "decoding with bytes\n", encoding); + v = PyBytes_FromStringAndSize(rowitem, length); + } + else if (converter == (PyObject*)&PyInt_Type) { + //fprintf(stderr, "decoding with int\n", encoding); + v = PyInt_FromString(rowitem, NULL, 10); + } + else if (converter != Py_None) { + //fprintf(stderr, "decoding with callback\n"); + //PyObject_Print(converter, stderr, 0); + //fprintf(stderr, "\n"); v = PyObject_CallFunction(converter, #ifdef IS_PY3K binary ? "y#" : "s#", @@ -1120,6 +1159,7 @@ _mysql_field_to_python( rowitem, (int)length); } else { + //fprintf(stderr, "converter=None\n"); #ifdef IS_PY3K if (!binary) { v = PyUnicode_FromStringAndSize(rowitem, (int)length); @@ -1153,7 +1193,7 @@ _mysql_row_to_tuple( for (i=0; iconverter, i); - v = _mysql_field_to_python(c, row[i], length[i], &fields[i]); + v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding); if (!v) goto error; PyTuple_SET_ITEM(r, i, v); } @@ -1180,7 +1220,7 @@ _mysql_row_to_dict( for (i=0; iconverter, i); - v = _mysql_field_to_python(c, row[i], length[i], &fields[i]); + v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding); if (!v) goto error; if (!PyMapping_HasKeyString(r, fields[i].name)) { PyMapping_SetItemString(r, fields[i].name, v); @@ -1219,7 +1259,7 @@ _mysql_row_to_dict_old( for (i=0; iconverter, i); - v = _mysql_field_to_python(c, row[i], length[i], &fields[i]); + v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding); if (!v) goto error; { int len=0; @@ -1422,8 +1462,7 @@ _mysql_ConnectionObject_set_character_set( err = mysql_set_character_set(&(self->connection), s); Py_END_ALLOW_THREADS if (err) return _mysql_Exception(self); - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } #if MYSQL_VERSION_ID >= 50010 @@ -2634,12 +2673,10 @@ init_mysql(void) _mysql_ConnectionObject_Type.ob_type = &PyType_Type; _mysql_ResultObject_Type.ob_type = &PyType_Type; #endif -#if PY_VERSION_HEX >= 0x02020000 _mysql_ConnectionObject_Type.tp_alloc = PyType_GenericAlloc; _mysql_ConnectionObject_Type.tp_new = PyType_GenericNew; _mysql_ResultObject_Type.tp_alloc = PyType_GenericAlloc; _mysql_ResultObject_Type.tp_new = PyType_GenericNew; -#endif #ifdef IS_PY3K if (PyType_Ready(&_mysql_ConnectionObject_Type) < 0) return NULL; diff --git a/MySQLdb/connections.py b/MySQLdb/connections.py index 27a8d437..e902912f 100644 --- a/MySQLdb/connections.py +++ b/MySQLdb/connections.py @@ -114,7 +114,7 @@ class object, used to create cursors (keyword only) documentation for the MySQL C API for some hints on what they do. """ from MySQLdb.constants import CLIENT, FIELD_TYPE - from MySQLdb.converters import conversions + from MySQLdb.converters import conversions, _bytes_or_str from weakref import proxy kwargs2 = kwargs.copy() @@ -174,9 +174,6 @@ class object, used to create cursors (keyword only) def unicode_literal(u, dummy=None): return db.string_literal(u.encode(db.encoding)) - def string_decoder(s): - return s.decode(db.encoding) - if not charset: charset = self.character_set_name() self.set_character_set(charset) @@ -187,7 +184,7 @@ def string_decoder(s): if use_unicode: for t in (FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING, FIELD_TYPE.VARCHAR, FIELD_TYPE.TINY_BLOB, FIELD_TYPE.MEDIUM_BLOB, FIELD_TYPE.LONG_BLOB, FIELD_TYPE.BLOB): - self.converter[t].append((None, string_decoder)) + self.converter[t] = _bytes_or_str self.encoders[unicode] = unicode_literal self._transactional = self.server_capabilities & CLIENT.TRANSACTIONS diff --git a/MySQLdb/converters.py b/MySQLdb/converters.py index 6394a5c6..7b1eb035 100644 --- a/MySQLdb/converters.py +++ b/MySQLdb/converters.py @@ -35,7 +35,7 @@ from MySQLdb._mysql import string_literal, escape from MySQLdb.constants import FIELD_TYPE, FLAG from MySQLdb.times import * -from MySQLdb.compat import PY2, long +from MySQLdb.compat import PY2, long, unicode NoneType = type(None) @@ -91,7 +91,7 @@ def quote_tuple(t, d): return "(%s)" % (','.join(escape_sequence(t, d))) # bytes or str regarding to BINARY_FLAG. -_bytes_or_str = [(FLAG.BINARY, bytes)] +_bytes_or_str = ((FLAG.BINARY, bytes), (None, unicode)) conversions = { int: Thing2Str, @@ -109,12 +109,12 @@ def quote_tuple(t, d): FIELD_TYPE.TINY: int, FIELD_TYPE.SHORT: int, - FIELD_TYPE.LONG: long, + FIELD_TYPE.LONG: int, FIELD_TYPE.FLOAT: float, FIELD_TYPE.DOUBLE: float, FIELD_TYPE.DECIMAL: Decimal, FIELD_TYPE.NEWDECIMAL: Decimal, - FIELD_TYPE.LONGLONG: long, + FIELD_TYPE.LONGLONG: int, FIELD_TYPE.INT24: int, FIELD_TYPE.YEAR: int, FIELD_TYPE.TIMESTAMP: mysql_timestamp_converter, @@ -122,11 +122,11 @@ def quote_tuple(t, d): FIELD_TYPE.TIME: TimeDelta_or_None, FIELD_TYPE.DATE: Date_or_None, - FIELD_TYPE.TINY_BLOB: _bytes_or_str, - FIELD_TYPE.MEDIUM_BLOB: _bytes_or_str, - FIELD_TYPE.LONG_BLOB: _bytes_or_str, - FIELD_TYPE.BLOB: _bytes_or_str, - FIELD_TYPE.STRING: _bytes_or_str, - FIELD_TYPE.VAR_STRING: _bytes_or_str, - FIELD_TYPE.VARCHAR: _bytes_or_str, + FIELD_TYPE.TINY_BLOB: bytes, + FIELD_TYPE.MEDIUM_BLOB: bytes, + FIELD_TYPE.LONG_BLOB: bytes, + FIELD_TYPE.BLOB: bytes, + FIELD_TYPE.STRING: bytes, + FIELD_TYPE.VAR_STRING: bytes, + FIELD_TYPE.VARCHAR: bytes, }