PyMySQL · methane · Dec 11, 2018 · Dec 10, 2018 · Dec 10, 2018
diff --git a/MySQLdb/_mysql.c b/MySQLdb/_mysql.c
@@ -37,6 +37,8 @@ PERFORMANCE OF THIS SOFTWARE.
 #include "Python.h"
 #if PY_MAJOR_VERSION >= 3
 #define IS_PY3K
+#define PyInt_Type PyLong_Type
+#define PyInt_FromString PyLong_FromString
 #define PyInt_FromLong(n) PyLong_FromLong(n)
 #define PyInt_Check(n) PyLong_Check(n)
 #define PyInt_AS_LONG(n) PyLong_AS_LONG(n)
@@ -87,6 +89,7 @@ typedef struct {
    int use;
    char has_next;
    PyObject *converter;
+    const char *encoding;
 } _mysql_ResultObject;

 extern PyTypeObject _mysql_ResultObject_Type;
@@ -203,6 +206,25 @@ _mysql_Exception(_mysql_ConnectionObject *c)
    return NULL;
 }

+static const char *utf8 = "utf8";
+
+static const char*
+_get_encoding(MYSQL *mysql)
+{
+    MY_CHARSET_INFO cs;
+    mysql_get_character_set_info(mysql, &cs);
+    if (strncmp(utf8, cs.csname, 4) == 0) { // utf8, utf8mb3, utf8mb4
+        return utf8;
+    }
+    else if (strncmp("koi8r", cs.csname, 5) == 0) {
+        return "koi8_r";
+    }
+    else if (strncmp("koi8u", cs.csname, 5) == 0) {
+        return "koi8_u";
+    }
+    return cs.csname;
+}
+
 static char _mysql_thread_safe__doc__[] =
 "Indicates whether the client is compiled as thread-safe.";

@@ -251,6 +273,9 @@ _mysql_ResultObject_Initialize(
    self->result = result;
    self->has_next = (char)mysql_more_results(&(conn->connection));
    Py_END_ALLOW_THREADS ;
+
+    self->encoding = _get_encoding(&(conn->connection));
+    //fprintf(stderr, "encoding=%s\n", self->encoding);
    if (!result) {
        if (mysql_errno(&(conn->connection))) {
            _mysql_Exception(conn);
@@ -289,6 +314,8 @@ _mysql_ResultObject_Initialize(
            int j, n2=PySequence_Size(fun);
            if (fields[i].charsetnr != 63) { /* maaagic */
                flags &= ~BINARY_FLAG;
+            } else {
+                flags |= BINARY_FLAG;
            }
            for (j=0; j<n2; j++) {
                PyObject *t = PySequence_GetItem(fun, j);
@@ -943,8 +970,6 @@ _mysql_string_literal(
    return (str);
 }

-static PyObject *_mysql_NULL;
-
 static PyObject *
 _escape_item(
    PyObject *self,
@@ -1011,28 +1036,21 @@ _mysql_ResultObject_describe(
 {
    PyObject *d;
    MYSQL_FIELD *fields;
-    MY_CHARSET_INFO cs;
-    int isutf8 = 0;
    unsigned int i, n;

    check_result_connection(self);

-    mysql_get_character_set_info(&result_connection(self)->connection, &cs);
-    if (strncmp("utf8", cs.name, 4) == 0) { // utf8, utf8mb3, utf8mb4
-        isutf8 = 1;
-    }
-
    n = mysql_num_fields(self->result);
    fields = mysql_fetch_fields(self->result);
    if (!(d = PyTuple_New(n))) return NULL;
    for (i=0; i<n; i++) {
        PyObject *t;
 #ifdef IS_PY3K
        PyObject *name;
-        if (isutf8) {
+        if (self->encoding == utf8) {
            name = PyUnicode_DecodeUTF8(fields[i].name, fields[i].name_length, "replace");
        } else {
-            name = PyUnicode_Decode(fields[i].name, fields[i].name_length, cs.name, "replace");
+            name = PyUnicode_Decode(fields[i].name, fields[i].name_length, self->encoding, "replace");
        }
        if (name == NULL) {
            goto error;
@@ -1089,9 +1107,10 @@ _mysql_ResultObject_field_flags(
 static PyObject *
 _mysql_field_to_python(
    PyObject *converter,
-    char *rowitem,
+    const char *rowitem,
    unsigned long length,
-    MYSQL_FIELD *field)
+    MYSQL_FIELD *field,
+    const char *encoding)
 {
    PyObject *v;
 #ifdef IS_PY3K
@@ -1110,7 +1129,27 @@ _mysql_field_to_python(
    }
 #endif
    if (rowitem) {
-        if (converter != Py_None) {
+        if (converter == (PyObject*)&PyUnicode_Type) {
+            if (encoding == utf8) {
+                //fprintf(stderr, "decoding with utf8!\n");
+                v = PyUnicode_DecodeUTF8(rowitem, length, NULL);
+            } else {
+                //fprintf(stderr, "decoding with %s\n", encoding);
+                v = PyUnicode_Decode(rowitem, length, encoding, NULL);
+            }
+        }
+        else if (converter == (PyObject*)&PyBytes_Type) {
+            //fprintf(stderr, "decoding with bytes\n", encoding);
+            v = PyBytes_FromStringAndSize(rowitem, length);
+        }
+        else if (converter == (PyObject*)&PyInt_Type) {
+            //fprintf(stderr, "decoding with int\n", encoding);
+            v = PyInt_FromString(rowitem, NULL, 10);
+        }
+        else if (converter != Py_None) {
+            //fprintf(stderr, "decoding with callback\n");
+            //PyObject_Print(converter, stderr, 0);
+            //fprintf(stderr, "\n");
            v = PyObject_CallFunction(converter,
 #ifdef IS_PY3K
                          binary ? "y#" : "s#",
@@ -1120,6 +1159,7 @@ _mysql_field_to_python(
                          rowitem,
                          (int)length);
        } else {
+            //fprintf(stderr, "converter=None\n");
 #ifdef IS_PY3K
            if (!binary) {
                v = PyUnicode_FromStringAndSize(rowitem, (int)length);
@@ -1153,7 +1193,7 @@ _mysql_row_to_tuple(
    for (i=0; i<n; i++) {
        PyObject *v;
        c = PyTuple_GET_ITEM(self->converter, i);
-        v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
+        v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
        if (!v) goto error;
        PyTuple_SET_ITEM(r, i, v);
    }
@@ -1180,7 +1220,7 @@ _mysql_row_to_dict(
    for (i=0; i<n; i++) {
        PyObject *v;
        c = PyTuple_GET_ITEM(self->converter, i);
-        v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
+        v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
        if (!v) goto error;
        if (!PyMapping_HasKeyString(r, fields[i].name)) {
            PyMapping_SetItemString(r, fields[i].name, v);
@@ -1219,7 +1259,7 @@ _mysql_row_to_dict_old(
    for (i=0; i<n; i++) {
        PyObject *v;
        c = PyTuple_GET_ITEM(self->converter, i);
-        v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
+        v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
        if (!v) goto error;
        {
            int len=0;
@@ -1422,8 +1462,7 @@ _mysql_ConnectionObject_set_character_set(
    err = mysql_set_character_set(&(self->connection), s);
    Py_END_ALLOW_THREADS
    if (err) return _mysql_Exception(self);
-    Py_INCREF(Py_None);
-    return Py_None;
+    Py_RETURN_NONE;
 }

 #if MYSQL_VERSION_ID >= 50010
@@ -2634,12 +2673,10 @@ init_mysql(void)
    _mysql_ConnectionObject_Type.ob_type = &PyType_Type;
    _mysql_ResultObject_Type.ob_type = &PyType_Type;
 #endif
-#if PY_VERSION_HEX >= 0x02020000
    _mysql_ConnectionObject_Type.tp_alloc = PyType_GenericAlloc;
    _mysql_ConnectionObject_Type.tp_new = PyType_GenericNew;
    _mysql_ResultObject_Type.tp_alloc = PyType_GenericAlloc;
    _mysql_ResultObject_Type.tp_new = PyType_GenericNew;
-#endif
 #ifdef IS_PY3K
    if (PyType_Ready(&_mysql_ConnectionObject_Type) < 0)
        return NULL;

diff --git a/MySQLdb/connections.py b/MySQLdb/connections.py
@@ -114,7 +114,7 @@ class object, used to create cursors (keyword only)
        documentation for the MySQL C API for some hints on what they do.
        """
        from MySQLdb.constants import CLIENT, FIELD_TYPE
-        from MySQLdb.converters import conversions
+        from MySQLdb.converters import conversions, _bytes_or_str
        from weakref import proxy

        kwargs2 = kwargs.copy()
@@ -174,9 +174,6 @@ class object, used to create cursors (keyword only)
        def unicode_literal(u, dummy=None):
            return db.string_literal(u.encode(db.encoding))

-        def string_decoder(s):
-            return s.decode(db.encoding)
-
        if not charset:
            charset = self.character_set_name()
        self.set_character_set(charset)
@@ -187,7 +184,7 @@ def string_decoder(s):
        if use_unicode:
            for t in (FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING, FIELD_TYPE.VARCHAR, FIELD_TYPE.TINY_BLOB,
                      FIELD_TYPE.MEDIUM_BLOB, FIELD_TYPE.LONG_BLOB, FIELD_TYPE.BLOB):
-                self.converter[t].append((None, string_decoder))
+                self.converter[t] = _bytes_or_str

        self.encoders[unicode] = unicode_literal
        self._transactional = self.server_capabilities & CLIENT.TRANSACTIONS

diff --git a/MySQLdb/converters.py b/MySQLdb/converters.py
@@ -35,7 +35,7 @@
 from MySQLdb._mysql import string_literal, escape
 from MySQLdb.constants import FIELD_TYPE, FLAG
 from MySQLdb.times import *
-from MySQLdb.compat import PY2, long
+from MySQLdb.compat import PY2, long, unicode

 NoneType = type(None)

@@ -91,7 +91,7 @@ def quote_tuple(t, d):
    return "(%s)" % (','.join(escape_sequence(t, d)))

 # bytes or str regarding to BINARY_FLAG.
-_bytes_or_str = [(FLAG.BINARY, bytes)]
+_bytes_or_str = ((FLAG.BINARY, bytes), (None, unicode))

 conversions = {
    int: Thing2Str,
@@ -109,24 +109,24 @@ def quote_tuple(t, d):

    FIELD_TYPE.TINY: int,
    FIELD_TYPE.SHORT: int,
-    FIELD_TYPE.LONG: long,
+    FIELD_TYPE.LONG: int,
    FIELD_TYPE.FLOAT: float,
    FIELD_TYPE.DOUBLE: float,
    FIELD_TYPE.DECIMAL: Decimal,
    FIELD_TYPE.NEWDECIMAL: Decimal,
-    FIELD_TYPE.LONGLONG: long,
+    FIELD_TYPE.LONGLONG: int,
    FIELD_TYPE.INT24: int,
    FIELD_TYPE.YEAR: int,
    FIELD_TYPE.TIMESTAMP: mysql_timestamp_converter,
    FIELD_TYPE.DATETIME: DateTime_or_None,
    FIELD_TYPE.TIME: TimeDelta_or_None,
    FIELD_TYPE.DATE: Date_or_None,

-    FIELD_TYPE.TINY_BLOB: _bytes_or_str,
-    FIELD_TYPE.MEDIUM_BLOB: _bytes_or_str,
-    FIELD_TYPE.LONG_BLOB: _bytes_or_str,
-    FIELD_TYPE.BLOB: _bytes_or_str,
-    FIELD_TYPE.STRING: _bytes_or_str,
-    FIELD_TYPE.VAR_STRING: _bytes_or_str,
-    FIELD_TYPE.VARCHAR: _bytes_or_str,
+    FIELD_TYPE.TINY_BLOB: bytes,
+    FIELD_TYPE.MEDIUM_BLOB: bytes,
+    FIELD_TYPE.LONG_BLOB: bytes,
+    FIELD_TYPE.BLOB: bytes,
+    FIELD_TYPE.STRING: bytes,
+    FIELD_TYPE.VAR_STRING: bytes,
+    FIELD_TYPE.VARCHAR: bytes,
 }