Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 3bea8cd

Browse filesBrowse files
methanemethane
authored
Faster string and int decoder (PyMySQL#305)
1 parent c7188e6 commit 3bea8cd
Copy full SHA for 3bea8cd

File tree

Expand file treeCollapse file tree

3 files changed

+71
-37
lines changed
Filter options
Expand file treeCollapse file tree

3 files changed

+71
-37
lines changed

‎MySQLdb/_mysql.c

Copy file name to clipboardExpand all lines: MySQLdb/_mysql.c
+58-21Lines changed: 58 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ PERFORMANCE OF THIS SOFTWARE.
3737
#include "Python.h"
3838
#if PY_MAJOR_VERSION >= 3
3939
#define IS_PY3K
40+
#define PyInt_Type PyLong_Type
41+
#define PyInt_FromString PyLong_FromString
4042
#define PyInt_FromLong(n) PyLong_FromLong(n)
4143
#define PyInt_Check(n) PyLong_Check(n)
4244
#define PyInt_AS_LONG(n) PyLong_AS_LONG(n)
@@ -87,6 +89,7 @@ typedef struct {
8789
int use;
8890
char has_next;
8991
PyObject *converter;
92+
const char *encoding;
9093
} _mysql_ResultObject;
9194

9295
extern PyTypeObject _mysql_ResultObject_Type;
@@ -203,6 +206,25 @@ _mysql_Exception(_mysql_ConnectionObject *c)
203206
return NULL;
204207
}
205208

209+
static const char *utf8 = "utf8";
210+
211+
static const char*
212+
_get_encoding(MYSQL *mysql)
213+
{
214+
MY_CHARSET_INFO cs;
215+
mysql_get_character_set_info(mysql, &cs);
216+
if (strncmp(utf8, cs.csname, 4) == 0) { // utf8, utf8mb3, utf8mb4
217+
return utf8;
218+
}
219+
else if (strncmp("koi8r", cs.csname, 5) == 0) {
220+
return "koi8_r";
221+
}
222+
else if (strncmp("koi8u", cs.csname, 5) == 0) {
223+
return "koi8_u";
224+
}
225+
return cs.csname;
226+
}
227+
206228
static char _mysql_thread_safe__doc__[] =
207229
"Indicates whether the client is compiled as thread-safe.";
208230

@@ -251,6 +273,9 @@ _mysql_ResultObject_Initialize(
251273
self->result = result;
252274
self->has_next = (char)mysql_more_results(&(conn->connection));
253275
Py_END_ALLOW_THREADS ;
276+
277+
self->encoding = _get_encoding(&(conn->connection));
278+
//fprintf(stderr, "encoding=%s\n", self->encoding);
254279
if (!result) {
255280
if (mysql_errno(&(conn->connection))) {
256281
_mysql_Exception(conn);
@@ -289,6 +314,8 @@ _mysql_ResultObject_Initialize(
289314
int j, n2=PySequence_Size(fun);
290315
if (fields[i].charsetnr != 63) { /* maaagic */
291316
flags &= ~BINARY_FLAG;
317+
} else {
318+
flags |= BINARY_FLAG;
292319
}
293320
for (j=0; j<n2; j++) {
294321
PyObject *t = PySequence_GetItem(fun, j);
@@ -943,8 +970,6 @@ _mysql_string_literal(
943970
return (str);
944971
}
945972

946-
static PyObject *_mysql_NULL;
947-
948973
static PyObject *
949974
_escape_item(
950975
PyObject *self,
@@ -1011,28 +1036,21 @@ _mysql_ResultObject_describe(
10111036
{
10121037
PyObject *d;
10131038
MYSQL_FIELD *fields;
1014-
MY_CHARSET_INFO cs;
1015-
int isutf8 = 0;
10161039
unsigned int i, n;
10171040

10181041
check_result_connection(self);
10191042

1020-
mysql_get_character_set_info(&result_connection(self)->connection, &cs);
1021-
if (strncmp("utf8", cs.name, 4) == 0) { // utf8, utf8mb3, utf8mb4
1022-
isutf8 = 1;
1023-
}
1024-
10251043
n = mysql_num_fields(self->result);
10261044
fields = mysql_fetch_fields(self->result);
10271045
if (!(d = PyTuple_New(n))) return NULL;
10281046
for (i=0; i<n; i++) {
10291047
PyObject *t;
10301048
#ifdef IS_PY3K
10311049
PyObject *name;
1032-
if (isutf8) {
1050+
if (self->encoding == utf8) {
10331051
name = PyUnicode_DecodeUTF8(fields[i].name, fields[i].name_length, "replace");
10341052
} else {
1035-
name = PyUnicode_Decode(fields[i].name, fields[i].name_length, cs.name, "replace");
1053+
name = PyUnicode_Decode(fields[i].name, fields[i].name_length, self->encoding, "replace");
10361054
}
10371055
if (name == NULL) {
10381056
goto error;
@@ -1089,9 +1107,10 @@ _mysql_ResultObject_field_flags(
10891107
static PyObject *
10901108
_mysql_field_to_python(
10911109
PyObject *converter,
1092-
char *rowitem,
1110+
const char *rowitem,
10931111
unsigned long length,
1094-
MYSQL_FIELD *field)
1112+
MYSQL_FIELD *field,
1113+
const char *encoding)
10951114
{
10961115
PyObject *v;
10971116
#ifdef IS_PY3K
@@ -1110,7 +1129,27 @@ _mysql_field_to_python(
11101129
}
11111130
#endif
11121131
if (rowitem) {
1113-
if (converter != Py_None) {
1132+
if (converter == (PyObject*)&PyUnicode_Type) {
1133+
if (encoding == utf8) {
1134+
//fprintf(stderr, "decoding with utf8!\n");
1135+
v = PyUnicode_DecodeUTF8(rowitem, length, NULL);
1136+
} else {
1137+
//fprintf(stderr, "decoding with %s\n", encoding);
1138+
v = PyUnicode_Decode(rowitem, length, encoding, NULL);
1139+
}
1140+
}
1141+
else if (converter == (PyObject*)&PyBytes_Type) {
1142+
//fprintf(stderr, "decoding with bytes\n", encoding);
1143+
v = PyBytes_FromStringAndSize(rowitem, length);
1144+
}
1145+
else if (converter == (PyObject*)&PyInt_Type) {
1146+
//fprintf(stderr, "decoding with int\n", encoding);
1147+
v = PyInt_FromString(rowitem, NULL, 10);
1148+
}
1149+
else if (converter != Py_None) {
1150+
//fprintf(stderr, "decoding with callback\n");
1151+
//PyObject_Print(converter, stderr, 0);
1152+
//fprintf(stderr, "\n");
11141153
v = PyObject_CallFunction(converter,
11151154
#ifdef IS_PY3K
11161155
binary ? "y#" : "s#",
@@ -1120,6 +1159,7 @@ _mysql_field_to_python(
11201159
rowitem,
11211160
(int)length);
11221161
} else {
1162+
//fprintf(stderr, "converter=None\n");
11231163
#ifdef IS_PY3K
11241164
if (!binary) {
11251165
v = PyUnicode_FromStringAndSize(rowitem, (int)length);
@@ -1153,7 +1193,7 @@ _mysql_row_to_tuple(
11531193
for (i=0; i<n; i++) {
11541194
PyObject *v;
11551195
c = PyTuple_GET_ITEM(self->converter, i);
1156-
v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
1196+
v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
11571197
if (!v) goto error;
11581198
PyTuple_SET_ITEM(r, i, v);
11591199
}
@@ -1180,7 +1220,7 @@ _mysql_row_to_dict(
11801220
for (i=0; i<n; i++) {
11811221
PyObject *v;
11821222
c = PyTuple_GET_ITEM(self->converter, i);
1183-
v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
1223+
v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
11841224
if (!v) goto error;
11851225
if (!PyMapping_HasKeyString(r, fields[i].name)) {
11861226
PyMapping_SetItemString(r, fields[i].name, v);
@@ -1219,7 +1259,7 @@ _mysql_row_to_dict_old(
12191259
for (i=0; i<n; i++) {
12201260
PyObject *v;
12211261
c = PyTuple_GET_ITEM(self->converter, i);
1222-
v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
1262+
v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
12231263
if (!v) goto error;
12241264
{
12251265
int len=0;
@@ -1422,8 +1462,7 @@ _mysql_ConnectionObject_set_character_set(
14221462
err = mysql_set_character_set(&(self->connection), s);
14231463
Py_END_ALLOW_THREADS
14241464
if (err) return _mysql_Exception(self);
1425-
Py_INCREF(Py_None);
1426-
return Py_None;
1465+
Py_RETURN_NONE;
14271466
}
14281467

14291468
#if MYSQL_VERSION_ID >= 50010
@@ -2634,12 +2673,10 @@ init_mysql(void)
26342673
_mysql_ConnectionObject_Type.ob_type = &PyType_Type;
26352674
_mysql_ResultObject_Type.ob_type = &PyType_Type;
26362675
#endif
2637-
#if PY_VERSION_HEX >= 0x02020000
26382676
_mysql_ConnectionObject_Type.tp_alloc = PyType_GenericAlloc;
26392677
_mysql_ConnectionObject_Type.tp_new = PyType_GenericNew;
26402678
_mysql_ResultObject_Type.tp_alloc = PyType_GenericAlloc;
26412679
_mysql_ResultObject_Type.tp_new = PyType_GenericNew;
2642-
#endif
26432680
#ifdef IS_PY3K
26442681
if (PyType_Ready(&_mysql_ConnectionObject_Type) < 0)
26452682
return NULL;

‎MySQLdb/connections.py

Copy file name to clipboardExpand all lines: MySQLdb/connections.py
+2-5Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ class object, used to create cursors (keyword only)
114114
documentation for the MySQL C API for some hints on what they do.
115115
"""
116116
from MySQLdb.constants import CLIENT, FIELD_TYPE
117-
from MySQLdb.converters import conversions
117+
from MySQLdb.converters import conversions, _bytes_or_str
118118
from weakref import proxy
119119

120120
kwargs2 = kwargs.copy()
@@ -174,9 +174,6 @@ class object, used to create cursors (keyword only)
174174
def unicode_literal(u, dummy=None):
175175
return db.string_literal(u.encode(db.encoding))
176176

177-
def string_decoder(s):
178-
return s.decode(db.encoding)
179-
180177
if not charset:
181178
charset = self.character_set_name()
182179
self.set_character_set(charset)
@@ -187,7 +184,7 @@ def string_decoder(s):
187184
if use_unicode:
188185
for t in (FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING, FIELD_TYPE.VARCHAR, FIELD_TYPE.TINY_BLOB,
189186
FIELD_TYPE.MEDIUM_BLOB, FIELD_TYPE.LONG_BLOB, FIELD_TYPE.BLOB):
190-
self.converter[t].append((None, string_decoder))
187+
self.converter[t] = _bytes_or_str
191188

192189
self.encoders[unicode] = unicode_literal
193190
self._transactional = self.server_capabilities & CLIENT.TRANSACTIONS

‎MySQLdb/converters.py

Copy file name to clipboardExpand all lines: MySQLdb/converters.py
+11-11Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
from MySQLdb._mysql import string_literal, escape
3636
from MySQLdb.constants import FIELD_TYPE, FLAG
3737
from MySQLdb.times import *
38-
from MySQLdb.compat import PY2, long
38+
from MySQLdb.compat import PY2, long, unicode
3939

4040
NoneType = type(None)
4141

@@ -91,7 +91,7 @@ def quote_tuple(t, d):
9191
return "(%s)" % (','.join(escape_sequence(t, d)))
9292

9393
# bytes or str regarding to BINARY_FLAG.
94-
_bytes_or_str = [(FLAG.BINARY, bytes)]
94+
_bytes_or_str = ((FLAG.BINARY, bytes), (None, unicode))
9595

9696
conversions = {
9797
int: Thing2Str,
@@ -109,24 +109,24 @@ def quote_tuple(t, d):
109109

110110
FIELD_TYPE.TINY: int,
111111
FIELD_TYPE.SHORT: int,
112-
FIELD_TYPE.LONG: long,
112+
FIELD_TYPE.LONG: int,
113113
FIELD_TYPE.FLOAT: float,
114114
FIELD_TYPE.DOUBLE: float,
115115
FIELD_TYPE.DECIMAL: Decimal,
116116
FIELD_TYPE.NEWDECIMAL: Decimal,
117-
FIELD_TYPE.LONGLONG: long,
117+
FIELD_TYPE.LONGLONG: int,
118118
FIELD_TYPE.INT24: int,
119119
FIELD_TYPE.YEAR: int,
120120
FIELD_TYPE.TIMESTAMP: mysql_timestamp_converter,
121121
FIELD_TYPE.DATETIME: DateTime_or_None,
122122
FIELD_TYPE.TIME: TimeDelta_or_None,
123123
FIELD_TYPE.DATE: Date_or_None,
124124

125-
FIELD_TYPE.TINY_BLOB: _bytes_or_str,
126-
FIELD_TYPE.MEDIUM_BLOB: _bytes_or_str,
127-
FIELD_TYPE.LONG_BLOB: _bytes_or_str,
128-
FIELD_TYPE.BLOB: _bytes_or_str,
129-
FIELD_TYPE.STRING: _bytes_or_str,
130-
FIELD_TYPE.VAR_STRING: _bytes_or_str,
131-
FIELD_TYPE.VARCHAR: _bytes_or_str,
125+
FIELD_TYPE.TINY_BLOB: bytes,
126+
FIELD_TYPE.MEDIUM_BLOB: bytes,
127+
FIELD_TYPE.LONG_BLOB: bytes,
128+
FIELD_TYPE.BLOB: bytes,
129+
FIELD_TYPE.STRING: bytes,
130+
FIELD_TYPE.VAR_STRING: bytes,
131+
FIELD_TYPE.VARCHAR: bytes,
132132
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.