From 80fbc297e1e2aa4d0db00bac431fd2cbfb8b9add Mon Sep 17 00:00:00 2001 From: TIGirardi Date: Fri, 29 May 2020 22:38:54 -0300 Subject: [PATCH 1/5] bpo-38324: Fix test__locale.py Windows failures Use wide-char _W_* fields of lconv structure on Windows Remove "ps_AF" from test__locale.known_numerics on Windows --- Lib/test/test__locale.py | 4 ++++ Modules/_localemodule.c | 25 ++++++++++++++++++++++--- Python/fileutils.c | 15 +++++++++++++-- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index cda0ee91b700f73..30afe80e39db274 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -72,6 +72,10 @@ def accept(loc): 'ps_AF': ('\u066b', '\u066c'), } +if uname().system == 'Windows': + # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) + del known_numerics['ps_AF'] + class _LocaleTests(unittest.TestCase): def setUp(self): diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 0819d0e1924087f..dfab64268958782 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -144,6 +144,7 @@ locale_is_ascii(const char *str) static int locale_decode_monetary(PyObject *dict, struct lconv *lc) { +#ifndef MS_WINDOWS int change_locale; change_locale = (!locale_is_ascii(lc->int_curr_symbol) || !locale_is_ascii(lc->currency_symbol) @@ -179,12 +180,18 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc) } } +#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL) +#else /* MS_WINDOWS */ +/* Use _W_* fields of Windows struct lconv */ +#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1) +#endif /* MS_WINDOWS */ + int res = -1; #define RESULT_STRING(ATTR) \ do { \ PyObject *obj; \ - obj = PyUnicode_DecodeLocale(lc->ATTR, NULL); \ + obj = GET_LOCALE_STRING(ATTR); \ if (obj == NULL) { \ goto done; \ } \ @@ -200,14 +207,17 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc) RESULT_STRING(mon_decimal_point); RESULT_STRING(mon_thousands_sep); #undef RESULT_STRING +#undef GET_LOCALE_STRING res = 0; done: +#ifndef MS_WINDOWS if (loc != NULL) { setlocale(LC_CTYPE, oldloc); } PyMem_Free(oldloc); +#endif return res; } @@ -243,9 +253,15 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored)) Py_DECREF(obj); \ } while (0) +#ifdef MS_WINDOWS +/* Use _W_* fields of Windows struct lconv */ +#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1) +#else +#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(ls->ATTR, NULL) +#endif #define RESULT_STRING(s)\ do { \ - x = PyUnicode_DecodeLocale(lc->s, NULL); \ + x = GET_LOCALE_STRING(s); \ RESULT(#s, x); \ } while (0) @@ -274,8 +290,10 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored)) RESULT_INT(n_sign_posn); /* Numeric information: LC_NUMERIC encoding */ - PyObject *decimal_point, *thousands_sep; + PyObject *decimal_point = NULL, *thousands_sep = NULL; if (_Py_GetLocaleconvNumeric(lc, &decimal_point, &thousands_sep) < 0) { + Py_XDECREF(decimal_point); + Py_XDECREF(thousands_sep); goto failed; } @@ -304,6 +322,7 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored)) #undef RESULT #undef RESULT_STRING #undef RESULT_INT +#undef GET_LOCALE_STRING } #if defined(HAVE_WCSCOLL) diff --git a/Python/fileutils.c b/Python/fileutils.c index 439bc351596f7aa..fb1e35a826bbab5 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2016,6 +2016,7 @@ _Py_GetLocaleconvNumeric(struct lconv *lc, assert(decimal_point != NULL); assert(thousands_sep != NULL); +#ifndef MS_WINDOWS int change_locale = 0; if ((strlen(lc->decimal_point) > 1 || ((unsigned char)lc->decimal_point[0]) > 127)) { change_locale = 1; @@ -2054,14 +2055,20 @@ _Py_GetLocaleconvNumeric(struct lconv *lc, } } +#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL) +#else /* MS_WINDOWS */ +/* Use _W_* fields of Windows strcut lconv */ +#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1) +#endif /* MS_WINDOWS */ + int res = -1; - *decimal_point = PyUnicode_DecodeLocale(lc->decimal_point, NULL); + *decimal_point = GET_LOCALE_STRING(decimal_point); if (*decimal_point == NULL) { goto done; } - *thousands_sep = PyUnicode_DecodeLocale(lc->thousands_sep, NULL); + *thousands_sep = GET_LOCALE_STRING(thousands_sep); if (*thousands_sep == NULL) { goto done; } @@ -2069,9 +2076,13 @@ _Py_GetLocaleconvNumeric(struct lconv *lc, res = 0; done: +#ifndef MS_WINDOWS if (loc != NULL) { setlocale(LC_CTYPE, oldloc); } PyMem_Free(oldloc); +#endif return res; + +#undef GET_LOCALE_STRING } From c05b6afa5a739c959e0240dcf05c3e66e4c44447 Mon Sep 17 00:00:00 2001 From: TIGirardi Date: Fri, 29 May 2020 23:21:45 -0300 Subject: [PATCH 2/5] bpo-38324: Fix typo --- Modules/_localemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index dfab64268958782..2e353bba00bf35b 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -257,7 +257,7 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored)) /* Use _W_* fields of Windows struct lconv */ #define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1) #else -#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(ls->ATTR, NULL) +#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL) #endif #define RESULT_STRING(s)\ do { \ From e54d98e141c61c4b59ac7931adf90a315e614fd2 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 30 May 2020 02:46:43 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst diff --git a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst new file mode 100644 index 000000000000000..c6fa62ff231f4db --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst @@ -0,0 +1 @@ +Use wide-char _W_* fields of struct lconv on Windows \ No newline at end of file From 0208a88358de45b52d33cffb5ffa77c55cf1f9b2 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 19 Oct 2020 19:24:42 -0400 Subject: [PATCH 4/5] Update Lib/test/test__locale.py Co-authored-by: Steve Dower --- Lib/test/test__locale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 30afe80e39db274..59a00bad7d98a3d 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -72,7 +72,7 @@ def accept(loc): 'ps_AF': ('\u066b', '\u066c'), } -if uname().system == 'Windows': +if sys.platform == 'win32': # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) del known_numerics['ps_AF'] From 952ec52f11bbd0b36063a099e19052327b51e361 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 19 Oct 2020 19:25:08 -0400 Subject: [PATCH 5/5] Update Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst Co-authored-by: Steve Dower --- .../next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst index c6fa62ff231f4db..c45aa13091429e0 100644 --- a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst +++ b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst @@ -1 +1 @@ -Use wide-char _W_* fields of struct lconv on Windows \ No newline at end of file +Avoid Unicode errors when accessing certain locale data on Windows.