From 960abcda4a1e0d4e38473ffda31d29c779c8eb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hakan=20=C3=87elik?= Date: Sun, 24 May 2020 11:45:21 +0300 Subject: [PATCH 1/4] Support pathlike objects on dbm/shelve --- Lib/dbm/__init__.py | 2 + Lib/dbm/dumb.py | 1 + Lib/test/test_dbm.py | 52 +++++++++++-------- Lib/test/test_dbm_dumb.py | 4 ++ Lib/test/test_dbm_gnu.py | 4 ++ Lib/test/test_dbm_ndbm.py | 4 ++ Lib/test/test_shelve.py | 31 ++++++----- .../2020-05-21-01-42-32.bpo-40563.fDn5bP.rst | 1 + 8 files changed, 62 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-05-21-01-42-32.bpo-40563.fDn5bP.rst diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index f65da521af4da8..51164fb0068108 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -75,6 +75,7 @@ def open(file, flag='r', mode=0o666): raise ImportError("no dbm clone found; tried %s" % _names) # guess the type of an existing database, if not creating a new one + file = os.fspath(file) result = whichdb(file) if 'n' not in flag else None if result is None: # db doesn't exist or 'n' flag was specified to create a new db @@ -109,6 +110,7 @@ def whichdb(filename): """ # Check for ndbm first -- this has a .pag and a .dir file + filename = os.fspath(filename) try: f = io.open(filename + ".pag", "rb") f.close() diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 864ad371ec9525..0ad075bb6290d3 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -46,6 +46,7 @@ class _Database(collections.abc.MutableMapping): _io = _io # for _commit() def __init__(self, filebasename, mode, flag='c'): + filebasename = self._os.fspath(filebasename) self._mode = mode self._readonly = (flag == 'r') diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 1db3bef6f41367..e091baecbf2078 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -3,6 +3,7 @@ import unittest import glob import test.support +from pathlib import Path # Skip tests if dbm module doesn't exist. dbm = test.support.import_module('dbm') @@ -128,6 +129,9 @@ def test_anydbm_access(self): assert(f[key] == b"Python:") f.close() + def test_open_with_patlib_path(self): + dbm.open(Path(_fname), "c").close() + def read_helper(self, f): keys = self.keys_helper(f) for key in self._dict: @@ -143,34 +147,36 @@ def setUp(self): class WhichDBTestCase(unittest.TestCase): def test_whichdb(self): - for module in dbm_iterator(): - # Check whether whichdb correctly guesses module name - # for databases opened with "module" module. - # Try with empty files first - name = module.__name__ - if name == 'dbm.dumb': - continue # whichdb can't support dbm.dumb - delete_files() - f = module.open(_fname, 'c') - f.close() - self.assertEqual(name, self.dbm.whichdb(_fname)) - # Now add a key - f = module.open(_fname, 'w') - f[b"1"] = b"1" - # and test that we can find it - self.assertIn(b"1", f) - # and read it - self.assertEqual(f[b"1"], b"1") - f.close() - self.assertEqual(name, self.dbm.whichdb(_fname)) + for path in [_fname, Path(_fname)]: + for module in dbm_iterator(): + # Check whether whichdb correctly guesses module name + # for databases opened with "module" module. + # Try with empty files first + name = module.__name__ + if name == 'dbm.dumb': + continue # whichdb can't support dbm.dumb + delete_files() + f = module.open(path, 'c') + f.close() + self.assertEqual(name, self.dbm.whichdb(path)) + # Now add a key + f = module.open(path, 'w') + f[b"1"] = b"1" + # and test that we can find it + self.assertIn(b"1", f) + # and read it + self.assertEqual(f[b"1"], b"1") + f.close() + self.assertEqual(name, self.dbm.whichdb(path)) @unittest.skipUnless(ndbm, reason='Test requires ndbm') def test_whichdb_ndbm(self): # Issue 17198: check that ndbm which is referenced in whichdb is defined db_file = '{}_ndbm.db'.format(_fname) - with open(db_file, 'w'): - self.addCleanup(test.support.unlink, db_file) - self.assertIsNone(self.dbm.whichdb(db_file[:-3])) + for path in [db_file, Path(db_file)]: + with open(path, 'w'): + self.addCleanup(test.support.unlink, path) + self.assertIsNone(self.dbm.whichdb(path[:-3])) def tearDown(self): delete_files() diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index 0a60778207d96a..b23731a37accad 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -292,6 +292,10 @@ def test_nonascii_filename(self): self.assertTrue(b'key' in db) self.assertEqual(db[b'key'], b'value') + def test_open_with_patlib_path(self): + from pathlib import Path + dumbdbm.open(Path(_fname), "c").close() + def tearDown(self): _delete_files() diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index f1c7d34085c5ee..8774fa6a5d5317 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -162,6 +162,10 @@ def test_nonexisting_file(self): self.assertIn(nonexisting_file, str(cm.exception)) self.assertEqual(cm.exception.filename, nonexisting_file) + def test_open_with_patlib_path(self): + from pathlib import Path + gdbm.open(Path(filename), "c").close() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_dbm_ndbm.py b/Lib/test/test_dbm_ndbm.py index 7ac75c5fea1b5e..4b8a43b25ebc6f 100644 --- a/Lib/test/test_dbm_ndbm.py +++ b/Lib/test/test_dbm_ndbm.py @@ -123,6 +123,10 @@ def test_nonexisting_file(self): self.assertIn(nonexisting_file, str(cm.exception)) self.assertEqual(cm.exception.filename, nonexisting_file) + def test_open_with_patlib_path(self): + from pathlib import Path + dbm.ndbm.open(Path(self.filename), "c").close() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 9ffe2cbeae4d86..368b8c8c8c039b 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -62,29 +62,32 @@ def test_close(self): else: self.fail('Closed shelf should not find a key') - def test_ascii_file_shelf(self): - s = shelve.open(self.fn, protocol=0) + def test_open_template(self, filename=None, protocol=None): + kwargs = { + "filename": filename or self.fn, + } + if protocol: + kwargs["protocol"] = protocol + s = shelve.open(**kwargs) try: s['key1'] = (1,2,3,4) self.assertEqual(s['key1'], (1,2,3,4)) finally: s.close() + def test_ascii_file_shelf(self): + self.test_open_template(protocol=0) + def test_binary_file_shelf(self): - s = shelve.open(self.fn, protocol=1) - try: - s['key1'] = (1,2,3,4) - self.assertEqual(s['key1'], (1,2,3,4)) - finally: - s.close() + self.test_open_template(protocol=1) def test_proto2_file_shelf(self): - s = shelve.open(self.fn, protocol=2) - try: - s['key1'] = (1,2,3,4) - self.assertEqual(s['key1'], (1,2,3,4)) - finally: - s.close() + self.test_open_template(protocol=2) + + def test_patlib_path_file_shelf(self): + from pathlib import Path + self.test_open_template(filename=Path(self.fn)) + def test_in_memory_shelf(self): d1 = byteskeydict() diff --git a/Misc/NEWS.d/next/Library/2020-05-21-01-42-32.bpo-40563.fDn5bP.rst b/Misc/NEWS.d/next/Library/2020-05-21-01-42-32.bpo-40563.fDn5bP.rst new file mode 100644 index 00000000000000..f5e5bd1b2d3b11 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-21-01-42-32.bpo-40563.fDn5bP.rst @@ -0,0 +1 @@ +Support pathlike objects on dbm/shelve. Patch by Hakan Çelik. From 9ef0ec1ef68e89768e49d7d3cbdaf84113758515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hakan=20=C3=87elik?= Date: Sun, 24 May 2020 12:29:20 +0300 Subject: [PATCH 2/4] doc update --- Doc/library/dbm.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 57ae547b833cc0..9366ca70eb4f37 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -33,6 +33,8 @@ the Oracle Berkeley DB. file's format can't be guessed; or a string containing the required module name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'``. +.. versionchanged:: 3.10 + Accepts :term:`path-like object` for filename. .. function:: open(file, flag='r', mode=0o666) @@ -77,6 +79,9 @@ available, as well as :meth:`get` and :meth:`setdefault`. Deleting a key from a read-only database raises database module specific error instead of :exc:`KeyError`. +.. versionchanged:: 3.10 + Accepts :term:`path-like object` for file. + Key and values are always stored as bytes. This means that when strings are used they are implicitly converted to the default encoding before being stored. @@ -202,6 +207,9 @@ supported. In addition to the dictionary-like methods, ``gdbm`` objects have the following methods: + .. versionchanged:: 3.10 + Accepts :term:`path-like object` for filename. + .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the @@ -298,6 +306,9 @@ to locate the appropriate header file to simplify building this module. In addition to the dictionary-like methods, ``ndbm`` objects provide the following method: + .. versionchanged:: 3.10 + Accepts :term:`path-like object` for filename. + .. method:: ndbm.close() Close the ``ndbm`` database. @@ -379,6 +390,9 @@ The module defines the following: flags ``'r'`` and ``'w'`` no longer creates a database if it does not exist. + .. versionchanged:: 3.10 + Accepts :term:`path-like object` for filename. + In addition to the methods provided by the :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects provide the following methods: From 40d066dbe9d27b2407e1d644c916ee0865aca19d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hakan=20=C3=87elik?= Date: Sun, 24 May 2020 15:51:08 +0300 Subject: [PATCH 3/4] update _gdbmmodule.c --- Modules/_gdbmmodule.c | 14 ++++++++++++-- Modules/clinic/_gdbmmodule.c.h | 9 +-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index dd4c6b16f745cf..6f6f180155c673 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -569,7 +569,7 @@ static PyTypeObject Dbmtype = { /*[clinic input] _gdbm.open as dbmopen - filename: unicode + filename: object flags: str="r" mode: int(py_default="0o666") = 0o666 / @@ -601,7 +601,7 @@ when the database has to be created. It defaults to octal 0o666. static PyObject * dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, int mode) -/*[clinic end generated code: output=9527750f5df90764 input=3be0b0875974b928]*/ +/*[clinic end generated code: output=9527750f5df90764 input=0bf3159ce7952fee]*/ { int iflags; @@ -649,6 +649,16 @@ dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, } } + PyObject *filenamebytes = PyOS_FSPath(filename); + if (filenamebytes == NULL) + return NULL; + if (PyUnicode_Check(filenamebytes) { + PyObject *tmp = PyUnicode_EncodeFSDefault(filenamebytes); + Py_DECREF(filenamebytes); + filenamebytes = tmp; + if (tmp == NULL) + return NULL; + } PyObject *filenamebytes = PyUnicode_EncodeFSDefault(filename); if (filenamebytes == NULL) { return NULL; diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 195159104d9950..90c3fe594c8b53 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -256,13 +256,6 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("open", nargs, 1, 3)) { goto exit; } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("open", "argument 1", "str", args[0]); - goto exit; - } - if (PyUnicode_READY(args[0]) == -1) { - goto exit; - } filename = args[0]; if (nargs < 2) { goto skip_optional; @@ -293,4 +286,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c9d43f42677f4efb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0c87de857b014477 input=a9049054013a1b77]*/ From 0b8bc2dd74097d78e04a14c0d281a10b322a0504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hakan=20=C3=87elik?= Date: Sun, 7 Jun 2020 15:37:16 +0300 Subject: [PATCH 4/4] syntax errors fix --- Modules/_dbmmodule.c | 10 ++++++++++ Modules/_gdbmmodule.c | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 80a0503622c3fe..fde5b313105872 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -460,6 +460,16 @@ dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, return NULL; } + PyObject *filenamebytes = PyOS_FSPath(filename); + if (filenamebytes == NULL) + return NULL; + if (PyUnicode_Check(filenamebytes)) { + PyObject *tmp = PyUnicode_EncodeFSDefault(filenamebytes); + Py_DECREF(filenamebytes); + filenamebytes = tmp; + if (tmp == NULL) + return NULL; + } PyObject *filenamebytes = PyUnicode_EncodeFSDefault(filename); if (filenamebytes == NULL) { return NULL; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 6f6f180155c673..cf194a6449e84d 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -652,7 +652,7 @@ dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, PyObject *filenamebytes = PyOS_FSPath(filename); if (filenamebytes == NULL) return NULL; - if (PyUnicode_Check(filenamebytes) { + if (PyUnicode_Check(filenamebytes)) { PyObject *tmp = PyUnicode_EncodeFSDefault(filenamebytes); Py_DECREF(filenamebytes); filenamebytes = tmp;