Speedup ntpath.exists

Co-authored-by: Eryk Sun <eryksun@gmail.com>
python · nineteendo · Apr 13, 2024 · Apr 13, 2024 · Apr 13, 2024 · Apr 15, 2024
commit ab76c06e4b9e5116d46c8456bebdab3427589e72
diff --git a/Lib/ntpath.py b/Lib/ntpath.py
@@ -899,6 +899,7 @@ def commonpath(paths):
    from nt import _path_isfile as isfile
    from nt import _path_islink as islink
    from nt import _path_exists as exists
+    from nt import _path_lexists as lexists
 except ImportError:
    # Use genericpath.* as imported above
    pass

diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
@@ -128,31 +128,34 @@ def test_filetime(self):
        )

    def test_exists(self):
+        def check_exists(filename, expected):
+            bfilename = os.fsencode(filename)
+            self.assertIs(self.pathmodule.exists(filename), expected)
+            self.assertIs(self.pathmodule.exists(bfilename), expected)
+
+        def check_lexists(filename, expected):
+            bfilename = os.fsencode(filename)
+            self.assertIs(self.pathmodule.lexists(filename), expected)
+            self.assertIs(self.pathmodule.lexists(bfilename), expected)
+
        filename = os_helper.TESTFN
        bfilename = os.fsencode(filename)
        self.addCleanup(os_helper.unlink, filename)

-        self.assertIs(self.pathmodule.exists(filename), False)
-        self.assertIs(self.pathmodule.exists(bfilename), False)
+        check_exists(filename, False)
+        check_lexists(filename, False)

        create_file(filename)

-        self.assertIs(self.pathmodule.exists(filename), True)
-        self.assertIs(self.pathmodule.exists(bfilename), True)
-
+        check_exists(filename, True)
        self.assertIs(self.pathmodule.exists(filename + '\udfff'), False)
        self.assertIs(self.pathmodule.exists(bfilename + b'\xff'), False)
-        self.assertIs(self.pathmodule.exists(filename + '\x00'), False)
-        self.assertIs(self.pathmodule.exists(bfilename + b'\x00'), False)
-
-        if self.pathmodule is not genericpath:
-            self.assertIs(self.pathmodule.lexists(filename), True)
-            self.assertIs(self.pathmodule.lexists(bfilename), True)
+        check_exists(filename + '\x00', False)

-            self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False)
-            self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False)
-            self.assertIs(self.pathmodule.lexists(filename + '\x00'), False)
-            self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False)
+        check_lexists(filename, True)
+        self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False)
+        self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False)
+        check_lexists(filename + '\x00', False)

    @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
    @unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat")

diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
@@ -1115,6 +1115,8 @@ def test_fast_paths_in_use(self):
        self.assertFalse(inspect.isfunction(os.path.islink))
        self.assertTrue(os.path.exists is nt._path_exists)
        self.assertFalse(inspect.isfunction(os.path.exists))
+        self.assertTrue(os.path.lexists is nt._path_lexists)
+        self.assertFalse(inspect.isfunction(os.path.lexists))

    @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32")
    def test_isdevdrive(self):

diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
@@ -5280,25 +5280,12 @@ os__path_isfile_impl(PyObject *module, PyObject *path)
 }


-/*[clinic input]
-os._path_exists
-
-    path: 'O'
-
-Test whether a path exists.  Returns False for broken symbolic links
-
-[clinic start generated code]*/
-
 static PyObject *
-os__path_exists_impl(PyObject *module, PyObject *path)
-/*[clinic end generated code: output=f508c3b35e13a249 input=380f77cdfa0f7ae8]*/
+nt_exists(PyObject *path, int follow_symlinks)
 {
-    HANDLE hfile;
-    BOOL close_file = TRUE;
    path_t _path = PATH_T_INITIALIZE("exists", "path", 0, 1);
-    int result;
-    BOOL slow_path = TRUE;
-    FILE_STAT_BASIC_INFORMATION statInfo;
+    HANDLE hfile;
+    int result = 0;

    if (!path_converter(path, &_path)) {
        path_cleanup(&_path);
@@ -5310,49 +5297,56 @@ os__path_exists_impl(PyObject *module, PyObject *path)
    }

    Py_BEGIN_ALLOW_THREADS
-    if (_path.wide) {
+    if (_path.fd != -1) {
+        hfile = _Py_get_osfhandle_noraise(_path.fd);
+        if (hfile != INVALID_HANDLE_VALUE) {
+            result = 1;
+        }
+    }
+    else if (_path.wide) {
+        BOOL slow_path = TRUE;
+        FILE_STAT_BASIC_INFORMATION statInfo;
        if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo,
-                                         &statInfo, sizeof(statInfo))) {
-            if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
+                                         &statInfo, sizeof(statInfo)))
+        {
+            if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
+                !follow_symlinks &&
+                IsReparseTagNameSurrogate(statInfo.ReparseTag))
+            {
                slow_path = FALSE;
                result = 1;
            }
-        } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) {
-                    slow_path = FALSE;
-                    result = 0;
-        }
-    }
-    if (slow_path) {
-        if (_path.fd != -1) {
-            hfile = _Py_get_osfhandle_noraise(_path.fd);
-            close_file = FALSE;
-        }
-        else {
-            hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL,
-                                OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
-        }
-        if (hfile != INVALID_HANDLE_VALUE) {
-            result = 1;
-            if (close_file) {
-                CloseHandle(hfile);
-            }
+        } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(
+                        GetLastError()))
+        {
+            slow_path = FALSE;
        }
-        else {
+        if (slow_path) {
            STRUCT_STAT st;
-            switch (GetLastError()) {
-            case ERROR_ACCESS_DENIED:
-            case ERROR_SHARING_VIOLATION:
-            case ERROR_CANT_ACCESS_FILE:
-            case ERROR_INVALID_PARAMETER:
-                if (STAT(_path.wide, &st)) {
-                    result = 0;
+            if (!follow_symlinks) {
+                if (!LSTAT(_path.wide, &st)) {
+                    result = 1;
                }
-                else {
+            }
+            else {
+                hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL,
+                                    OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS,
+                                    NULL);
+                if (hfile != INVALID_HANDLE_VALUE) {
+                    CloseHandle(hfile);
                    result = 1;
                }
-                break;
-            default:
-                result = 0;
+                else {
+                    switch (GetLastError()) {
+                    case ERROR_ACCESS_DENIED:
+                    case ERROR_SHARING_VIOLATION:
+                    case ERROR_CANT_ACCESS_FILE:
+                    case ERROR_INVALID_PARAMETER:
+                        if (!STAT(_path.wide, &st)) {
+                            result = 1;
+                        }
+                    }
+                }
            }
        }
    }
@@ -5366,6 +5360,40 @@ os__path_exists_impl(PyObject *module, PyObject *path)
 }


+/*[clinic input]
+os._path_exists
+
+    path: 'O'
+
+Test whether a path exists.  Returns False for broken symbolic links
+
+[clinic start generated code]*/
+
+static PyObject *
+os__path_exists_impl(PyObject *module, PyObject *path)
+/*[clinic end generated code: output=f508c3b35e13a249 input=380f77cdfa0f7ae8]*/
+{
+    return nt_exists(path, 1);
+}
+
+
+/*[clinic input]
+os._path_lexists
+
+    path: 'O'
+
+Test whether a path exists.  Returns True for broken symbolic links
+
+[clinic start generated code]*/
+
+static PyObject *
+os__path_lexists_impl(PyObject *module, PyObject *path)
+/*[clinic end generated code: output=b9a42a50b1df6651 input=5f14942c64196ba2]*/
+{
+    return nt_exists(path, 0);
+}
+
+
 /*[clinic input]
 os._path_islink

@@ -16843,6 +16871,7 @@ static PyMethodDef posix_methods[] = {
    OS__PATH_ISFILE_METHODDEF
    OS__PATH_ISLINK_METHODDEF
    OS__PATH_EXISTS_METHODDEF
+    OS__PATH_LEXISTS_METHODDEF

    OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF
    {NULL,              NULL}            /* Sentinel */