From 3d9816b9a65bb3ed5fead3b70da52478679e39bf Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 23 Mar 2025 19:36:12 +0100 Subject: [PATCH 01/17] Add typehints to pathlib.types --- Lib/pathlib/types.py | 94 +++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index 85dd9e5b2d6b9a..703e8169367b48 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -14,7 +14,15 @@ from glob import _PathGlobber from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj from pathlib import PurePath, Path -from typing import Optional, Protocol, runtime_checkable +from typing import ( + Any, BinaryIO, Callable, Generator, Optional, Protocol, Sequence, TypeVar, Union, + runtime_checkable, +) + + +_JP = TypeVar("_JP", bound="_JoinablePath") +_RP = TypeVar("_RP", bound="_ReadablePath") +_WP = TypeVar("_WP", bound="_WritablePath") def _explode_path(path): @@ -72,14 +80,14 @@ class _JoinablePath(ABC): @property @abstractmethod - def parser(self): + def parser(self) -> _PathParser: """Implementation of pathlib._types.Parser used for low-level path parsing and manipulation. """ raise NotImplementedError @abstractmethod - def with_segments(self, *pathsegments): + def with_segments(self: _JP, *pathsegments: Union[_JP, str]) -> _JP: """Construct a new path object from any number of path-like objects. Subclasses may override this method to customize how new path objects are created from methods like `iterdir()`. @@ -87,23 +95,23 @@ def with_segments(self, *pathsegments): raise NotImplementedError @abstractmethod - def __str__(self): + def __str__(self) -> str: """Return the string representation of the path, suitable for passing to system calls.""" raise NotImplementedError @property - def anchor(self): + def anchor(self) -> str: """The concatenation of the drive and root, or ''.""" return _explode_path(self)[0] @property - def name(self): + def name(self) -> str: """The final path component, if any.""" return self.parser.split(str(self))[1] @property - def suffix(self): + def suffix(self) -> str: """ The final component's last suffix, if any. @@ -112,7 +120,7 @@ def suffix(self): return self.parser.splitext(self.name)[1] @property - def suffixes(self): + def suffixes(self) -> Sequence[str]: """ A list of the final component's suffixes, if any. @@ -127,11 +135,11 @@ def suffixes(self): return suffixes[::-1] @property - def stem(self): + def stem(self) -> str: """The final path component, minus its last suffix.""" return self.parser.splitext(self.name)[0] - def with_name(self, name): + def with_name(self: _JP, name: str) -> _JP: """Return a new path with the file name changed.""" split = self.parser.split if split(name)[0]: @@ -140,7 +148,7 @@ def with_name(self, name): path = path.removesuffix(split(path)[1]) + name return self.with_segments(path) - def with_stem(self, stem): + def with_stem(self: _JP, stem: str) -> _JP: """Return a new path with the stem changed.""" suffix = self.suffix if not suffix: @@ -151,7 +159,7 @@ def with_stem(self, stem): else: return self.with_name(stem + suffix) - def with_suffix(self, suffix): + def with_suffix(self: _JP, suffix: str) -> _JP: """Return a new path with the file suffix changed. If the path has no suffix, add given suffix. If the given suffix is an empty string, remove the suffix from the path. @@ -166,7 +174,7 @@ def with_suffix(self, suffix): return self.with_name(stem + suffix) @property - def parts(self): + def parts(self) -> Sequence[str]: """An object providing sequence-like access to the components in the filesystem path.""" anchor, parts = _explode_path(self) @@ -174,7 +182,7 @@ def parts(self): parts.append(anchor) return tuple(reversed(parts)) - def joinpath(self, *pathsegments): + def joinpath(self: _JP, *pathsegments: Union[_JP, str]) -> _JP: """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative paths) or a totally different path (if one of the arguments is @@ -182,20 +190,20 @@ def joinpath(self, *pathsegments): """ return self.with_segments(str(self), *pathsegments) - def __truediv__(self, key): + def __truediv__(self: _JP, key: Union[_JP, str]) -> _JP: try: return self.with_segments(str(self), key) except TypeError: return NotImplemented - def __rtruediv__(self, key): + def __rtruediv__(self: _JP, key: Union[_JP, str]) -> _JP: try: return self.with_segments(key, str(self)) except TypeError: return NotImplemented @property - def parent(self): + def parent(self: _JP) -> _JP: """The logical parent of the path.""" path = str(self) parent = self.parser.split(path)[0] @@ -204,7 +212,7 @@ def parent(self): return self @property - def parents(self): + def parents(self: _JP) -> Sequence[_JP]: """A sequence of this path's logical parents.""" split = self.parser.split path = str(self) @@ -216,7 +224,7 @@ def parents(self): parent = split(path)[0] return tuple(parents) - def full_match(self, pattern): + def full_match(self: _JP, pattern: Union[_JP, str]) -> bool: """ Return True if this path matches the given glob-style pattern. The pattern is matched against the entire path. @@ -240,7 +248,7 @@ class _ReadablePath(_JoinablePath): @property @abstractmethod - def info(self): + def info(self) -> PathInfo: """ A PathInfo object that exposes the file type and other file attributes of this path. @@ -248,21 +256,26 @@ def info(self): raise NotImplementedError @abstractmethod - def __open_rb__(self, buffering=-1): + def __open_rb__(self, buffering: int = -1) -> BinaryIO: """ Open the file pointed to by this path for reading in binary mode and return a file object, like open(mode='rb'). """ raise NotImplementedError - def read_bytes(self): + def read_bytes(self) -> bytes: """ Open the file in bytes mode, read it, and close the file. """ with magic_open(self, mode='rb', buffering=0) as f: return f.read() - def read_text(self, encoding=None, errors=None, newline=None): + def read_text( + self, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + ) -> str: """ Open the file in text mode, read it, and close the file. """ @@ -270,7 +283,7 @@ def read_text(self, encoding=None, errors=None, newline=None): return f.read() @abstractmethod - def iterdir(self): + def iterdir(self: _RP) -> Generator[_RP, None, None]: """Yield path objects of the directory contents. The children are yielded in arbitrary order, and the @@ -278,7 +291,7 @@ def iterdir(self): """ raise NotImplementedError - def glob(self, pattern, *, recurse_symlinks=True): + def glob(self: _RP, pattern: Union[_RP, str], *, recurse_symlinks: bool = True) -> Generator[_RP, None, None]: """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ @@ -294,7 +307,12 @@ def glob(self, pattern, *, recurse_symlinks=True): select = globber.selector(parts) return select(self.joinpath('')) - def walk(self, top_down=True, on_error=None, follow_symlinks=False): + def walk( + self: _RP, + top_down: bool = True, + on_error: Optional[Callable[[Exception], None]] = None, + follow_symlinks: bool = False, + ) -> Generator[tuple[_RP, list[str], list[str]], None, None]: """Walk the directory tree from this directory, similar to os.walk().""" paths = [self] while paths: @@ -326,13 +344,13 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): paths += [path.joinpath(d) for d in reversed(dirnames)] @abstractmethod - def readlink(self): + def readlink(self: _RP) -> _RP: """ Return the path to which the symbolic link points. """ raise NotImplementedError - def copy(self, target, **kwargs): + def copy(self, target: _WP, **kwargs: Any) -> _WP: """ Recursively copy this file or directory tree to the given destination. """ @@ -346,7 +364,7 @@ def copy(self, target, **kwargs): copy_to_target(self, **kwargs) return target.joinpath() # Empty join to ensure fresh metadata. - def copy_into(self, target_dir, **kwargs): + def copy_into(self, target_dir: _WP, **kwargs: Any) -> _WP: """ Copy this file or directory tree into the given existing directory. """ @@ -370,7 +388,7 @@ class _WritablePath(_JoinablePath): __slots__ = () @abstractmethod - def symlink_to(self, target, target_is_directory=False): + def symlink_to(self: _WP, target: _WP, target_is_directory: bool = False) -> None: """ Make this path a symlink pointing to the target path. Note the order of arguments (link, target) is the reverse of os.symlink. @@ -378,21 +396,21 @@ def symlink_to(self, target, target_is_directory=False): raise NotImplementedError @abstractmethod - def mkdir(self): + def mkdir(self) -> None: """ Create a new directory at this given path. """ raise NotImplementedError @abstractmethod - def __open_wb__(self, buffering=-1): + def __open_wb__(self, buffering: int = -1) -> BinaryIO: """ Open the file pointed to by this path for writing in binary mode and return a file object, like open(mode='wb'). """ raise NotImplementedError - def write_bytes(self, data): + def write_bytes(self, data: bytes) -> int: """ Open the file in bytes mode, write to it, and close the file. """ @@ -401,7 +419,13 @@ def write_bytes(self, data): with magic_open(self, mode='wb') as f: return f.write(view) - def write_text(self, data, encoding=None, errors=None, newline=None): + def write_text( + self, + data: str, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + ) -> int: """ Open the file in text mode, write to it, and close the file. """ @@ -411,7 +435,7 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with magic_open(self, mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) - def _copy_from(self, source, follow_symlinks=True): + def _copy_from(self, source: _ReadablePath, follow_symlinks: bool = True) -> None: """ Recursively copy the given path to this path. """ From 310c245d8bda2b414333bc0674140a456d40e88f Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Mon, 24 Mar 2025 11:00:52 +0100 Subject: [PATCH 02/17] Iterator type for .glob and .iterdir --- Lib/pathlib/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index 703e8169367b48..2c42acbb4dcd79 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -15,7 +15,7 @@ from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj from pathlib import PurePath, Path from typing import ( - Any, BinaryIO, Callable, Generator, Optional, Protocol, Sequence, TypeVar, Union, + Any, BinaryIO, Callable, Generator, Iterator, Optional, Protocol, Sequence, TypeVar, Union, runtime_checkable, ) @@ -283,7 +283,7 @@ def read_text( return f.read() @abstractmethod - def iterdir(self: _RP) -> Generator[_RP, None, None]: + def iterdir(self: _RP) -> Iterator[_RP]: """Yield path objects of the directory contents. The children are yielded in arbitrary order, and the @@ -291,7 +291,7 @@ def iterdir(self: _RP) -> Generator[_RP, None, None]: """ raise NotImplementedError - def glob(self: _RP, pattern: Union[_RP, str], *, recurse_symlinks: bool = True) -> Generator[_RP, None, None]: + def glob(self: _RP, pattern: Union[_RP, str], *, recurse_symlinks: bool = True) -> Iterator[_RP]: """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ From d1583c02e0d883cca70cc6af8736dba326da8c1d Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Mon, 24 Mar 2025 22:28:10 +0100 Subject: [PATCH 03/17] Update types - pathsegments are `str`: joinpath, __truediv__, __rtruediv__, with_segments - symlink_to target is `str` - glob recurse_symlinks is `Literal[True]` - full_match and glob pattern is `str` --- Lib/pathlib/types.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index 47d64acc58c1d0..3960355d2d35b6 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -15,17 +15,16 @@ from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj from pathlib import PurePath, Path from typing import ( - Any, BinaryIO, Callable, Generator, Iterator, Optional, Protocol, Sequence, TypeVar, Union, + Any, BinaryIO, Callable, Generator, Iterator, Literal, Optional, Protocol, Sequence, TypeVar, runtime_checkable, ) - _JP = TypeVar("_JP", bound="_JoinablePath") _RP = TypeVar("_RP", bound="_ReadablePath") _WP = TypeVar("_WP", bound="_WritablePath") -def _explode_path(path, split): +def _explode_path(path: str, split: Callable[[str], tuple[str, str]]) -> tuple[str, list[str]]: """ Split the path into a 2-tuple (anchor, parts), where *anchor* is the uppermost parent of the path (equivalent to path.parents[-1]), and @@ -85,7 +84,7 @@ def parser(self) -> _PathParser: raise NotImplementedError @abstractmethod - def with_segments(self: _JP, *pathsegments: Union[_JP, str]) -> _JP: + def with_segments(self: _JP, *pathsegments: str) -> _JP: """Construct a new path object from any number of path-like objects. Subclasses may override this method to customize how new path objects are created from methods like `iterdir()`. @@ -180,7 +179,7 @@ def parts(self) -> Sequence[str]: parts.append(anchor) return tuple(reversed(parts)) - def joinpath(self: _JP, *pathsegments: Union[_JP, str]) -> _JP: + def joinpath(self: _JP, *pathsegments: str) -> _JP: """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative paths) or a totally different path (if one of the arguments is @@ -188,13 +187,13 @@ def joinpath(self: _JP, *pathsegments: Union[_JP, str]) -> _JP: """ return self.with_segments(str(self), *pathsegments) - def __truediv__(self: _JP, key: Union[_JP, str]) -> _JP: + def __truediv__(self: _JP, key: str) -> _JP: try: return self.with_segments(str(self), key) except TypeError: return NotImplemented - def __rtruediv__(self: _JP, key: Union[_JP, str]) -> _JP: + def __rtruediv__(self: _JP, key: str) -> _JP: try: return self.with_segments(key, str(self)) except TypeError: @@ -222,7 +221,7 @@ def parents(self: _JP) -> Sequence[_JP]: parent = split(path)[0] return tuple(parents) - def full_match(self: _JP, pattern: Union[_JP, str]) -> bool: + def full_match(self: _JP, pattern: str) -> bool: """ Return True if this path matches the given glob-style pattern. The pattern is matched against the entire path. @@ -287,7 +286,7 @@ def iterdir(self: _RP) -> Iterator[_RP]: """ raise NotImplementedError - def glob(self: _RP, pattern: Union[_RP, str], *, recurse_symlinks: bool = True) -> Iterator[_RP]: + def glob(self: _RP, pattern: str, *, recurse_symlinks: Literal[True] = True) -> Iterator[_RP]: """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ @@ -374,7 +373,7 @@ class _WritablePath(_JoinablePath): __slots__ = () @abstractmethod - def symlink_to(self: _WP, target: _WP, target_is_directory: bool = False) -> None: + def symlink_to(self, target: str, target_is_directory: bool = False) -> None: """ Make this path a symlink pointing to the target path. Note the order of arguments (link, target) is the reverse of os.symlink. From 50a8f885f00adf2c66f441aa29972d9a676825f3 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Mon, 24 Mar 2025 23:04:54 +0100 Subject: [PATCH 04/17] Update pathlib.types to use Self --- Lib/pathlib/types.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index 3960355d2d35b6..1278e7bf8b6ffc 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -15,12 +15,10 @@ from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj from pathlib import PurePath, Path from typing import ( - Any, BinaryIO, Callable, Generator, Iterator, Literal, Optional, Protocol, Sequence, TypeVar, + Any, BinaryIO, Callable, Generator, Iterator, Literal, Optional, Protocol, Self, Sequence, TypeVar, runtime_checkable, ) -_JP = TypeVar("_JP", bound="_JoinablePath") -_RP = TypeVar("_RP", bound="_ReadablePath") _WP = TypeVar("_WP", bound="_WritablePath") @@ -84,7 +82,7 @@ def parser(self) -> _PathParser: raise NotImplementedError @abstractmethod - def with_segments(self: _JP, *pathsegments: str) -> _JP: + def with_segments(self, *pathsegments: str) -> Self: """Construct a new path object from any number of path-like objects. Subclasses may override this method to customize how new path objects are created from methods like `iterdir()`. @@ -136,7 +134,7 @@ def stem(self) -> str: """The final path component, minus its last suffix.""" return self.parser.splitext(self.name)[0] - def with_name(self: _JP, name: str) -> _JP: + def with_name(self, name: str) -> Self: """Return a new path with the file name changed.""" split = self.parser.split if split(name)[0]: @@ -145,7 +143,7 @@ def with_name(self: _JP, name: str) -> _JP: path = path.removesuffix(split(path)[1]) + name return self.with_segments(path) - def with_stem(self: _JP, stem: str) -> _JP: + def with_stem(self, stem: str) -> Self: """Return a new path with the stem changed.""" suffix = self.suffix if not suffix: @@ -156,7 +154,7 @@ def with_stem(self: _JP, stem: str) -> _JP: else: return self.with_name(stem + suffix) - def with_suffix(self: _JP, suffix: str) -> _JP: + def with_suffix(self, suffix: str) -> Self: """Return a new path with the file suffix changed. If the path has no suffix, add given suffix. If the given suffix is an empty string, remove the suffix from the path. @@ -179,7 +177,7 @@ def parts(self) -> Sequence[str]: parts.append(anchor) return tuple(reversed(parts)) - def joinpath(self: _JP, *pathsegments: str) -> _JP: + def joinpath(self, *pathsegments: str) -> Self: """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative paths) or a totally different path (if one of the arguments is @@ -187,20 +185,20 @@ def joinpath(self: _JP, *pathsegments: str) -> _JP: """ return self.with_segments(str(self), *pathsegments) - def __truediv__(self: _JP, key: str) -> _JP: + def __truediv__(self, key: str) -> Self: try: return self.with_segments(str(self), key) except TypeError: return NotImplemented - def __rtruediv__(self: _JP, key: str) -> _JP: + def __rtruediv__(self, key: str) -> Self: try: return self.with_segments(key, str(self)) except TypeError: return NotImplemented @property - def parent(self: _JP) -> _JP: + def parent(self) -> Self: """The logical parent of the path.""" path = str(self) parent = self.parser.split(path)[0] @@ -209,7 +207,7 @@ def parent(self: _JP) -> _JP: return self @property - def parents(self: _JP) -> Sequence[_JP]: + def parents(self) -> Sequence[Self]: """A sequence of this path's logical parents.""" split = self.parser.split path = str(self) @@ -221,7 +219,7 @@ def parents(self: _JP) -> Sequence[_JP]: parent = split(path)[0] return tuple(parents) - def full_match(self: _JP, pattern: str) -> bool: + def full_match(self, pattern: str) -> bool: """ Return True if this path matches the given glob-style pattern. The pattern is matched against the entire path. @@ -278,7 +276,7 @@ def read_text( return f.read() @abstractmethod - def iterdir(self: _RP) -> Iterator[_RP]: + def iterdir(self) -> Iterator[Self]: """Yield path objects of the directory contents. The children are yielded in arbitrary order, and the @@ -286,7 +284,7 @@ def iterdir(self: _RP) -> Iterator[_RP]: """ raise NotImplementedError - def glob(self: _RP, pattern: str, *, recurse_symlinks: Literal[True] = True) -> Iterator[_RP]: + def glob(self, pattern: str, *, recurse_symlinks: Literal[True] = True) -> Iterator[Self]: """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ @@ -303,11 +301,11 @@ def glob(self: _RP, pattern: str, *, recurse_symlinks: Literal[True] = True) -> return select(self.joinpath('')) def walk( - self: _RP, + self, top_down: bool = True, on_error: Optional[Callable[[Exception], None]] = None, follow_symlinks: bool = False, - ) -> Generator[tuple[_RP, list[str], list[str]], None, None]: + ) -> Generator[tuple[Self, list[str], list[str]], None, None]: """Walk the directory tree from this directory, similar to os.walk().""" paths = [self] while paths: @@ -339,7 +337,7 @@ def walk( paths += [path.joinpath(d) for d in reversed(dirnames)] @abstractmethod - def readlink(self: _RP) -> _RP: + def readlink(self) -> Self: """ Return the path to which the symbolic link points. """ From e21b8840b49addc7e5bb4649b849a8dd364ef8df Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Mon, 24 Mar 2025 23:05:19 +0100 Subject: [PATCH 05/17] Rely on python3.13 Generator defaults --- Lib/pathlib/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index 1278e7bf8b6ffc..a8bf935d99e33b 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -305,7 +305,7 @@ def walk( top_down: bool = True, on_error: Optional[Callable[[Exception], None]] = None, follow_symlinks: bool = False, - ) -> Generator[tuple[Self, list[str], list[str]], None, None]: + ) -> Generator[tuple[Self, list[str], list[str]]]: """Walk the directory tree from this directory, similar to os.walk().""" paths = [self] while paths: From b004c2da4f033e5980a0033d0b583b9698921a2b Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 26 Mar 2025 23:05:36 +0100 Subject: [PATCH 06/17] Switch walk to Iterator[...] --- Lib/pathlib/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index a8bf935d99e33b..470ecb02bd1bfe 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -305,7 +305,7 @@ def walk( top_down: bool = True, on_error: Optional[Callable[[Exception], None]] = None, follow_symlinks: bool = False, - ) -> Generator[tuple[Self, list[str], list[str]]]: + ) -> Iterator[tuple[Self, list[str], list[str]]]: """Walk the directory tree from this directory, similar to os.walk().""" paths = [self] while paths: From b1de80ec64a70fe2302cd34555d030723ce002d2 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 26 Mar 2025 23:07:41 +0100 Subject: [PATCH 07/17] Use abstract base classes from collections.abc --- Lib/pathlib/types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index 470ecb02bd1bfe..a5ebd7fa4fdb3a 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -11,11 +11,12 @@ from abc import ABC, abstractmethod +from collections.abc import Callable, Iterator, Sequence from glob import _PathGlobber from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj from pathlib import PurePath, Path from typing import ( - Any, BinaryIO, Callable, Generator, Iterator, Literal, Optional, Protocol, Self, Sequence, TypeVar, + Any, BinaryIO, Literal, Optional, Protocol, Self, TypeVar, runtime_checkable, ) From 4aa254005169e48f714b443f4518ce06b39fc508 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 26 Mar 2025 23:08:25 +0100 Subject: [PATCH 08/17] Use _PathParser in _explode_path --- Lib/pathlib/types.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index a5ebd7fa4fdb3a..39c1fb02977431 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -23,12 +23,13 @@ _WP = TypeVar("_WP", bound="_WritablePath") -def _explode_path(path: str, split: Callable[[str], tuple[str, str]]) -> tuple[str, list[str]]: +def _explode_path(path: str, parser: "_PathParser") -> tuple[str, list[str]]: """ Split the path into a 2-tuple (anchor, parts), where *anchor* is the uppermost parent of the path (equivalent to path.parents[-1]), and *parts* is a reversed list of parts following the anchor. """ + split = parser.split parent, name = split(path) names = [] while path != parent: @@ -99,7 +100,7 @@ def __str__(self) -> str: @property def anchor(self) -> str: """The concatenation of the drive and root, or ''.""" - return _explode_path(str(self), self.parser.split)[0] + return _explode_path(str(self), self.parser)[0] @property def name(self) -> str: @@ -173,7 +174,7 @@ def with_suffix(self, suffix: str) -> Self: def parts(self) -> Sequence[str]: """An object providing sequence-like access to the components in the filesystem path.""" - anchor, parts = _explode_path(str(self), self.parser.split) + anchor, parts = _explode_path(str(self), self.parser) if anchor: parts.append(anchor) return tuple(reversed(parts)) @@ -289,7 +290,7 @@ def glob(self, pattern: str, *, recurse_symlinks: Literal[True] = True) -> Itera """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ - anchor, parts = _explode_path(pattern, self.parser.split) + anchor, parts = _explode_path(pattern, self.parser) if anchor: raise NotImplementedError("Non-relative patterns are unsupported") elif not parts: From 54acfa3879acb087ae1415237a343d31c8d695d5 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 26 Mar 2025 23:09:41 +0100 Subject: [PATCH 09/17] Rename TypeVar to _WritablePathT --- Lib/pathlib/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index 39c1fb02977431..a72e8ef94d217d 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -20,7 +20,7 @@ runtime_checkable, ) -_WP = TypeVar("_WP", bound="_WritablePath") +_WritablePathT = TypeVar("_WritablePathT", bound="_WritablePath") def _explode_path(path: str, parser: "_PathParser") -> tuple[str, list[str]]: @@ -345,7 +345,7 @@ def readlink(self) -> Self: """ raise NotImplementedError - def copy(self, target: _WP, **kwargs: Any) -> _WP: + def copy(self, target: _WritablePathT, **kwargs: Any) -> _WritablePathT: """ Recursively copy this file or directory tree to the given destination. """ @@ -353,7 +353,7 @@ def copy(self, target: _WP, **kwargs: Any) -> _WP: target._copy_from(self, **kwargs) return target.joinpath() # Empty join to ensure fresh metadata. - def copy_into(self, target_dir: _WP, **kwargs: Any) -> _WP: + def copy_into(self, target_dir: _WritablePathT, **kwargs: Any) -> _WritablePathT: """ Copy this file or directory tree into the given existing directory. """ From d2dc4b138b105fd67a813afff9383870108512dc Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 26 Mar 2025 23:37:19 +0100 Subject: [PATCH 10/17] Move _explode_path --- Lib/pathlib/types.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index a72e8ef94d217d..a7f56944b30ea2 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -23,22 +23,6 @@ _WritablePathT = TypeVar("_WritablePathT", bound="_WritablePath") -def _explode_path(path: str, parser: "_PathParser") -> tuple[str, list[str]]: - """ - Split the path into a 2-tuple (anchor, parts), where *anchor* is the - uppermost parent of the path (equivalent to path.parents[-1]), and - *parts* is a reversed list of parts following the anchor. - """ - split = parser.split - parent, name = split(path) - names = [] - while path != parent: - names.append(name) - path = parent - parent, name = split(path) - return path, names - - @runtime_checkable class _PathParser(Protocol): """Protocol for path parsers, which do low-level path manipulation. @@ -55,6 +39,22 @@ def splitext(self, path: str) -> tuple[str, str]: ... def normcase(self, path: str) -> str: ... +def _explode_path(path: str, parser: _PathParser) -> tuple[str, list[str]]: + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + split = parser.split + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + + @runtime_checkable class PathInfo(Protocol): """Protocol for path info objects, which support querying the file type. From 44b1b6f7973bdf709a7282b2cda9064a41fe78c8 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Thu, 27 Mar 2025 08:06:19 +0100 Subject: [PATCH 11/17] Update Lib/pathlib/types.py --- Lib/pathlib/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index a7f56944b30ea2..c44f12f6600588 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -420,7 +420,7 @@ def write_text( with magic_open(self, mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) - def _copy_from(self, source: _ReadablePath, follow_symlinks: bool = True) -> None: + def _copy_from(self, source: _ReadablePath, *, follow_symlinks: bool = True) -> None: """ Recursively copy the given path to this path. """ From 73d79da92e4c12de92cab06efa166a54f521595e Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 23 Apr 2025 09:11:41 +0200 Subject: [PATCH 12/17] pathlib.types: don't rely on typing.Optional --- Lib/pathlib/types.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index c44f12f6600588..9920754fab0d7f 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -16,7 +16,7 @@ from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj from pathlib import PurePath, Path from typing import ( - Any, BinaryIO, Literal, Optional, Protocol, Self, TypeVar, + Any, BinaryIO, Literal, Protocol, Self, TypeVar, runtime_checkable, ) @@ -33,7 +33,7 @@ class _PathParser(Protocol): """ sep: str - altsep: Optional[str] + altsep: str | None def split(self, path: str) -> tuple[str, str]: ... def splitext(self, path: str) -> tuple[str, str]: ... def normcase(self, path: str) -> str: ... @@ -267,9 +267,9 @@ def read_bytes(self) -> bytes: def read_text( self, - encoding: Optional[str] = None, - errors: Optional[str] = None, - newline: Optional[str] = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, ) -> str: """ Open the file in text mode, read it, and close the file. @@ -305,7 +305,7 @@ def glob(self, pattern: str, *, recurse_symlinks: Literal[True] = True) -> Itera def walk( self, top_down: bool = True, - on_error: Optional[Callable[[Exception], None]] = None, + on_error: Callable[[Exception], None] | None = None, follow_symlinks: bool = False, ) -> Iterator[tuple[Self, list[str], list[str]]]: """Walk the directory tree from this directory, similar to os.walk().""" @@ -407,9 +407,9 @@ def write_bytes(self, data: bytes) -> int: def write_text( self, data: str, - encoding: Optional[str] = None, - errors: Optional[str] = None, - newline: Optional[str] = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, ) -> int: """ Open the file in text mode, write to it, and close the file. From 965c3c74a5a47b43272e91fb24ab43296125b6af Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 23 Apr 2025 11:49:08 +0200 Subject: [PATCH 13/17] avoid typing imports that are not needed at runtime --- Lib/pathlib/types.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index 9920754fab0d7f..e3e8ef38bf034b 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -15,12 +15,11 @@ from glob import _PathGlobber from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj from pathlib import PurePath, Path -from typing import ( - Any, BinaryIO, Literal, Protocol, Self, TypeVar, - runtime_checkable, -) +from typing import Protocol, runtime_checkable -_WritablePathT = TypeVar("_WritablePathT", bound="_WritablePath") +# typing +if False: + from typing import Any, BinaryIO, Literal, Self @runtime_checkable @@ -345,7 +344,7 @@ def readlink(self) -> Self: """ raise NotImplementedError - def copy(self, target: _WritablePathT, **kwargs: Any) -> _WritablePathT: + def copy[T: _WritablePath](self, target: T, **kwargs: Any) -> T: """ Recursively copy this file or directory tree to the given destination. """ @@ -353,7 +352,7 @@ def copy(self, target: _WritablePathT, **kwargs: Any) -> _WritablePathT: target._copy_from(self, **kwargs) return target.joinpath() # Empty join to ensure fresh metadata. - def copy_into(self, target_dir: _WritablePathT, **kwargs: Any) -> _WritablePathT: + def copy_into[T: _WritablePath](self, target_dir: T, **kwargs: Any) -> T: """ Copy this file or directory tree into the given existing directory. """ From b952ab0baad1a09c0918bf0708d43dad45a7d226 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Thu, 24 Apr 2025 00:15:52 +0200 Subject: [PATCH 14/17] add pathlib to Misc/mypy tests --- .github/workflows/mypy.yml | 1 + Lib/pathlib/mypy.ini | 25 +++++++++++++++++++++++++ Misc/mypy/pathlib | 1 + Misc/mypy/typed-stdlib.txt | 1 + 4 files changed, 28 insertions(+) create mode 100644 Lib/pathlib/mypy.ini create mode 120000 Misc/mypy/pathlib diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 212f3e8d70c836..7b0e2c54c8d03b 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -41,6 +41,7 @@ jobs: matrix: target: [ "Lib/_pyrepl", + "Lib/pathlib", "Lib/test/libregrtest", "Tools/build", "Tools/cases_generator", diff --git a/Lib/pathlib/mypy.ini b/Lib/pathlib/mypy.ini new file mode 100644 index 00000000000000..ce804ca1bb6f1f --- /dev/null +++ b/Lib/pathlib/mypy.ini @@ -0,0 +1,25 @@ +# Config file for running mypy on pathlib. +# Run mypy by invoking `mypy --config-file Lib/pathlib/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/pathlib +mypy_path = $MYPY_CONFIG_FILE_DIR/../../Misc/mypy +explicit_package_bases = True +python_version = 3.13 +platform = linux +pretty = True + +# ... Enable most stricter settings +enable_error_code = ignore-without-code,redundant-expr +strict = True + +# can't enable before glob isn't typed ... +warn_return_any = False +disable_error_code = attr-defined + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False diff --git a/Misc/mypy/pathlib b/Misc/mypy/pathlib new file mode 120000 index 00000000000000..d4940a89e63c24 --- /dev/null +++ b/Misc/mypy/pathlib @@ -0,0 +1 @@ +../../Lib/pathlib \ No newline at end of file diff --git a/Misc/mypy/typed-stdlib.txt b/Misc/mypy/typed-stdlib.txt index 8cd6858b4e591e..19338a29b3f0fa 100644 --- a/Misc/mypy/typed-stdlib.txt +++ b/Misc/mypy/typed-stdlib.txt @@ -2,3 +2,4 @@ _colorize.py _pyrepl +pathlib From d83531f7211f9876bddb4476bc88858cb98aaca9 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Thu, 24 Apr 2025 00:17:57 +0200 Subject: [PATCH 15/17] pathlib: fix most typing issues --- Lib/pathlib/__init__.py | 9 ++++++++- Lib/pathlib/_os.py | 17 +++++++++++++++++ Lib/pathlib/types.py | 9 ++++++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 12cf9f579cb32d..16887c53a90319 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -17,6 +17,13 @@ from stat import S_ISDIR, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence +# types +if False: + from types import ModuleType + + pwd: ModuleType | None + grp: ModuleType | None + try: import pwd except ImportError: @@ -26,7 +33,7 @@ except ImportError: grp = None -from pathlib._os import ( +from ._os import ( PathInfo, DirEntryInfo, ensure_different_files, ensure_distinct_paths, copyfile2, copyfileobj, magic_open, copy_info, diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py index e3751bbcb62377..a9b64ba5ebe77e 100644 --- a/Lib/pathlib/_os.py +++ b/Lib/pathlib/_os.py @@ -7,6 +7,23 @@ import io import os import sys + +# typing +if False: + from collections.abc import Callable + from types import ModuleType + + type CopyFunc = Callable[[int, int], None] + + fcntl: ModuleType | None + posix: ModuleType | None + _winapi: ModuleType | None + _ficlone: CopyFunc | None + _fcopyfile: CopyFunc | None + _copy_file_range: CopyFunc | None + _sendfile: CopyFunc | None + copyfile2: CopyFunc | None + try: import fcntl except ImportError: diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index e3e8ef38bf034b..d02c0258827777 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -13,10 +13,11 @@ from abc import ABC, abstractmethod from collections.abc import Callable, Iterator, Sequence from glob import _PathGlobber -from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj -from pathlib import PurePath, Path from typing import Protocol, runtime_checkable +from . import PurePath, Path +from ._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj + # typing if False: from typing import Any, BinaryIO, Literal, Self @@ -308,7 +309,9 @@ def walk( follow_symlinks: bool = False, ) -> Iterator[tuple[Self, list[str], list[str]]]: """Walk the directory tree from this directory, similar to os.walk().""" - paths = [self] + dirnames: list[str] + filenames: list[str] + paths: list[Self | tuple[Self, list[str], list[str]]] = [self] while paths: path = paths.pop() if isinstance(path, tuple): From e59613055ddd79b59dd171fc3e93d7e3ea4d2266 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Thu, 24 Apr 2025 00:18:56 +0200 Subject: [PATCH 16/17] prevent missing glob typing to bleed into _pyrepl mypy tests --- Lib/pathlib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 16887c53a90319..4bbd0c933f6ba4 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -12,7 +12,7 @@ import posixpath import sys from errno import * -from glob import _StringGlobber, _no_recurse_symlinks +from glob import _StringGlobber, _no_recurse_symlinks # type: ignore[attr-defined] from itertools import chain from stat import S_ISDIR, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence From 7cd07f12215912cf9157abe37d2381d29f4cab5f Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Thu, 24 Apr 2025 00:28:31 +0200 Subject: [PATCH 17/17] workaround crashing mypy --- Lib/pathlib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 4bbd0c933f6ba4..27f3435d4c37b2 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -53,7 +53,7 @@ class UnsupportedOperation(NotImplementedError): pass -class _PathParents(Sequence): +class _PathParents(Sequence["PurePath"]): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" __slots__ = ('_path', '_drv', '_root', '_tail')