Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit aaab6d2

Browse filesBrowse files
jaracojaraco
committed
gh-123987: Fix issues in importlib.resources.
Also addresses gh-123085.
1 parent 4ed7d1d commit aaab6d2
Copy full SHA for aaab6d2

File tree

8 files changed

+134
-21
lines changed
Filter options

8 files changed

+134
-21
lines changed

‎Lib/importlib/resources/__init__.py

Copy file name to clipboardExpand all lines: Lib/importlib/resources/__init__.py
+8-1Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
1-
"""Read resources contained within a package."""
1+
"""
2+
Read resources contained within a package.
3+
4+
This codebase is shared between importlib.resources in the stdlib
5+
and importlib_resources in PyPI. See
6+
https://github.com/python/importlib_metadata/wiki/Development-Methodology
7+
for more detail.
8+
"""
29

310
from ._common import (
411
as_file,

‎Lib/importlib/resources/_common.py

Copy file name to clipboardExpand all lines: Lib/importlib/resources/_common.py
+5-4Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,10 @@ def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]:
6666
# zipimport.zipimporter does not support weak references, resulting in a
6767
# TypeError. That seems terrible.
6868
spec = package.__spec__
69-
reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore
69+
reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore[union-attr]
7070
if reader is None:
7171
return None
72-
return reader(spec.name) # type: ignore
72+
return reader(spec.name) # type: ignore[union-attr]
7373

7474

7575
@functools.singledispatch
@@ -93,12 +93,13 @@ def _infer_caller():
9393
"""
9494

9595
def is_this_file(frame_info):
96-
return frame_info.filename == __file__
96+
return frame_info.filename == stack[0].filename
9797

9898
def is_wrapper(frame_info):
9999
return frame_info.function == 'wrapper'
100100

101-
not_this_file = itertools.filterfalse(is_this_file, inspect.stack())
101+
stack = inspect.stack()
102+
not_this_file = itertools.filterfalse(is_this_file, stack)
102103
# also exclude 'wrapper' due to singledispatch in the call stack
103104
callers = itertools.filterfalse(is_wrapper, not_this_file)
104105
return next(callers).frame

‎Lib/importlib/resources/readers.py

Copy file name to clipboardExpand all lines: Lib/importlib/resources/readers.py
+13-6Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from __future__ import annotations
2+
13
import collections
24
import contextlib
35
import itertools
@@ -6,6 +8,7 @@
68
import re
79
import warnings
810
import zipfile
11+
from collections.abc import Iterator
912

1013
from . import abc
1114

@@ -135,27 +138,31 @@ class NamespaceReader(abc.TraversableResources):
135138
def __init__(self, namespace_path):
136139
if 'NamespacePath' not in str(namespace_path):
137140
raise ValueError('Invalid path')
138-
self.path = MultiplexedPath(*map(self._resolve, namespace_path))
141+
self.path = MultiplexedPath(*filter(bool, map(self._resolve, namespace_path)))
139142

140143
@classmethod
141-
def _resolve(cls, path_str) -> abc.Traversable:
144+
def _resolve(cls, path_str) -> abc.Traversable | None:
142145
r"""
143146
Given an item from a namespace path, resolve it to a Traversable.
144147
145148
path_str might be a directory on the filesystem or a path to a
146149
zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or
147150
``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``.
151+
152+
path_str might also be a sentinel used by editable packages to
153+
trigger other behaviors (see python/importlib_resources#311).
154+
In that case, return None.
148155
"""
149-
(dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir())
150-
return dir
156+
dirs = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir())
157+
return next(dirs, None)
151158

152159
@classmethod
153-
def _candidate_paths(cls, path_str):
160+
def _candidate_paths(cls, path_str: str) -> Iterator[abc.Traversable]:
154161
yield pathlib.Path(path_str)
155162
yield from cls._resolve_zip_path(path_str)
156163

157164
@staticmethod
158-
def _resolve_zip_path(path_str):
165+
def _resolve_zip_path(path_str: str):
159166
for match in reversed(list(re.finditer(r'[\\/]', path_str))):
160167
with contextlib.suppress(
161168
FileNotFoundError,

‎Lib/importlib/resources/simple.py

Copy file name to clipboardExpand all lines: Lib/importlib/resources/simple.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ class ResourceHandle(Traversable):
7777

7878
def __init__(self, parent: ResourceContainer, name: str):
7979
self.parent = parent
80-
self.name = name # type: ignore
80+
self.name = name # type: ignore[misc]
8181

8282
def is_file(self):
8383
return True

‎Lib/test/test_importlib/resources/_path.py

Copy file name to clipboardExpand all lines: Lib/test/test_importlib/resources/_path.py
+44-6Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,44 @@
22
import functools
33

44
from typing import Dict, Union
5+
from typing import runtime_checkable
6+
from typing import Protocol
57

68

79
####
8-
# from jaraco.path 3.4.1
10+
# from jaraco.path 3.7.1
911

10-
FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore
1112

13+
class Symlink(str):
14+
"""
15+
A string indicating the target of a symlink.
16+
"""
17+
18+
19+
FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']]
20+
21+
22+
@runtime_checkable
23+
class TreeMaker(Protocol):
24+
def __truediv__(self, *args, **kwargs): ... # pragma: no cover
25+
26+
def mkdir(self, **kwargs): ... # pragma: no cover
27+
28+
def write_text(self, content, **kwargs): ... # pragma: no cover
29+
30+
def write_bytes(self, content): ... # pragma: no cover
1231

13-
def build(spec: FilesSpec, prefix=pathlib.Path()):
32+
def symlink_to(self, target): ... # pragma: no cover
33+
34+
35+
def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker:
36+
return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value]
37+
38+
39+
def build(
40+
spec: FilesSpec,
41+
prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment]
42+
):
1443
"""
1544
Build a set of files/directories, as described by the spec.
1645
@@ -25,21 +54,25 @@ def build(spec: FilesSpec, prefix=pathlib.Path()):
2554
... "__init__.py": "",
2655
... },
2756
... "baz.py": "# Some code",
28-
... }
57+
... "bar.py": Symlink("baz.py"),
58+
... },
59+
... "bing": Symlink("foo"),
2960
... }
3061
>>> target = getfixture('tmp_path')
3162
>>> build(spec, target)
3263
>>> target.joinpath('foo/baz.py').read_text(encoding='utf-8')
3364
'# Some code'
65+
>>> target.joinpath('bing/bar.py').read_text(encoding='utf-8')
66+
'# Some code'
3467
"""
3568
for name, contents in spec.items():
36-
create(contents, pathlib.Path(prefix) / name)
69+
create(contents, _ensure_tree_maker(prefix) / name)
3770

3871

3972
@functools.singledispatch
4073
def create(content: Union[str, bytes, FilesSpec], path):
4174
path.mkdir(exist_ok=True)
42-
build(content, prefix=path) # type: ignore
75+
build(content, prefix=path) # type: ignore[arg-type]
4376

4477

4578
@create.register
@@ -52,5 +85,10 @@ def _(content: str, path):
5285
path.write_text(content, encoding='utf-8')
5386

5487

88+
@create.register
89+
def _(content: Symlink, path):
90+
path.symlink_to(content)
91+
92+
5593
# end from jaraco.path
5694
####

‎Lib/test/test_importlib/resources/test_files.py

Copy file name to clipboardExpand all lines: Lib/test/test_importlib/resources/test_files.py
+58-3Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
import os
2+
import pathlib
3+
import py_compile
4+
import shutil
15
import textwrap
26
import unittest
37
import warnings
@@ -7,6 +11,7 @@
711
from importlib import resources
812
from importlib.resources.abc import Traversable
913
from . import util
14+
from test.support import os_helper, import_helper
1015

1116

1217
@contextlib.contextmanager
@@ -55,6 +60,26 @@ class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
5560
class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase):
5661
MODULE = 'namespacedata01'
5762

63+
def test_non_paths_in_dunder_path(self):
64+
"""
65+
Non-path items in a namespace package's ``__path__`` are ignored.
66+
67+
As reported in python/importlib_resources#311, some tools
68+
like Setuptools, when creating editable packages, will inject
69+
non-paths into a namespace package's ``__path__``, a
70+
sentinel like
71+
``__editable__.sample_namespace-1.0.finder.__path_hook__``
72+
to cause the ``PathEntryFinder`` to be called when searching
73+
for packages. In that case, resources should still be loadable.
74+
"""
75+
import namespacedata01
76+
77+
namespacedata01.__path__.append(
78+
'__editable__.sample_namespace-1.0.finder.__path_hook__'
79+
)
80+
81+
resources.files(namespacedata01)
82+
5883

5984
class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
6085
ZIP_MODULE = 'namespacedata01'
@@ -81,7 +106,7 @@ def test_module_resources(self):
81106
"""
82107
A module can have resources found adjacent to the module.
83108
"""
84-
import mod
109+
import mod # type: ignore[import-not-found]
85110

86111
actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8')
87112
assert actual == self.spec['res.txt']
@@ -97,8 +122,8 @@ class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.Test
97122

98123
class ImplicitContextFiles:
99124
set_val = textwrap.dedent(
100-
"""
101-
import importlib.resources as res
125+
f"""
126+
import {resources.__name__} as res
102127
val = res.files().joinpath('res.txt').read_text(encoding='utf-8')
103128
"""
104129
)
@@ -108,6 +133,10 @@ class ImplicitContextFiles:
108133
'submod.py': set_val,
109134
'res.txt': 'resources are the best',
110135
},
136+
'frozenpkg': {
137+
'__init__.py': set_val.replace(resources.__name__, 'c_resources'),
138+
'res.txt': 'resources are the best',
139+
},
111140
}
112141

113142
def test_implicit_files_package(self):
@@ -122,6 +151,32 @@ def test_implicit_files_submodule(self):
122151
"""
123152
assert importlib.import_module('somepkg.submod').val == 'resources are the best'
124153

154+
def _compile_importlib(self):
155+
"""
156+
Make a compiled-only copy of the importlib resources package.
157+
"""
158+
bin_site = self.fixtures.enter_context(os_helper.temp_dir())
159+
c_resources = pathlib.Path(bin_site, 'c_resources')
160+
sources = pathlib.Path(resources.__file__).parent
161+
shutil.copytree(sources, c_resources, ignore=lambda *_: ['__pycache__'])
162+
163+
for dirpath, _, filenames in os.walk(c_resources):
164+
for filename in filenames:
165+
source_path = pathlib.Path(dirpath) / filename
166+
cfile = source_path.with_suffix('.pyc')
167+
py_compile.compile(source_path, cfile)
168+
pathlib.Path.unlink(source_path)
169+
self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site))
170+
171+
def test_implicit_files_with_compiled_importlib(self):
172+
"""
173+
Caller detection works for compiled-only resources module.
174+
175+
python/cpython#123085
176+
"""
177+
self._compile_importlib()
178+
assert importlib.import_module('frozenpkg').val == 'resources are the best'
179+
125180

126181
class ImplicitContextFilesDiskTests(
127182
DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase

‎Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst

Copy file name to clipboard
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fixed issue in NamespaceReader where a non-path item in a namespace path,
2+
such as a sentinel added by an editable installer, would break resource
3+
loading.

‎Misc/NEWS.d/next/Library/2024-09-12-14-26-05.gh-issue-123085.Naekav.rst

Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fixed issue in bare ``importlib.resources.files()`` when the library is
2+
available as compiled only (no source).

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.