python · ericsnowcurrently · Oct 5, 2021 · Sep 28, 2021 · Sep 28, 2021 · Sep 29, 2021
@@ -10,6 +10,13 @@ extern PyStatus _PyImport_ReInitLock(void);
 #endif
 extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate);

+struct _module_alias {
+    const char *name;                 /* ASCII encoded string */
+    const char *orig;                 /* ASCII encoded string */
+};
+
+extern const struct _module_alias * _PyImport_FrozenAliases;
+
 #ifdef __cplusplus
 }
 #endif

@@ -824,16 +824,39 @@ def module_repr(m):
                       "slated for removal in Python 3.12", DeprecationWarning)
        return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)

+    @classmethod
+    def _setup_module(cls, module):
+        assert not hasattr(module, '__file__'), module.__file__
+        ispkg = hasattr(module, '__path__')
+        assert not ispkg or not module.__path__, module.__path__
+        spec = module.__spec__
+        assert not ispkg or not spec.submodule_search_locations
+
+        if spec.loader_state is None:
+            spec.loader_state = type(sys.implementation)(
+                data=None,
+                origname=None,
+            )
+        elif not hasattr(spec.loader_state, 'data'):
+            spec.loader_state.data = None
+        if not getattr(spec.loader_state, 'origname', None):
+            origname = vars(module).pop('__origname__', None)
+            assert origname, 'see PyImport_ImportFrozenModuleObject()'
+            spec.loader_state.origname = origname
+
    @classmethod
    def find_spec(cls, fullname, path=None, target=None):
        info = _call_with_frames_removed(_imp.find_frozen, fullname)
        if info is None:
            return None
-        data, ispkg = info
+        data, ispkg, origname = info
        spec = spec_from_loader(fullname, cls,
                                origin=cls._ORIGIN,
                                is_package=ispkg)
-        spec.loader_state = data
+        spec.loader_state = type(sys.implementation)(
+            data=data,
+            origname=origname,
+        )
        return spec

    @classmethod
@@ -857,7 +880,7 @@ def exec_module(module):
        spec = module.__spec__
        name = spec.name
        try:
-            data = spec.loader_state
+            data = spec.loader_state.data
        except AttributeError:
            if not _imp.is_frozen(name):
                raise ImportError('{!r} is not a frozen module'.format(name),
@@ -868,7 +891,7 @@ def exec_module(module):
            # Note that if this method is called again (e.g. by
            # importlib.reload()) then _imp.get_frozen_object() will notice
            # no data was provided and will look it up.
-            spec.loader_state = None
+            spec.loader_state.data = None
        code = _call_with_frames_removed(_imp.get_frozen_object, name, data)
        exec(code, module.__dict__)

@@ -1220,6 +1243,8 @@ def _setup(sys_module, _imp_module):
                continue
            spec = _spec_from_module(module, loader)
            _init_module_attrs(spec, module)
+            if loader is FrozenImporter:
+                loader._setup_module(module)

    # Directly load built-in modules needed during bootstrap.
    self_module = sys.modules[__name__]

@@ -9,7 +9,15 @@
 import unittest
 import warnings

-from test.support import import_helper, REPO_ROOT
+from test.support import import_helper, REPO_ROOT, STDLIB_DIR
+
+
+def resolve_stdlib_file(name, ispkg=False):
+    assert name
+    if ispkg:
+        return os.path.join(STDLIB_DIR, *name.split('.'), '__init__.py')
+    else:
+        return os.path.join(STDLIB_DIR, *name.split('.')) + '.py'


 class FindSpecTests(abc.FinderTests):
@@ -32,16 +40,30 @@ def check_basic(self, spec, name, ispkg=False):
            self.assertIsNone(spec.submodule_search_locations)
        self.assertIsNotNone(spec.loader_state)

-    def check_data(self, spec):
+    def check_loader_state(self, spec, origname=None, filename=None):
+        if not filename:
+            if not origname:
+                origname = spec.name
+
+        actual = dict(vars(spec.loader_state))
+
+        # Check the code object used to import the frozen module.
+        # We can't compare the marshaled data directly because
+        # marshal.dumps() would mark "expected" (below) as a ref,
+        # which slightly changes the output.
+        # (See https://bugs.python.org/issue34093.)
+        data = actual.pop('data')
        with import_helper.frozen_modules():
            expected = _imp.get_frozen_object(spec.name)
-        data = spec.loader_state
-        # We can't compare the marshaled data directly because
-        # marshal.dumps() would mark "expected" as a ref, which slightly
-        # changes the output.  (See https://bugs.python.org/issue34093.)
        code = marshal.loads(data)
        self.assertEqual(code, expected)

+        # Check the rest of spec.loader_state.
+        expected = dict(
+            origname=origname,
+        )
+        self.assertDictEqual(actual, expected)
+
    def check_search_locations(self, spec):
        # Frozen packages do not have any path entries.
        # (See https://bugs.python.org/issue21736.)
@@ -58,7 +80,7 @@ def test_module(self):
            with self.subTest(f'{name} -> {name}'):
                spec = self.find(name)
                self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec)
        modules = {
            '__hello_alias__': '__hello__',
            '_frozen_importlib': 'importlib._bootstrap',
@@ -67,46 +89,50 @@ def test_module(self):
            with self.subTest(f'{name} -> {origname}'):
                spec = self.find(name)
                self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname)
        modules = [
            '__phello__.__init__',
            '__phello__.ham.__init__',
        ]
        for name in modules:
-            origname = name.rpartition('.')[0]
+            origname = '<' + name.rpartition('.')[0]
+            filename = resolve_stdlib_file(name)
            with self.subTest(f'{name} -> {origname}'):
                spec = self.find(name)
                self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)
        modules = {
            '__hello_only__': ('Tools', 'freeze', 'flag.py'),
        }
        for name, path in modules.items():
+            origname = None
            filename = os.path.join(REPO_ROOT, *path)
            with self.subTest(f'{name} -> {filename}'):
                spec = self.find(name)
                self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)

    def test_package(self):
        packages = [
            '__phello__',
            '__phello__.ham',
        ]
        for name in packages:
+            filename = resolve_stdlib_file(name, ispkg=True)
            with self.subTest(f'{name} -> {name}'):
                spec = self.find(name)
                self.check_basic(spec, name, ispkg=True)
-                self.check_data(spec)
+                self.check_loader_state(spec, name, filename)
                self.check_search_locations(spec)
        packages = {
            '__phello_alias__': '__hello__',
        }
        for name, origname in packages.items():
+            filename = resolve_stdlib_file(origname, ispkg=False)
            with self.subTest(f'{name} -> {origname}'):
                spec = self.find(name)
                self.check_basic(spec, name, ispkg=True)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)
                self.check_search_locations(spec)

    # These are covered by test_module() and test_package().

@@ -32,17 +32,19 @@ def fresh(name, *, oldapi=False):

 class ExecModuleTests(abc.LoaderTests):

-    def exec_module(self, name):
+    def exec_module(self, name, origname=None):
        with import_helper.frozen_modules():
            is_package = self.machinery.FrozenImporter.is_package(name)
            code = _imp.get_frozen_object(name)
-        data = marshal.dumps(code)
        spec = self.machinery.ModuleSpec(
            name,
            self.machinery.FrozenImporter,
            origin='frozen',
            is_package=is_package,
-            loader_state=data,
+            loader_state=types.SimpleNamespace(
+                data=marshal.dumps(code),
+                origname=origname or name,
+            ),
        )
        module = types.ModuleType(name)
        module.__spec__ = spec
@@ -66,7 +68,8 @@ def test_module(self):
            self.assertEqual(getattr(module, attr), value)
        self.assertEqual(output, 'Hello world!\n')
        self.assertTrue(hasattr(module, '__spec__'))
-        self.assertIsNone(module.__spec__.loader_state)
+        self.assertIsNone(module.__spec__.loader_state.data)
+        self.assertEqual(module.__spec__.loader_state.origname, name)

    def test_package(self):
        name = '__phello__'
@@ -79,7 +82,8 @@ def test_package(self):
                                 name=name, attr=attr, given=attr_value,
                                 expected=value))
        self.assertEqual(output, 'Hello world!\n')
-        self.assertIsNone(module.__spec__.loader_state)
+        self.assertIsNone(module.__spec__.loader_state.data)
+        self.assertEqual(module.__spec__.loader_state.origname, name)

    def test_lacking_parent(self):
        name = '__phello__.spam'

diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-06-54.bpo-45020.Cj5VQN.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-06-54.bpo-45020.Cj5VQN.rst
@@ -0,0 +1,5 @@
+For frozen stdlib modules, record the original module name as
+``module.__spec__.loader_state.origname``.  If the value is different than
+``module.__spec__.name`` then the module was defined as an alias in
+Tools/scripts/freeze_modules.py.  If it is ``None`` then the module comes
+from a source file outside the stdlib.
@@ -9,6 +9,7 @@

 #include <Python.h>
 #include <marshal.h>
+#include <pycore_import.h>

 #include <stdio.h>
 #include <sys/types.h>
@@ -24,8 +25,12 @@
 static const struct _frozen _PyImport_FrozenModules[] = {
    {0, 0, 0} /* sentinel */
 };
+static const struct _module_alias aliases[] = {
+    {0, 0} /* sentinel */
+};

 const struct _frozen *PyImport_FrozenModules;
+const struct _module_alias *_PyImport_FrozenAliases;

 static const char header[] =
    "/* Auto-generated by Programs/_freeze_module.c */";
@@ -183,6 +188,7 @@ main(int argc, char *argv[])
    const char *name, *inpath, *outpath;

    PyImport_FrozenModules = _PyImport_FrozenModules;
+    _PyImport_FrozenAliases = aliases;

    if (argc != 4) {
        fprintf(stderr, "need to specify the name, input and output paths\n");

diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h
@@ -36,6 +36,7 @@
   and __phello__.spam.  Loading any will print some famous words... */

 #include "Python.h"
+#include "pycore_import.h"

 /* Includes for frozen modules: */
 #include "frozen_modules/importlib._bootstrap.h"
@@ -102,9 +103,24 @@ static const struct _frozen _PyImport_FrozenModules[] = {
    {"__phello__.spam", _Py_M____phello___spam,
        (int)sizeof(_Py_M____phello___spam)},
    {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)},
-    {0, 0, 0} /* sentinel */
+    {0, 0, 0} /* modules sentinel */
 };

+static const struct _module_alias aliases[] = {
+    {"_frozen_importlib", "importlib._bootstrap"},
+    {"_frozen_importlib_external", "importlib._bootstrap_external"},
+    {"os.path", "posixpath"},
+    {"__hello_alias__", "__hello__"},
+    {"__phello_alias__", "__hello__"},
+    {"__phello_alias__.spam", "__hello__"},
+    {"__phello__.__init__", "<__phello__"},
+    {"__phello__.ham.__init__", "<__phello__.ham"},
+    {"__hello_only__", NULL},
+    {0, 0} /* aliases sentinel */
+};
+const struct _module_alias *_PyImport_FrozenAliases = aliases;
+
+
 /* Embedding apps may change this pointer to point to their favorite
   collection of frozen modules: */