From 1901e8047dd5b05270cb64e1b715ad1ea1fe3bdf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 4 Sep 2025 00:29:46 +0100 Subject: [PATCH 01/12] Install ``build-details.json`` on Windows --- Lib/test/test_build_details.py | 10 ++++++---- .../2025-09-04-00-32-28.gh-issue-138489.Adc9Lx.rst | 3 +++ PCbuild/python.vcxproj | 7 +++++++ PCbuild/pythoncore.vcxproj | 3 +++ Tools/build/generate-build-details.py | 5 ++++- 5 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-04-00-32-28.gh-issue-138489.Adc9Lx.rst diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index bc04963f5ad613..621b726cecce4b 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -90,7 +90,6 @@ def test_implementation(self): ) -@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') @unittest.skipIf(is_wasm32, 'Feature not available on WebAssembly builds') class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): """Test CPython's install details file implementation.""" @@ -98,9 +97,12 @@ class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): @property def location(self): if sysconfig.is_python_build(): - projectdir = sysconfig.get_config_var('projectbase') - with open(os.path.join(projectdir, 'pybuilddir.txt')) as f: - dirname = os.path.join(projectdir, f.read()) + if sys.platform == 'win32': + dirname = sysconfig.get_config_var('BINDIR') + else: + projectdir = sysconfig.get_config_var('projectbase') + with open(os.path.join(projectdir, 'pybuilddir.txt')) as f: + dirname = os.path.join(projectdir, f.read()) else: dirname = sysconfig.get_path('stdlib') return os.path.join(dirname, 'build-details.json') diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-04-00-32-28.gh-issue-138489.Adc9Lx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-04-00-32-28.gh-issue-138489.Adc9Lx.rst new file mode 100644 index 00000000000000..38de0930767dc5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-04-00-32-28.gh-issue-138489.Adc9Lx.rst @@ -0,0 +1,3 @@ +The :pep:`739` :file:`build-details.json` file is now generated and +installed on Windows. +Patch by Adam Turner. diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 70dabaa3c8bc02..5cf4f99c8ae9d1 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -126,6 +126,13 @@ + + + + + ucrtbase diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 517103acea8d8e..15477701bb66bc 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -735,4 +735,7 @@ + + + diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 8cd23e2f54f529..ad3163a6756fc2 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -61,7 +61,10 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: if '_multiarch' in data['implementation']: data['implementation']['_multiarch'] = sysconfig.get_config_var('MULTIARCH') - data['abi']['flags'] = list(sys.abiflags) + if sys.platform != 'win32': + data['abi']['flags'] = list(sys.abiflags) + else: + data['abi']['flags'] = [] data['suffixes']['source'] = importlib.machinery.SOURCE_SUFFIXES data['suffixes']['bytecode'] = importlib.machinery.BYTECODE_SUFFIXES From dbe1cbdcc1b2936543f5e3a1a403cd4068c59094 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 Sep 2025 01:21:01 +0100 Subject: [PATCH 02/12] Update python.vcxproj --- PCbuild/python.vcxproj | 1 - 1 file changed, 1 deletion(-) diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 5cf4f99c8ae9d1..976c0d52a1d0b1 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -128,7 +128,6 @@ - From c80344ca8b335838b4438cb91c38fcbfb9e53d1e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 4 Sep 2025 19:41:08 +0100 Subject: [PATCH 03/12] Fix ```--relative-paths`` * KeyError is not raised for defaultdict * Fix relative paths on different drives on Windows --- Tools/build/generate-build-details.py | 34 ++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index ad3163a6756fc2..0280fdb5296ccf 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -61,7 +61,7 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: if '_multiarch' in data['implementation']: data['implementation']['_multiarch'] = sysconfig.get_config_var('MULTIARCH') - if sys.platform != 'win32': + if os.name != 'nt': data['abi']['flags'] = list(sys.abiflags) else: data['abi']['flags'] = [] @@ -107,7 +107,7 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: data['abi']['extension_suffix'] = sysconfig.get_config_var('EXT_SUFFIX') # EXTENSION_SUFFIXES has been constant for a long time, and currently we - # don't have a better information source to find the stable ABI suffix. + # don't have a better information source to find the stable ABI suffix. for suffix in importlib.machinery.EXTENSION_SUFFIXES: if suffix.startswith('.abi'): data['abi']['stable_abi_suffix'] = suffix @@ -136,33 +136,51 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: def make_paths_relative(data: dict[str, Any], config_path: str | None = None) -> None: # Make base_prefix relative to the config_path directory if config_path: - data['base_prefix'] = os.path.relpath(data['base_prefix'], os.path.dirname(config_path)) + data['base_prefix'] = relative_path(data['base_prefix'], + os.path.dirname(config_path)) + base_prefix = data['base_prefix'] + # Update path values to make them relative to base_prefix - PATH_KEYS = [ + PATH_KEYS = ( 'base_interpreter', 'libpython.dynamic', 'libpython.dynamic_stableabi', 'libpython.static', 'c_api.headers', 'c_api.pkgconfig_path', - ] + ) for entry in PATH_KEYS: - parent, _, child = entry.rpartition('.') + *parents, child = entry.split('.') # Get the key container object try: container = data - for part in parent.split('.'): + for part in parents: container = container[part] + if child not in container: + raise KeyError current_path = container[child] except KeyError: continue # Get the relative path - new_path = os.path.relpath(current_path, data['base_prefix']) + new_path = relative_path(current_path, base_prefix) # Join '.' so that the path is formated as './path' instead of 'path' new_path = os.path.join('.', new_path) container[child] = new_path +def relative_path(path: str, base: str) -> str: + if os.name != 'nt': + return os.path.relpath(path, base) + + # There are no relative paths between drives on Windows. + path_drv, path_root, _ = os.path.splitroot(path) + base_drv, base_root, _ = os.path.splitroot(base) + if path_drv.lower() == base_drv.lower() and path_root == base_root: + return os.path.relpath(path, base) + + return path + + def main() -> None: parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('location') From ee39ea7171ee0bb38f706b9512c3f21bbcdb1eec Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 4 Sep 2025 19:43:42 +0100 Subject: [PATCH 04/12] Specify encoding when writing file --- Tools/build/generate-build-details.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 0280fdb5296ccf..0216dfc41d0acc 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -207,8 +207,9 @@ def main() -> None: make_paths_relative(data, args.config_file_path) json_output = json.dumps(data, indent=2) - with open(args.location, 'w') as f: - print(json_output, file=f) + with open(args.location, 'w', encoding='utf-8') as f: + f.write(json_output) + f.write('\n') if __name__ == '__main__': From f5f0548e369443f972128c6de1bbcc48d3d6f7ff Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 4 Sep 2025 20:42:13 +0100 Subject: [PATCH 05/12] Add test for ``--relative-paths`` --- Lib/test/test_build_details.py | 104 +++++++++++++++++++++++--- Tools/build/generate-build-details.py | 2 +- 2 files changed, 95 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 621b726cecce4b..afa91b950ce725 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -1,12 +1,34 @@ +import importlib import json import os +import os.path import sys import sysconfig import string import unittest +from pathlib import Path from test.support import is_android, is_apple_mobile, is_wasm32 +BASE_PATH = Path( + __file__, # Lib/test/test_build_details.py + '..', # Lib/test + '..', # Lib + '..', # +).resolve() +MODULE_PATH = BASE_PATH / 'Tools' / 'build' / 'generate-build-details.py' + +try: + # Import "generate-build-details.py" as "generate_build_details" + spec = importlib.util.spec_from_file_location( + "generate_build_details", MODULE_PATH + ) + generate_build_details = importlib.util.module_from_spec(spec) + sys.modules["generate_build_details"] = generate_build_details + spec.loader.exec_module(generate_build_details) +except ImportError: + generate_build_details = None + class FormatTestsBase: @property @@ -31,16 +53,15 @@ def key(self, name): value = value[part] return value - def test_parse(self): - self.data - def test_top_level_container(self): self.assertIsInstance(self.data, dict) for key, value in self.data.items(): with self.subTest(key=key): - if key in ('schema_version', 'base_prefix', 'base_interpreter', 'platform'): + if key in ('schema_version', 'base_prefix', 'base_interpreter', + 'platform'): self.assertIsInstance(value, str) - elif key in ('language', 'implementation', 'abi', 'suffixes', 'libpython', 'c_api', 'arbitrary_data'): + elif key in ('language', 'implementation', 'abi', 'suffixes', + 'libpython', 'c_api', 'arbitrary_data'): self.assertIsInstance(value, dict) def test_base_prefix(self): @@ -71,15 +92,20 @@ def test_language_version_info(self): self.assertEqual(len(value), sys.version_info.n_fields) for part_name, part_value in value.items(): with self.subTest(part=part_name): - self.assertEqual(part_value, getattr(sys.version_info, part_name)) + sys_version_value = getattr(sys.version_info, part_name) + self.assertEqual(part_value, sys_version_value) def test_implementation(self): + impl_ver = sys.implementation.version for key, value in self.key('implementation').items(): with self.subTest(part=key): if key == 'version': - self.assertEqual(len(value), len(sys.implementation.version)) + self.assertEqual(len(value), len(impl_ver)) for part_name, part_value in value.items(): - self.assertEqual(getattr(sys.implementation.version, part_name), part_value) + assert not isinstance(sys.implementation.version, dict) + getattr(sys.implementation.version, part_name) + sys_implementation_value = getattr(impl_ver, part_name) + self.assertEqual(sys_implementation_value, part_value) else: self.assertEqual(getattr(sys.implementation, key), value) @@ -101,7 +127,8 @@ def location(self): dirname = sysconfig.get_config_var('BINDIR') else: projectdir = sysconfig.get_config_var('projectbase') - with open(os.path.join(projectdir, 'pybuilddir.txt')) as f: + pybuilddir = os.path.join(projectdir, 'pybuilddir.txt') + with open(pybuilddir, encoding='utf-8') as f: dirname = os.path.join(projectdir, f.read()) else: dirname = sysconfig.get_path('stdlib') @@ -109,7 +136,7 @@ def location(self): @property def contents(self): - with open(self.location, 'r') as f: + with open(self.location, 'r', encoding='utf-8') as f: return f.read() @needs_installed_python @@ -149,5 +176,62 @@ def test_c_api(self): self.assertTrue(os.path.exists(os.path.join(value['pkgconfig_path'], f'python-{version}.pc'))) +@unittest.skipIf( + generate_build_details is None, + "Failed to import generate-build-details" +) +class BuildDetailsRelativePathsTests(unittest.TestCase): + @property + def build_details_absolute_paths(self): + data = generate_build_details.generate_data(schema_version='1.0') + return json.loads(json.dumps(data)) + + @property + def build_details_relative_paths(self): + data = self.build_details_absolute_paths + generate_build_details.make_paths_relative(data, config_path=None) + return data + + def test_round_trip(self): + data_abs_path = self.build_details_absolute_paths + data_rel_path = self.build_details_relative_paths + + self.assertEqual(data_abs_path['base_prefix'], + data_rel_path['base_prefix']) + + base_prefix = data_abs_path['base_prefix'] + + top_level_keys = ('base_interpreter',) + for key in top_level_keys: + self.assertEqual(key in data_abs_path, key in data_rel_path) + if key not in data_abs_path: + continue + + abs_rel_path = os.path.join(base_prefix, data_rel_path[key]) + abs_rel_path = os.path.normpath(abs_rel_path) + self.assertEqual(data_abs_path[key], abs_rel_path) + + second_level_keys = ( + ('libpython', 'dynamic'), + ('libpython', 'dynamic_stableabi'), + ('libpython', 'static'), + ('c_api', 'headers'), + ('c_api', 'pkgconfig_path'), + + ) + for part, key in second_level_keys: + self.assertEqual(part in data_abs_path, part in data_rel_path) + if part not in data_abs_path: + continue + self.assertEqual(key in data_abs_path[part], + key in data_rel_path[part]) + if key not in data_abs_path[part]: + continue + + abs_rel_path = os.path.join(base_prefix, data_rel_path[part][key]) + abs_rel_path = os.path.normpath(abs_rel_path) + self.assertEqual(data_abs_path[part][key], abs_rel_path) + + if __name__ == '__main__': unittest.main() diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 0216dfc41d0acc..64b1168b495509 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -55,7 +55,7 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: data['language']['version'] = sysconfig.get_python_version() data['language']['version_info'] = version_info_to_dict(sys.version_info) - data['implementation'] = vars(sys.implementation) + data['implementation'] = vars(sys.implementation).copy() data['implementation']['version'] = version_info_to_dict(sys.implementation.version) # Fix cross-compilation if '_multiarch' in data['implementation']: From fdba9f0547693e390ed1250fe758725e3da1b129 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 4 Sep 2025 20:46:22 +0100 Subject: [PATCH 06/12] Lint (splitroot requires 3.12) --- Tools/build/generate-build-details.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 64b1168b495509..928b93d460a47a 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -173,9 +173,9 @@ def relative_path(path: str, base: str) -> str: return os.path.relpath(path, base) # There are no relative paths between drives on Windows. - path_drv, path_root, _ = os.path.splitroot(path) - base_drv, base_root, _ = os.path.splitroot(base) - if path_drv.lower() == base_drv.lower() and path_root == base_root: + path_drv, _ = os.path.splitdrive(path) + base_drv, _ = os.path.splitdrive(base) + if path_drv.lower() == base_drv.lower(): return os.path.relpath(path, base) return path From 6a2cc71029b88ba8c952f0a27bb680dff24d1c62 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 4 Sep 2025 20:55:36 +0100 Subject: [PATCH 07/12] Revert relative-path changes. Split to https://github.com/python/cpython/pull/138510 --- Lib/test/test_build_details.py | 99 +++------------------------ Tools/build/generate-build-details.py | 30 ++------ 2 files changed, 14 insertions(+), 115 deletions(-) diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index afa91b950ce725..c4ca947f924c9f 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -1,34 +1,12 @@ -import importlib import json import os -import os.path import sys import sysconfig import string import unittest -from pathlib import Path from test.support import is_android, is_apple_mobile, is_wasm32 -BASE_PATH = Path( - __file__, # Lib/test/test_build_details.py - '..', # Lib/test - '..', # Lib - '..', # -).resolve() -MODULE_PATH = BASE_PATH / 'Tools' / 'build' / 'generate-build-details.py' - -try: - # Import "generate-build-details.py" as "generate_build_details" - spec = importlib.util.spec_from_file_location( - "generate_build_details", MODULE_PATH - ) - generate_build_details = importlib.util.module_from_spec(spec) - sys.modules["generate_build_details"] = generate_build_details - spec.loader.exec_module(generate_build_details) -except ImportError: - generate_build_details = None - class FormatTestsBase: @property @@ -53,15 +31,16 @@ def key(self, name): value = value[part] return value + def test_parse(self): + self.data + def test_top_level_container(self): self.assertIsInstance(self.data, dict) for key, value in self.data.items(): with self.subTest(key=key): - if key in ('schema_version', 'base_prefix', 'base_interpreter', - 'platform'): + if key in ('schema_version', 'base_prefix', 'base_interpreter', 'platform'): self.assertIsInstance(value, str) - elif key in ('language', 'implementation', 'abi', 'suffixes', - 'libpython', 'c_api', 'arbitrary_data'): + elif key in ('language', 'implementation', 'abi', 'suffixes', 'libpython', 'c_api', 'arbitrary_data'): self.assertIsInstance(value, dict) def test_base_prefix(self): @@ -92,20 +71,15 @@ def test_language_version_info(self): self.assertEqual(len(value), sys.version_info.n_fields) for part_name, part_value in value.items(): with self.subTest(part=part_name): - sys_version_value = getattr(sys.version_info, part_name) - self.assertEqual(part_value, sys_version_value) + self.assertEqual(part_value, getattr(sys.version_info, part_name)) def test_implementation(self): - impl_ver = sys.implementation.version for key, value in self.key('implementation').items(): with self.subTest(part=key): if key == 'version': - self.assertEqual(len(value), len(impl_ver)) + self.assertEqual(len(value), len(sys.implementation.version)) for part_name, part_value in value.items(): - assert not isinstance(sys.implementation.version, dict) - getattr(sys.implementation.version, part_name) - sys_implementation_value = getattr(impl_ver, part_name) - self.assertEqual(sys_implementation_value, part_value) + self.assertEqual(getattr(sys.implementation.version, part_name), part_value) else: self.assertEqual(getattr(sys.implementation, key), value) @@ -176,62 +150,5 @@ def test_c_api(self): self.assertTrue(os.path.exists(os.path.join(value['pkgconfig_path'], f'python-{version}.pc'))) -@unittest.skipIf( - generate_build_details is None, - "Failed to import generate-build-details" -) -class BuildDetailsRelativePathsTests(unittest.TestCase): - @property - def build_details_absolute_paths(self): - data = generate_build_details.generate_data(schema_version='1.0') - return json.loads(json.dumps(data)) - - @property - def build_details_relative_paths(self): - data = self.build_details_absolute_paths - generate_build_details.make_paths_relative(data, config_path=None) - return data - - def test_round_trip(self): - data_abs_path = self.build_details_absolute_paths - data_rel_path = self.build_details_relative_paths - - self.assertEqual(data_abs_path['base_prefix'], - data_rel_path['base_prefix']) - - base_prefix = data_abs_path['base_prefix'] - - top_level_keys = ('base_interpreter',) - for key in top_level_keys: - self.assertEqual(key in data_abs_path, key in data_rel_path) - if key not in data_abs_path: - continue - - abs_rel_path = os.path.join(base_prefix, data_rel_path[key]) - abs_rel_path = os.path.normpath(abs_rel_path) - self.assertEqual(data_abs_path[key], abs_rel_path) - - second_level_keys = ( - ('libpython', 'dynamic'), - ('libpython', 'dynamic_stableabi'), - ('libpython', 'static'), - ('c_api', 'headers'), - ('c_api', 'pkgconfig_path'), - - ) - for part, key in second_level_keys: - self.assertEqual(part in data_abs_path, part in data_rel_path) - if part not in data_abs_path: - continue - self.assertEqual(key in data_abs_path[part], - key in data_rel_path[part]) - if key not in data_abs_path[part]: - continue - - abs_rel_path = os.path.join(base_prefix, data_rel_path[part][key]) - abs_rel_path = os.path.normpath(abs_rel_path) - self.assertEqual(data_abs_path[part][key], abs_rel_path) - - if __name__ == '__main__': unittest.main() diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 928b93d460a47a..52420c9c3ff62f 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -136,51 +136,33 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: def make_paths_relative(data: dict[str, Any], config_path: str | None = None) -> None: # Make base_prefix relative to the config_path directory if config_path: - data['base_prefix'] = relative_path(data['base_prefix'], - os.path.dirname(config_path)) - base_prefix = data['base_prefix'] - + data['base_prefix'] = os.path.relpath(data['base_prefix'], os.path.dirname(config_path)) # Update path values to make them relative to base_prefix - PATH_KEYS = ( + PATH_KEYS = [ 'base_interpreter', 'libpython.dynamic', 'libpython.dynamic_stableabi', 'libpython.static', 'c_api.headers', 'c_api.pkgconfig_path', - ) + ] for entry in PATH_KEYS: - *parents, child = entry.split('.') + parent, _, child = entry.rpartition('.') # Get the key container object try: container = data - for part in parents: + for part in parent.split('.'): container = container[part] - if child not in container: - raise KeyError current_path = container[child] except KeyError: continue # Get the relative path - new_path = relative_path(current_path, base_prefix) + new_path = os.path.relpath(current_path, data['base_prefix']) # Join '.' so that the path is formated as './path' instead of 'path' new_path = os.path.join('.', new_path) container[child] = new_path -def relative_path(path: str, base: str) -> str: - if os.name != 'nt': - return os.path.relpath(path, base) - - # There are no relative paths between drives on Windows. - path_drv, _ = os.path.splitdrive(path) - base_drv, _ = os.path.splitdrive(base) - if path_drv.lower() == base_drv.lower(): - return os.path.relpath(path, base) - - return path - - def main() -> None: parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('location') From 8d58be4cb21fed8b3c8455e47b5c040a78f7cb3c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 16 Sep 2025 10:32:17 +0100 Subject: [PATCH 08/12] Regenerate build-details.json via PC\layout\ --- .github/workflows/mypy.yml | 2 +- Lib/test/test_build_details.py | 27 ++------- Makefile.pre.in | 2 +- PC/layout/main.py | 10 ++++ PC/layout/support/build_details.py | 33 +++++++++++ PC/layout/support/options.py | 9 ++- PCbuild/python.vcxproj | 2 +- ...d-details.py => generate_build_details.py} | 59 +++++++++++++------ Tools/build/mypy.ini | 2 +- 9 files changed, 103 insertions(+), 43 deletions(-) create mode 100644 PC/layout/support/build_details.py rename Tools/build/{generate-build-details.py => generate_build_details.py} (85%) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 5d5d77f29f6eb1..56bcc990bd8cf1 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -17,7 +17,7 @@ on: - "Tools/build/check_warnings.py" - "Tools/build/compute-changes.py" - "Tools/build/deepfreeze.py" - - "Tools/build/generate-build-details.py" + - "Tools/build/generate_build_details.py" - "Tools/build/generate_sbom.py" - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/mypy.ini" diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 4e8b81e1426488..fc41508460bf12 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -1,4 +1,3 @@ -import importlib import json import os import os.path @@ -6,27 +5,14 @@ import sysconfig import string import unittest -from pathlib import Path from test.support import is_android, is_apple_mobile, is_wasm32 - -BASE_PATH = Path( - __file__, # Lib/test/test_build_details.py - '..', # Lib/test - '..', # Lib - '..', # -).resolve() -MODULE_PATH = BASE_PATH / 'Tools' / 'build' / 'generate-build-details.py' +from test.test_tools import imports_under_tool try: - # Import "generate-build-details.py" as "generate_build_details" - spec = importlib.util.spec_from_file_location( - "generate_build_details", MODULE_PATH - ) - generate_build_details = importlib.util.module_from_spec(spec) - sys.modules["generate_build_details"] = generate_build_details - spec.loader.exec_module(generate_build_details) -except (FileNotFoundError, ImportError): + with imports_under_tool('build'): + import generate_build_details +except ImportError: generate_build_details = None @@ -178,9 +164,8 @@ def test_c_api(self): @unittest.skipIf( generate_build_details is None, - "Failed to import generate-build-details" + "Failed to import generate_build_details", ) -@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') @unittest.skipIf(is_wasm32, 'Feature not available on WebAssembly builds') class BuildDetailsRelativePathsTests(unittest.TestCase): @property @@ -191,7 +176,7 @@ def build_details_absolute_paths(self): @property def build_details_relative_paths(self): data = self.build_details_absolute_paths - generate_build_details.make_paths_relative(data, config_path=None) + generate_build_details.make_paths_relative(data, base_path=None) return data def test_round_trip(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index 34bd4540efb0b8..62f6d16ac932a7 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1004,7 +1004,7 @@ pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) fi build-details.json: pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/build-details.json + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_build_details.py `cat pybuilddir.txt`/build-details.json # Build static library $(LIBRARY): $(LIBRARY_OBJS) diff --git a/PC/layout/main.py b/PC/layout/main.py index 8543e7c56e1c41..4ccc1a724ed68b 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -22,6 +22,7 @@ __path__ = [str(Path(__file__).resolve().parent)] from .support.appxmanifest import * +from .support.build_details import * from .support.catalog import * from .support.constants import * from .support.filesets import * @@ -319,6 +320,9 @@ def _c(d): if ns.include_install_json or ns.include_install_embed_json or ns.include_install_test_json: yield "__install__.json", ns.temp / "__install__.json" + if ns.include_build_details_json: + yield "build-details.json", ns.temp / "build-details.json" + def _compile_one_py(src, dest, name, optimize, checked=True): import py_compile @@ -426,6 +430,12 @@ def generate_source_files(ns): with open(ns.temp / "__install__.json", "w", encoding="utf-8") as f: json.dump(calculate_install_json(ns, for_test=True), f, indent=2) + if ns.include_build_details_json: + log_info("Generating build-details.json in {}", ns.temp) + ns.temp.mkdir(parents=True, exist_ok=True) + base_path = Path(sys.base_prefix, "build-details.json") + write_relative_build_details(ns.temp / "build-details.json", base_path) + def _create_zip_file(ns): if not ns.zip: diff --git a/PC/layout/support/build_details.py b/PC/layout/support/build_details.py new file mode 100644 index 00000000000000..c701cd3ab0a8fe --- /dev/null +++ b/PC/layout/support/build_details.py @@ -0,0 +1,33 @@ +""" +Generate the PEP 739 'build-details.json' document. +""" + +import sys +from pathlib import Path + +PEP739_SCHEMA_VERSION = '1.0' + +ROOT_DIR = Path( + __file__, # PC/layout/support/build_details.py + '..', # PC/layout/support + '..', # PC/layout + '..', # PC + '..', # +).resolve() +TOOLS_BUILD_DIR = ROOT_DIR / 'Tools' / 'build' + +sys_path = sys.path[:] +try: + sys.path.insert(0, str(TOOLS_BUILD_DIR)) + import generate_build_details +finally: + sys.path = sys_path + del sys_path + + +def write_relative_build_details(out_path, base_path): + generate_build_details.write_build_details( + schema_version=PEP739_SCHEMA_VERSION, + base_path=base_path, + location=out_path, + ) diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index e8c393385425e7..cba1c9fcdcc786 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -39,6 +39,7 @@ def public(f): "install-json": {"help": "a PyManager __install__.json file"}, "install-embed-json": {"help": "a PyManager __install__.json file for embeddable distro"}, "install-test-json": {"help": "a PyManager __install__.json for the test distro"}, + "build-details-json": {"help": "a PEP 739 build-details.json file"}, } @@ -56,6 +57,7 @@ def public(f): "appxmanifest", "alias", "alias3x", + "build-details-json", # XXX: Disabled for now "precompile", ], }, @@ -69,9 +71,10 @@ def public(f): "props", "nuspec", "alias", + "build-details-json", ], }, - "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]}, + "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip", "build-details-json"]}, "default": { "help": "development kit package", "options": [ @@ -85,6 +88,7 @@ def public(f): "symbols", "html-doc", "alias", + "build-details-json", ], }, "embed": { @@ -96,6 +100,7 @@ def public(f): "flat-dlls", "underpth", "precompile", + "build-details-json", ], }, "pymanager": { @@ -109,6 +114,7 @@ def public(f): "dev", "html-doc", "install-json", + "build-details-json", ], }, "pymanager-test": { @@ -124,6 +130,7 @@ def public(f): "symbols", "tests", "install-test-json", + "build-details-json", ], }, } diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 976c0d52a1d0b1..c639af2b83141f 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -130,7 +130,7 @@ +"$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)Tools\build\generate_build_details.py" "$(OutDir)build-details.json"' ContinueOnError="true" /> diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate_build_details.py similarity index 85% rename from Tools/build/generate-build-details.py rename to Tools/build/generate_build_details.py index eac89e7558acfe..2c111bba57f61b 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate_build_details.py @@ -12,10 +12,30 @@ import os import sys import sysconfig +from pathlib import Path TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any + from typing import Any, Literal + + type StrPath = str | os.PathLike[str] + type ValidSchemaVersion = Literal['1.0'] + + +def write_build_details( + *, + schema_version: ValidSchemaVersion, + base_path: StrPath | None, + location: StrPath, +) -> None: + data = generate_data(schema_version) + if base_path is not None: + make_paths_relative(data, base_path) + + json_output = json.dumps(data, indent=2) + with open(location, 'w', encoding='utf-8') as f: + f.write(json_output) + f.write('\n') def version_info_to_dict(obj: sys._version_info) -> dict[str, Any]: @@ -29,7 +49,9 @@ def get_dict_key(container: dict[str, Any], key: str) -> dict[str, Any]: return container -def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: +def generate_data( + schema_version: ValidSchemaVersion +) -> collections.defaultdict[str, Any]: """Generate the build-details.json data (PEP 739). :param schema_version: The schema version of the data we want to generate. @@ -133,11 +155,7 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: return data -def make_paths_relative(data: dict[str, Any], config_path: str | None = None) -> None: - # Make base_prefix relative to the config_path directory - if config_path: - data['base_prefix'] = relative_path(data['base_prefix'], - os.path.dirname(config_path)) +def make_paths_relative(data: dict[str, Any], base_path: str | None = None) -> None: base_prefix = data['base_prefix'] # Update path values to make them relative to base_prefix @@ -167,8 +185,12 @@ def make_paths_relative(data: dict[str, Any], config_path: str | None = None) -> new_path = os.path.join('.', new_path) container[child] = new_path + if base_path: + # Make base_prefix relative to the base_path directory + config_dir = Path(base_path).resolve().parent + data['base_prefix'] = relative_path(base_prefix, config_dir) -def relative_path(path: str, base: str) -> str: +def relative_path(path: StrPath, base: StrPath) -> str: if os.name != 'nt': return os.path.relpath(path, base) @@ -201,15 +223,18 @@ def main() -> None: ) args = parser.parse_args() - - data = generate_data(args.schema_version) - if args.relative_paths: - make_paths_relative(data, args.config_file_path) - - json_output = json.dumps(data, indent=2) - with open(args.location, 'w', encoding='utf-8') as f: - f.write(json_output) - f.write('\n') + if os.name == 'nt': + # Windows builds are relocatable; always make paths relative. + base_path = args.config_file_path or args.location + elif args.relative_paths: + base_path = args.config_file_path + else: + base_path = None + write_build_details( + schema_version=args.schema_version, + base_path=base_path, + location=args.location, + ) if __name__ == '__main__': diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 331bada6f47d2e..5e5ed9bf9f2eec 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -7,7 +7,7 @@ files = Tools/build/check_warnings.py, Tools/build/compute-changes.py, Tools/build/deepfreeze.py, - Tools/build/generate-build-details.py, + Tools/build/generate_build_details.py, Tools/build/generate_sbom.py, Tools/build/generate_stdlib_module_names.py, Tools/build/verify_ensurepip_wheels.py, From a9276015ad6c41a6c0024a6dfe3e8334aa4b3530 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 16 Sep 2025 11:36:30 +0100 Subject: [PATCH 09/12] Fix the generated data for Windows --- .gitignore | 2 ++ Tools/build/generate_build_details.py | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index e842676d866bf8..e13fa9cdd73065 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,8 @@ gmon.out .pytest_cache/ .ruff_cache/ .DS_Store +__install__.json +build-details.json *.exe diff --git a/Tools/build/generate_build_details.py b/Tools/build/generate_build_details.py index 2c111bba57f61b..3016a29a93229b 100644 --- a/Tools/build/generate_build_details.py +++ b/Tools/build/generate_build_details.py @@ -68,10 +68,15 @@ def generate_data( data['base_prefix'] = sysconfig.get_config_var('installed_base') #data['base_interpreter'] = sys._base_executable - data['base_interpreter'] = os.path.join( - sysconfig.get_path('scripts'), - 'python' + sysconfig.get_config_var('VERSION'), - ) + if os.name == 'nt': + data['base_interpreter'] = os.path.join( + data['base_prefix'], os.path.basename(sys._base_executable) + ) + else: + data['base_interpreter'] = os.path.join( + sysconfig.get_path('scripts'), + 'python' + sysconfig.get_config_var('VERSION'), + ) data['platform'] = sysconfig.get_platform() data['language']['version'] = sysconfig.get_python_version() @@ -94,13 +99,19 @@ def generate_data( #data['suffixes']['debug_bytecode'] = importlib.machinery.DEBUG_BYTECODE_SUFFIXES data['suffixes']['extensions'] = importlib.machinery.EXTENSION_SUFFIXES - LIBDIR = sysconfig.get_config_var('LIBDIR') + if os.name == 'nt': + LIBDIR = data['base_prefix'] + else: + LIBDIR = sysconfig.get_config_var('LIBDIR') LDLIBRARY = sysconfig.get_config_var('LDLIBRARY') LIBRARY = sysconfig.get_config_var('LIBRARY') PY3LIBRARY = sysconfig.get_config_var('PY3LIBRARY') LIBPYTHON = sysconfig.get_config_var('LIBPYTHON') LIBPC = sysconfig.get_config_var('LIBPC') - INCLUDEPY = sysconfig.get_config_var('INCLUDEPY') + if os.name == 'nt': + INCLUDEPY = os.path.join(data['base_prefix'], 'include') + else: + INCLUDEPY = sysconfig.get_config_var('INCLUDEPY') if os.name == 'posix': # On POSIX, LIBRARY is always the static library, while LDLIBRARY is the From d743693e3f34dfdbc647e286827db8aa81754048 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 16 Sep 2025 11:39:13 +0100 Subject: [PATCH 10/12] Fail gracefully --- PC/layout/support/build_details.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/PC/layout/support/build_details.py b/PC/layout/support/build_details.py index c701cd3ab0a8fe..e70e528872290e 100644 --- a/PC/layout/support/build_details.py +++ b/PC/layout/support/build_details.py @@ -20,12 +20,16 @@ try: sys.path.insert(0, str(TOOLS_BUILD_DIR)) import generate_build_details +except ImportError: + generate_build_details = None finally: sys.path = sys_path del sys_path def write_relative_build_details(out_path, base_path): + if generate_build_details is None: + return generate_build_details.write_build_details( schema_version=PEP739_SCHEMA_VERSION, base_path=base_path, From d18f70e9102369477fa1952e09d7c7df0db8a527 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 16 Sep 2025 11:46:59 +0100 Subject: [PATCH 11/12] bin the type statements --- Tools/build/generate_build_details.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/build/generate_build_details.py b/Tools/build/generate_build_details.py index 3016a29a93229b..ef1a68a7673dee 100644 --- a/Tools/build/generate_build_details.py +++ b/Tools/build/generate_build_details.py @@ -16,10 +16,10 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Literal + from typing import Any, Literal, TypeAlias - type StrPath = str | os.PathLike[str] - type ValidSchemaVersion = Literal['1.0'] + StrPath: TypeAlias = str | os.PathLike[str] + ValidSchemaVersion: TypeAlias = Literal['1.0'] def write_build_details( From 57f3b26ad52966177ade82c3c5771077a705c5ec Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 16 Sep 2025 11:51:23 +0100 Subject: [PATCH 12/12] yet more linting banality --- Tools/build/generate_build_details.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Tools/build/generate_build_details.py b/Tools/build/generate_build_details.py index ef1a68a7673dee..372b81508db1d0 100644 --- a/Tools/build/generate_build_details.py +++ b/Tools/build/generate_build_details.py @@ -70,7 +70,8 @@ def generate_data( #data['base_interpreter'] = sys._base_executable if os.name == 'nt': data['base_interpreter'] = os.path.join( - data['base_prefix'], os.path.basename(sys._base_executable) + data['base_prefix'], + os.path.basename(sys._base_executable), # type: ignore[attr-defined] ) else: data['base_interpreter'] = os.path.join( @@ -166,7 +167,7 @@ def generate_data( return data -def make_paths_relative(data: dict[str, Any], base_path: str | None = None) -> None: +def make_paths_relative(data: dict[str, Any], base_path: StrPath | None = None) -> None: base_prefix = data['base_prefix'] # Update path values to make them relative to base_prefix @@ -211,7 +212,7 @@ def relative_path(path: StrPath, base: StrPath) -> str: if path_drv.lower() == base_drv.lower(): return os.path.relpath(path, base) - return path + return os.fspath(path) def main() -> None: