diff --git a/.cirrus.yml b/.cirrus.yml index 4d8c68fdc..f84e80792 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -77,8 +77,8 @@ opensuse-15_task: freebsd_task: freebsd_instance: - image_family: freebsd-14-0 - install_script: pkg install -y git ninja + image_family: freebsd-14-2 + install_script: pkg install -y git ninja patchelf << : *test macos-arm64_task: diff --git a/ci/alpine-3.docker b/ci/alpine-3.docker index 94d8a47a7..06e24cce6 100644 --- a/ci/alpine-3.docker +++ b/ci/alpine-3.docker @@ -2,6 +2,6 @@ # # SPDX-License-Identifier: MIT -# 20240817 +# 20250212 FROM alpine:3 RUN apk add --no-cache python3-dev py3-pip build-base ninja-is-really-ninja git patchelf diff --git a/ci/debian-11.docker b/ci/debian-11.docker index 5777afe83..e107fcd43 100644 --- a/ci/debian-11.docker +++ b/ci/debian-11.docker @@ -2,6 +2,6 @@ # # SPDX-License-Identifier: MIT -# 20240204 +# 20250212 FROM debian:bullseye RUN apt-get update && apt-get install -y git ninja-build patchelf python3-pip python3-venv && rm -rf /var/lib/apt/lists/* diff --git a/docs/how-to-guides/shared-libraries.rst b/docs/how-to-guides/shared-libraries.rst index 7fd731745..178620fc8 100644 --- a/docs/how-to-guides/shared-libraries.rst +++ b/docs/how-to-guides/shared-libraries.rst @@ -24,22 +24,22 @@ this ``libdir`` in this guide) or to a location in ``site-packages`` within the Python package install tree. All these scenarios are (or will be) supported, with some caveats: -+-----------------------+------------------+---------+-------+-------+ -| shared library source | install location | Windows | macOS | Linux | -+=======================+==================+=========+=======+=======+ -| internal | libdir | no (1) | ✓ | ✓ | -+-----------------------+------------------+---------+-------+-------+ -| internal | site-packages | ✓ | ✓ | ✓ | -+-----------------------+------------------+---------+-------+-------+ -| external | n/a | ✓ (2) | ✓ | ✓ | -+-----------------------+------------------+---------+-------+-------+ ++-----------------------+------------------+------------+-------+-------+ +| shared library source | install location | Windows | macOS | Linux | ++=======================+==================+============+=======+=======+ +| internal | libdir | ✓ :sup:`1` | ✓ | ✓ | ++-----------------------+------------------+------------+-------+-------+ +| internal | site-packages | ✓ | ✓ | ✓ | ++-----------------------+------------------+------------+-------+-------+ +| external | --- | ✓ :sup:`2` | ✓ | ✓ | ++-----------------------+------------------+------------+-------+-------+ .. TODO: add subproject as a source -1. Internal shared libraries on Windows cannot be automatically handled - correctly, and currently ``meson-python`` therefore raises an error for them. - `PR meson-python#551 `__ - may improve that situation in the near future. +1. Support for internal shared libraries on Windows is enabled with the + :option:`tool.meson-python.allow-windows-internal-shared-libs` option and + requires cooperation from the package to extend the DLL search path or + preload the required libraries. See below for more details. 2. External shared libraries require ``delvewheel`` usage on Windows (or some equivalent way, like amending the DLL search path to include the directory @@ -91,23 +91,26 @@ the lack of RPATH support: :start-after: start-literalinclude :end-before: end-literalinclude -If an internal shared library is not only used as part of a Python package, but -for example also as a regular shared library in a C/C++ project or as a -standalone library, then the method shown above won't work. The library is -then marked for installation into the system default ``libdir`` location. -Actually installing into ``libdir`` isn't possible with wheels, hence -``meson-python`` will instead do the following *on platforms other than -Windows*: +If an internal shared library is not only used as part of a Python package, +but for example also as a regular shared library then the method shown above +won't work. The library is then marked for installation into the system +default ``libdir`` location. Actually installing into ``libdir`` isn't +possible with wheels, hence ``meson-python`` will instead do the following: -1. Install the shared library to ``.mesonpy.libs`` (i.e., a +1. Install the shared library to the ``..mesonpy.libs`` top-level directory in the wheel, which on install will end up in - ``site-packages``). -2. Rewrite RPATH entries for install targets that depend on the shared library - to point to that new install location instead. + ``site-packages``. +2. On platforms other than Windows, rewrite RPATH entries for install targets + that depend on the shared library to point to that new install location + instead. -This will make the shared library work automatically, with no other action needed -from the package author. *However*, currently an error is raised for this situation -on Windows. This is documented also in :ref:`reference-limitations`. +On platforms other than Windows, this will make the shared library work +automatically, with no other action needed from the package author. On +Windows, due to the lack of RPATH support, the ``..mesonpy.libs`` +location search path needs to be added to the DLL search path, with code +similar to the one presented above. For this reason, handling of internal +shared libraries on Windows is conditional to setting the +:option:`tool.meson-python.allow-windows-internal-shared-libs` option. External shared libraries @@ -245,7 +248,7 @@ this will look something like: foo_dep = foo_subproj.get_variable('foo_dep') Now we can use ``foo_dep`` like a normal dependency, ``meson-python`` will -include it into the wheel in ``.mesonpy.libs`` just like an +include it into the wheel in ``..mesonpy.libs`` just like an internal shared library that targets ``libdir`` (see :ref:`internal-shared-libraries`). *Remember: this method doesn't support Windows (yet)!* diff --git a/docs/reference/limitations.rst b/docs/reference/limitations.rst index 7140bf2ae..78c3a30ac 100644 --- a/docs/reference/limitations.rst +++ b/docs/reference/limitations.rst @@ -25,22 +25,7 @@ run-time inside the package alongside the Python code, and use access them. -Shared libraries on Windows -=========================== - -On Windows, ``meson-python`` cannot encapsulate shared libraries -installed as part of the Meson project into the Python wheel for -Python extension modules or executables, in a way suitable for them to -be found at run-time. - -This limitation can be overcome with static linking or using -`delvewheel`_ to post-process the Python wheel to bundle the required -shared libraries and include the setup code to properly set the -library search path. - - .. _install_data(): https://mesonbuild.com/Reference-manual_functions.html#install_data .. _importlib-resources: https://importlib-resources.readthedocs.io/en/latest/index.html -.. _delvewheel: https://github.com/adang1345/delvewheel .. |install_data()| replace:: ``install_data()`` diff --git a/docs/reference/pyproject-settings.rst b/docs/reference/pyproject-settings.rst index 4ce6fddbe..b7806f569 100644 --- a/docs/reference/pyproject-settings.rst +++ b/docs/reference/pyproject-settings.rst @@ -13,6 +13,19 @@ This page lists the configuration settings supported by :ref:`how-to-guides-meson-args` guide for for information on how to use them and examples. +.. option:: tool.meson-python.allow-windows-internal-shared-libs + + Enable support for relocating internal shared libraries that would be + installed into the system shared library location to the + ``..mesonpy.libs`` folder also on Windows. The relocation can + be done transparently on UNIX platforms and on macOS, where the shared + library load path can be adjusted via RPATH or equivalent mechanisms. + Windows lacks a similar facility, thus the Python package is responsible to + extend the DLL load path to include this directory or to preload the + shared libraries. See :ref:`here ` for detailed + documentation. This option ensures that the package authors are aware of + this requirement. + .. option:: tool.meson-python.limited-api A boolean indicating whether the extension modules contained in the diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index bdc5f7766..51c6f0fb7 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -309,10 +309,12 @@ def __init__( metadata: Metadata, manifest: Dict[str, List[Tuple[pathlib.Path, str]]], limited_api: bool, + allow_windows_shared_libs: bool, ) -> None: self._metadata = metadata self._manifest = manifest self._limited_api = limited_api + self._allow_windows_shared_libs = allow_windows_shared_libs @property def _has_internal_libs(self) -> bool: @@ -422,6 +424,12 @@ def _install_path(self, wheel_file: mesonpy._wheelfile.WheelFile, origin: Path, if self._has_internal_libs: if _is_native(origin): + if sys.platform == 'win32' and not self._allow_windows_shared_libs: + raise NotImplementedError( + 'Loading shared libraries bundled in the Python wheel on Windows requires ' + 'setting the DLL load path or preloading. See the documentation for ' + 'the "tool.meson-python.allow-windows-internal-shared-libs" option.') + # When an executable, libray, or Python extension module is # dynamically linked to a library built as part of the project, # Meson adds a library load path to it pointing to the build @@ -567,6 +575,7 @@ def _string_or_path(value: Any, name: str) -> str: scheme = _table({ 'meson': _string_or_path, 'limited-api': _bool, + 'allow-windows-internal-shared-libs': _bool, 'args': _table({ name: _strings for name in _MESON_ARGS_KEYS }), @@ -784,6 +793,10 @@ def __init__( 'The package targets Python\'s Limited API, which is not supported by free-threaded CPython. ' 'The "python.allow_limited_api" Meson build option may be used to override the package default.') + # Shared library support on Windows requires collaboration + # from the package, make sure the developers acknowledge this. + self._allow_windows_shared_libs = pyproject_config.get('allow-windows-internal-shared-libs', False) + def _run(self, cmd: Sequence[str]) -> None: """Invoke a subprocess.""" # Flush the line to ensure that the log line with the executed @@ -988,13 +1001,13 @@ def sdist(self, directory: Path) -> pathlib.Path: def wheel(self, directory: Path) -> pathlib.Path: """Generates a wheel in the specified directory.""" self.build() - builder = _WheelBuilder(self._metadata, self._manifest, self._limited_api) + builder = _WheelBuilder(self._metadata, self._manifest, self._limited_api, self._allow_windows_shared_libs) return builder.build(directory) def editable(self, directory: Path) -> pathlib.Path: """Generates an editable wheel in the specified directory.""" self.build() - builder = _EditableWheelBuilder(self._metadata, self._manifest, self._limited_api) + builder = _EditableWheelBuilder(self._metadata, self._manifest, self._limited_api, self._allow_windows_shared_libs) return builder.build(directory, self._source_dir, self._build_dir, self._build_command, self._editable_verbose) diff --git a/mesonpy/_rpath.py b/mesonpy/_rpath.py index ab0e6471a..a7cbbb92a 100644 --- a/mesonpy/_rpath.py +++ b/mesonpy/_rpath.py @@ -16,25 +16,10 @@ from mesonpy._compat import Iterable, Path -if sys.platform == 'linux': - - def _get_rpath(filepath: Path) -> List[str]: - r = subprocess.run(['patchelf', '--print-rpath', os.fspath(filepath)], capture_output=True, text=True) - return r.stdout.strip().split(':') - - def _set_rpath(filepath: Path, rpath: Iterable[str]) -> None: - subprocess.run(['patchelf','--set-rpath', ':'.join(rpath), os.fspath(filepath)], check=True) +if sys.platform == 'win32' or sys.platform == 'cygwin': def fix_rpath(filepath: Path, libs_relative_path: str) -> None: - old_rpath = _get_rpath(filepath) - new_rpath = [] - for path in old_rpath: - if path.startswith('$ORIGIN/'): - path = '$ORIGIN/' + libs_relative_path - new_rpath.append(path) - if new_rpath != old_rpath: - _set_rpath(filepath, new_rpath) - + pass elif sys.platform == 'darwin': @@ -85,6 +70,21 @@ def fix_rpath(filepath: Path, libs_relative_path: str) -> None: _set_rpath(filepath, new_rpath) else: + # Assume that any other platform uses ELF binaries. + + def _get_rpath(filepath: Path) -> List[str]: + r = subprocess.run(['patchelf', '--print-rpath', os.fspath(filepath)], capture_output=True, text=True) + return r.stdout.strip().split(':') + + def _set_rpath(filepath: Path, rpath: Iterable[str]) -> None: + subprocess.run(['patchelf','--set-rpath', ':'.join(rpath), os.fspath(filepath)], check=True) def fix_rpath(filepath: Path, libs_relative_path: str) -> None: - raise NotImplementedError(f'Bundling libraries in wheel is not supported on {sys.platform}') + old_rpath = _get_rpath(filepath) + new_rpath = [] + for path in old_rpath: + if path.startswith('$ORIGIN/'): + path = '$ORIGIN/' + libs_relative_path + new_rpath.append(path) + if new_rpath != old_rpath: + _set_rpath(filepath, new_rpath) diff --git a/tests/packages/link-against-local-lib/__init__.py b/tests/packages/link-against-local-lib/__init__.py new file mode 100644 index 000000000..b7d66f5ea --- /dev/null +++ b/tests/packages/link-against-local-lib/__init__.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2025 The meson-python developers +# +# SPDX-License-Identifier: MIT + +import os +import sys + + +_path = os.path.join(os.path.dirname(__file__), '..', '.link_against_local_lib.mesonpy.libs') +if os.name == 'nt': + os.add_dll_directory(_path) +elif sys.platform == 'cygwin': + os.environ['PATH'] = os.pathsep.join((os.environ['PATH'], _path)) +del _path + + +from ._example import example_sum # noqa: E402 + + +__all__ = ['example_sum'] diff --git a/tests/packages/link-against-local-lib/examplemod.c b/tests/packages/link-against-local-lib/examplemod.c index 7d0204754..9e0b8d565 100644 --- a/tests/packages/link-against-local-lib/examplemod.c +++ b/tests/packages/link-against-local-lib/examplemod.c @@ -25,13 +25,13 @@ static PyMethodDef methods[] = { static struct PyModuleDef module = { PyModuleDef_HEAD_INIT, - "example", + "_example", NULL, -1, methods, }; -PyMODINIT_FUNC PyInit_example(void) +PyMODINIT_FUNC PyInit__example(void) { return PyModule_Create(&module); } diff --git a/tests/packages/link-against-local-lib/lib/examplelib.c b/tests/packages/link-against-local-lib/lib/examplelib.c index 9ba359db8..f7ef1e4e8 100644 --- a/tests/packages/link-against-local-lib/lib/examplelib.c +++ b/tests/packages/link-against-local-lib/lib/examplelib.c @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: MIT +#include "examplelib.h" + int sum(int a, int b) { return a + b; } diff --git a/tests/packages/link-against-local-lib/lib/examplelib.h b/tests/packages/link-against-local-lib/lib/examplelib.h index 74c8645b8..9a1317b80 100644 --- a/tests/packages/link-against-local-lib/lib/examplelib.h +++ b/tests/packages/link-against-local-lib/lib/examplelib.h @@ -2,4 +2,15 @@ // // SPDX-License-Identifier: MIT -int sum(int a, int b); +#pragma once + +#if defined(EXAMPLE_DLL_EXPORTS) + #define EXAMPLE_DLL __declspec(dllexport) +#elif defined(BAR_DLL_IMPORTS) + #define EXAMPLE_DLL __declspec(dllimport) +#else + #define EXAMPLE_DLL +#endif + + +EXAMPLE_DLL int sum(int a, int b); diff --git a/tests/packages/link-against-local-lib/lib/meson.build b/tests/packages/link-against-local-lib/lib/meson.build index 51167f9d1..cc756cf1f 100644 --- a/tests/packages/link-against-local-lib/lib/meson.build +++ b/tests/packages/link-against-local-lib/lib/meson.build @@ -5,5 +5,6 @@ example_lib = shared_library( 'example', 'examplelib.c', + c_args: lib_compile_args, install: true, ) diff --git a/tests/packages/link-against-local-lib/meson.build b/tests/packages/link-against-local-lib/meson.build index a45168cb4..e8cd28309 100644 --- a/tests/packages/link-against-local-lib/meson.build +++ b/tests/packages/link-against-local-lib/meson.build @@ -4,14 +4,28 @@ project('link-against-local-lib', 'c', version: '1.0.0') +if meson.get_compiler('c').get_id() in ['msvc', 'clang-cl', 'intel-cl'] + lib_compile_args = ['-DEXAMPLE_DLL_EXPORTS'] + link_args = ['-DEXAMPLE_DLL_IMPORTS'] +else + lib_compile_args = [] + link_args = ['-Wl,-rpath,custom-rpath'] +endif + subdir('lib') -py = import('python').find_installation() +py = import('python').find_installation(pure: false) + +py.install_sources( + '__init__.py', + subdir: 'example', +) py.extension_module( - 'example', + '_example', 'examplemod.c', link_with: example_lib, - link_args: ['-Wl,-rpath,custom-rpath'], + link_args: link_args, install: true, + subdir: 'example', ) diff --git a/tests/packages/link-against-local-lib/pyproject.toml b/tests/packages/link-against-local-lib/pyproject.toml index 2542e4395..ef8c7eed6 100644 --- a/tests/packages/link-against-local-lib/pyproject.toml +++ b/tests/packages/link-against-local-lib/pyproject.toml @@ -5,3 +5,6 @@ [build-system] build-backend = 'mesonpy' requires = ['meson-python'] + +[tool.meson-python] +allow-windows-internal-shared-libs = true diff --git a/tests/packages/sharedlib-in-package/mypkg/__init__.py b/tests/packages/sharedlib-in-package/mypkg/__init__.py index 9db5f9066..857d2e906 100644 --- a/tests/packages/sharedlib-in-package/mypkg/__init__.py +++ b/tests/packages/sharedlib-in-package/mypkg/__init__.py @@ -8,16 +8,15 @@ # start-literalinclude def _append_to_sharedlib_load_path(): - """ - Ensure the shared libraries in this package can be loaded on Windows. + """Ensure the shared libraries in this package can be loaded on Windows. Windows lacks a concept equivalent to RPATH: Python extension modules cannot find DLLs installed outside the DLL search path. This function ensures that the location of the shared libraries distributed inside this Python package is in the DLL search path of the process. - The Windows DLL search path includes the object depending on it is located: - the DLL search path needs to be augmented only when the Python extension + The Windows DLL search path includes the path to the object attempting + to load the DLL: it needs to be augmented only when the Python extension modules and the DLLs they require are installed in separate directories. Cygwin does not have the same default library search path: all locations where the shared libraries are installed need to be added to the search diff --git a/tests/test_project.py b/tests/test_project.py index 693fec6aa..9d57ffb6d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -330,7 +330,7 @@ def test_invalid_build_dir(package_pure, tmp_path, mocker): project.build() -@pytest.mark.skipif(not os.getenv('CI') or sys.platform != 'win32', reason='Requires MSVC') +@pytest.mark.skipif(not os.getenv('CI') or sys.platform != 'win32', reason='requires MSVC') def test_compiler(venv, package_detect_compiler, tmp_path): # Check that things are setup properly to use the MSVC compiler on # Windows. This effectively means running the compilation step diff --git a/tests/test_tags.py b/tests/test_tags.py index 8100681d1..433628b8f 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -77,7 +77,7 @@ def test_python_host_platform(monkeypatch): def wheel_builder_test_factory(content, pure=True, limited_api=False): manifest = defaultdict(list) manifest.update({key: [(pathlib.Path(x), os.path.join('build', x)) for x in value] for key, value in content.items()}) - return mesonpy._WheelBuilder(None, manifest, limited_api) + return mesonpy._WheelBuilder(None, manifest, limited_api, False) def test_tag_empty_wheel(): diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 5345305c3..b6f64ceec 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -158,7 +158,7 @@ def test_license_pep639(wheel_license_pep639): ''')) -@pytest.mark.skipif(sys.platform not in {'linux', 'darwin', 'sunos5'}, reason='Not supported on this platform') +@pytest.mark.skipif(sys.platform in {'win32', 'cygwin'}, reason='requires RPATH support') def test_contents(package_library, wheel_library): artifact = wheel.wheelfile.WheelFile(wheel_library) @@ -172,7 +172,6 @@ def test_contents(package_library, wheel_library): } -@pytest.mark.skipif(sys.platform not in {'linux', 'darwin', 'sunos5'}, reason='Not supported on this platform') def test_local_lib(venv, wheel_link_against_local_lib): venv.pip('install', wheel_link_against_local_lib) output = venv.python('-c', 'import example; print(example.example_sum(1, 2))') @@ -194,33 +193,32 @@ def test_link_library_in_subproject(venv, wheel_link_library_in_subproject): assert int(output) == 9 -@pytest.mark.skipif(sys.platform not in {'linux', 'darwin', 'sunos5'}, reason='Not supported on this platform') +@pytest.mark.skipif(sys.platform in {'win32', 'cygwin'}, reason='requires RPATH support') def test_rpath(wheel_link_against_local_lib, tmp_path): artifact = wheel.wheelfile.WheelFile(wheel_link_against_local_lib) artifact.extractall(tmp_path) - origin = {'linux': '$ORIGIN', 'darwin': '@loader_path', 'sunos5': '$ORIGIN'}[sys.platform] - expected = {f'{origin}/.link_against_local_lib.mesonpy.libs', 'custom-rpath',} + origin = '@loader_path' if sys.platform == 'darwin' else '$ORIGIN' + expected = {f'{origin}/../.link_against_local_lib.mesonpy.libs', 'custom-rpath',} - rpath = set(mesonpy._rpath._get_rpath(tmp_path / f'example{EXT_SUFFIX}')) + rpath = set(mesonpy._rpath._get_rpath(tmp_path / 'example' / f'_example{EXT_SUFFIX}')) # Verify that rpath is a superset of the expected one: linking to # the Python runtime may require additional rpath entries. assert rpath >= expected -@pytest.mark.skipif(sys.platform not in {'linux', 'darwin', 'sunos5'}, reason='Not supported on this platform') +@pytest.mark.skipif(sys.platform in {'win32', 'cygwin'}, reason='requires RPATH support') def test_uneeded_rpath(wheel_purelib_and_platlib, tmp_path): artifact = wheel.wheelfile.WheelFile(wheel_purelib_and_platlib) artifact.extractall(tmp_path) - origin = {'linux': '$ORIGIN', 'darwin': '@loader_path', 'sunos5': '$ORIGIN'}[sys.platform] - + origin = '@loader_path' if sys.platform == 'darwin' else '$ORIGIN' rpath = mesonpy._rpath._get_rpath(tmp_path / f'plat{EXT_SUFFIX}') for path in rpath: assert origin not in path -@pytest.mark.skipif(sys.platform not in {'linux', 'darwin', 'sunos5'}, reason='Not supported on this platform') +@pytest.mark.skipif(sys.platform in {'win32', 'cygwin'}, reason='requires executable bit support') def test_executable_bit(wheel_executable_bit): artifact = wheel.wheelfile.WheelFile(wheel_executable_bit) @@ -266,7 +264,8 @@ def test_entrypoints(wheel_full_metadata): def test_top_level_modules(package_module_types): with mesonpy._project() as project: - builder = mesonpy._EditableWheelBuilder(project._metadata, project._manifest, project._limited_api) + builder = mesonpy._EditableWheelBuilder( + project._metadata, project._manifest, project._limited_api, project._allow_windows_shared_libs) assert set(builder._top_level_modules) == { 'file', 'package',