From 1be9e944b4a84da6f56abb1b6242ff28647324d9 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sun, 25 May 2025 00:05:07 +0200 Subject: [PATCH 01/37] WheelHouse integrated management --- winpython/__init__.py | 2 +- winpython/wheelhouse.py | 140 ++++++++++++++++++++++++++++++++++++---- winpython/wppm.py | 12 +++- 3 files changed, 138 insertions(+), 16 deletions(-) diff --git a/winpython/__init__.py b/winpython/__init__.py index a9a855a8..c8ebca8a 100644 --- a/winpython/__init__.py +++ b/winpython/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '16.0.20250513' +__version__ = '16.1.20250524' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' diff --git a/winpython/wheelhouse.py b/winpython/wheelhouse.py index b29f560e..e93e7bc4 100644 --- a/winpython/wheelhouse.py +++ b/winpython/wheelhouse.py @@ -1,8 +1,14 @@ -# -# WheelHouse.py +#!/usr/bin/env python3 +""" +WheelHouse.py - manage WinPython local WheelHouse. +""" + import sys from pathlib import Path from collections import defaultdict +import shutil +import subprocess +from typing import Dict, List, Optional, Tuple # Use tomllib if available (Python 3.11+), otherwise fall back to tomli try: @@ -14,10 +20,9 @@ print("Please install tomli for Python < 3.11: pip install tomli") sys.exit(1) - - -def parse_pylock_toml(path): - with open(Path(path), "rb") as f: +def parse_pylock_toml(path: Path) -> Dict[str, Dict[str, str | List[str]]]: + """Parse a pylock.toml file and extract package information.""" + with open(path, "rb") as f: data = tomllib.load(f) # This dictionary maps package names to (version, [hashes]) @@ -46,9 +51,9 @@ def parse_pylock_toml(path): return package_hashes - -def write_requirements_txt(package_hashes, output_path="requirements.txt"): - with open(Path(output_path), "w") as f: +def write_requirements_txt(package_hashes: Dict[str, Dict[str, str | List[str]]], output_path: Path) -> None: + """Write package requirements to a requirements.txt file.""" + with open(output_path, "w") as f: for name, data in sorted(package_hashes.items()): version = data["version"] hashes = data["hashes"] @@ -63,13 +68,119 @@ def write_requirements_txt(package_hashes, output_path="requirements.txt"): print(f"✅ requirements.txt written to {output_path}") -def pylock_to_req(path, output_path=None): +def pylock_to_req(path: Path, output_path: Optional[Path] = None) -> None: + """Convert a pylock.toml file to requirements.txt.""" pkgs = parse_pylock_toml(path) if not output_path: - output_path = path.parent / (path.stem.replace('pylock','requirement_with_hash')+ '.txt') + output_path = path.parent / (path.stem.replace('pylock', 'requirement_with_hash') + '.txt') write_requirements_txt(pkgs, output_path) -if __name__ == "__main__": +def run_pip_command(command: List[str], check: bool = True, capture_output=True) -> Tuple[bool, Optional[str]]: + """Run a pip command and return the result.""" + print('\n', ' '.join(command),'\n') + try: + result = subprocess.run( + command, + capture_output=capture_output, + text=True, + check=check + ) + return (result.returncode == 0), (result.stderr or result.stdout) + except subprocess.CalledProcessError as e: + return False, e.stderr + except FileNotFoundError: + return False, "pip or Python not found." + except Exception as e: + return False, f"Unexpected error: {e}" + +def get_wheels(requirements: Path, wheeldir: Path, from_local: Optional[Path] = None + , only_check: bool = True,post_install: bool = False) -> bool: + """Download or check Python wheels based on requirements.""" + added = [] + if from_local: + added += ['--no-index', '--trusted-host=None', f'--find-links={from_local}'] + pre_checks = [sys.executable, "-m", "pip", "install", "--dry-run", "--no-deps", "--require-hashes", "-r", str(requirements)] + added + instruction = [sys.executable, "-m", "pip", "download", "--no-deps", "--require-hashes", "-r", str(requirements), "--dest", str(wheeldir)] + added + post_install_cmd = [sys.executable, "-m", "pip", "install", "--no-deps", "--require-hashes", "-r", str(requirements)] + added + + # Run pip dry-run, only if a move of directory + if from_local and from_local != wheeldir: + success, output = run_pip_command(pre_checks, check=False) + if not success: + print("❌ Dry-run failed. Here's the output:\n") + print(output or "") + return False + + print("✅ Requirements can be installed successfully (dry-run passed).\n") + + # All ok + if only_check and not post_install: + return True + + # Want to install + if not only_check and post_install: + success, output = run_pip_command(post_install_cmd, check=False, capture_output=False) + if not success: + print("❌ Installation failed. Here's the output:\n") + print(output or "") + return False + return True + + # Otherwise download also, but not install direct + success, output = run_pip_command(instruction) + if not success: + print("❌ Download failed. Here's the output:\n") + print(output or "") + return False + + return True + +def get_pylock_wheels(wheelhouse: Path, lockfile: Path, from_local: Optional[Path] = None) -> None: + """Get wheels for a pylock file.""" + filename = Path(lockfile).name + wheelhouse.mkdir(parents=True, exist_ok=True) + trusted_wheelhouse = wheelhouse / "included.wheels" + trusted_wheelhouse.mkdir(parents=True, exist_ok=True) + + filename_lock = wheelhouse / filename + filename_req = wheelhouse / (Path(lockfile).stem.replace('pylock', 'requirement_with_hash') + '.txt') + + pylock_to_req(Path(lockfile), filename_req) + + if not str(Path(lockfile)) == str(filename_lock): + shutil.copy2(lockfile, filename_lock) + + # We create a destination for wheels that is specific, so we can check all is there + destination_wheelhouse = wheelhouse / Path(lockfile).name.replace('.toml', '.wheels') + in_trusted = False + + if from_local is None: + # Try from trusted WheelHouse + print(f"\n\n*** Checking if we can install from our Local WheelHouse: ***\n {trusted_wheelhouse}\n\n") + in_trusted = get_wheels(filename_req, destination_wheelhouse, from_local=trusted_wheelhouse, only_check=True) + if in_trusted: + print(f"\n\n*** We can install from Local WheelHouse: ***\n {trusted_wheelhouse}\n\n") + user_input = input("Do you want to continue and install from {trusted_wheelhouse} ? (yes/no):") + if user_input.lower() == "yes": + in_installed = get_wheels(filename_req, trusted_wheelhouse, from_local=trusted_wheelhouse, only_check=True, post_install=True) + + if not in_trusted: + post_install = True if from_local and Path(from_local).is_dir and Path(from_local).samefile(destination_wheelhouse) else False + if post_install: + print(f"\n\n*** Installing from Local WheelHouse: ***\n {destination_wheelhouse}\n\n") + else: + print(f"\n\n*** Re-Checking if we can install from: {'pypi.org' if not from_local or from_local == '' else from_local}\n\n") + + in_pylock = get_wheels(filename_req, destination_wheelhouse, from_local=from_local, only_check=False, post_install=post_install) + if in_pylock: + if not post_install: + print(f"\n\n*** You can now install from this dedicated WheelHouse: ***\n {destination_wheelhouse}") + print(f"\n via:\n wppm {filename_lock} -wh {destination_wheelhouse}\n") + else: + print(f"\n\n*** We can't install {filename} ! ***\n\n") + +def main() -> None: + """Main entry point for the script.""" if len(sys.argv) != 2: print("Usage: python pylock_to_requirements.py pylock.toml") sys.exit(1) @@ -80,5 +191,8 @@ def pylock_to_req(path, output_path=None): sys.exit(1) pkgs = parse_pylock_toml(path) - dest = path.parent / (path.stem.replace('pylock','requirement_with_hash')+ '.txt') + dest = path.parent / (path.stem.replace('pylock', 'requirement_with_hash') + '.txt') write_requirements_txt(pkgs, dest) + +if __name__ == "__main__": + main() diff --git a/winpython/wppm.py b/winpython/wppm.py index 659d14a4..258795e3 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -231,7 +231,7 @@ def main(test=False): description="WinPython Package Manager: handle a WinPython Distribution and its packages", formatter_class=RawTextHelpFormatter, ) - parser.add_argument("fname", metavar="package", nargs="?", default="", type=str, help="optional package name or package wheel") + parser.add_argument("fname", metavar="package or lockfile", nargs="?", default="", type=str, help="optional package name or package wheel") parser.add_argument("-v", "--verbose", action="store_true", help="show more details on packages and actions") parser.add_argument( "--register", dest="registerWinPython", action="store_true", help=registerWinPythonHelp) # parser.add_argument( "--register_forall", action="store_true", help="Register distribution for all users") @@ -239,13 +239,14 @@ def main(test=False): # parser.add_argument( "--unregister_forall", action="store_true", help="un-Register distribution for all users") parser.add_argument("--fix", action="store_true", help="make WinPython fix") parser.add_argument("--movable", action="store_true", help="make WinPython movable") + parser.add_argument("-wh", "--wheelhouse", default=None, type=str, help="wheelhouse location to search for wheels: wppm pylock.toml -wh directory_of_wheels") parser.add_argument("-ls", "--list", action="store_true", help="list installed packages matching the given [optional] package expression: wppm -ls, wppm -ls pand") parser.add_argument("-lsa", dest="all", action="store_true",help=f"list details of package names matching given regular expression: wppm -lsa pandas -l1") parser.add_argument("-p",dest="pipdown",action="store_true",help="show Package dependencies of the given package[option]: wppm -p pandas[test]") parser.add_argument("-r", dest="pipup", action="store_true", help=f"show Reverse dependancies of the given package[option]: wppm -r pytest[test]") parser.add_argument("-l", "--levels", type=int, default=2, help="show 'LEVELS' levels of dependencies (with -p, -r), default is 2: wppm -p pandas -l1") parser.add_argument("-t", "--target", default=sys.prefix, help=f'path to target Python distribution (default: "{sys.prefix}")') - parser.add_argument("-i", "--install", action="store_true", help="install a given package wheel (use pip for more features)") + parser.add_argument("-i", "--install", action="store_true", help="install a given package wheel or pylock file (use pip for more features)") parser.add_argument("-u", "--uninstall", action="store_true", help="uninstall package (use pip for more features)") @@ -331,7 +332,14 @@ def main(test=False): sys.exit() else: raise FileNotFoundError(f"File not found: {args.fname}") + else: try: + filename = Path(args.fname).name + if filename.split('.')[0] == "pylock" and filename.split('.')[-1] == 'toml': + print(' a lock file !', args.fname, dist.target) + from winpython import wheelhouse as wh + wh.get_pylock_wheels(Path(dist.target).parent/ "WheelHouse", Path(args.fname), args.wheelhouse) + sys.exit() if args.uninstall: package = dist.find_package(args.fname) dist.uninstall(package) From 64a41c37e0c689020723755658c8de8d27502e67 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sun, 25 May 2025 11:10:23 +0200 Subject: [PATCH 02/37] the end of suffering --- make.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/make.py b/make.py index c1a6a93a..18aa8117 100644 --- a/make.py +++ b/make.py @@ -221,10 +221,7 @@ def build(self, rebuild: bool = True, requirements_files_list=None, winpy_dirnam if rebuild: self._print_action(f"Creating WinPython {self.winpython_directory} base directory") if self.winpython_directory.is_dir(): - try: - shutil.rmtree(self.winpython_directory, onexc=utils.onerror) - except TypeError: # before 3.12 - shutil.rmtree(self.winpython_directory, onerror=utils.onerror) + shutil.rmtree(self.winpython_directory) os.makedirs(self.winpython_directory, exist_ok=True) # preventive re-Creation of settings directory (self.winpython_directory / "settings" / "AppData" / "Roaming").mkdir(parents=True, exist_ok=True) From aa9ca121cfb2ec889956fbf1e92a6699591332c6 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sun, 25 May 2025 21:29:11 +0200 Subject: [PATCH 03/37] Wheelhouses and how to raise them --- generate_a_winpython_distro.bat | 89 +++++++++++++++++++------ generate_winpython_distros313_wheel.bat | 40 +++++++++++ make.py | 3 +- winpython/__init__.py | 2 +- winpython/wheelhouse.py | 40 ++++++----- winpython/wppm.py | 21 +++--- 6 files changed, 144 insertions(+), 51 deletions(-) create mode 100644 generate_winpython_distros313_wheel.bat diff --git a/generate_a_winpython_distro.bat b/generate_a_winpython_distro.bat index 0b466928..51c7dad0 100644 --- a/generate_a_winpython_distro.bat +++ b/generate_a_winpython_distro.bat @@ -2,7 +2,7 @@ rem generate_a_winpython_distro.bat: to be launched from a winpython directory, @echo on REM Initialize variables -if "%my_release_level%"=="" set my_release_level=b2 +if "%my_release_level%"=="" set my_release_level=b3 if "%my_create_installer%"=="" set my_create_installer=True rem Set archive directory and log file @@ -101,21 +101,47 @@ echo -------------------------------------- >>%my_archive_log% python -m pip install -r %my_requirements% -c %my_constraints% --pre --no-index --trusted-host=None --find-links=%my_find_links% >>%my_archive_log% python -c "from winpython import wppm;dist=wppm.Distribution(r'%WINPYDIR%');dist.patch_standard_packages('spyder', to_movable=True)" -REM Archive success +REM Add Wheelhouse (to replace per pip lock direct ? would allow paralellism) echo -------------------------------------- >>%my_archive_log% -echo "(%date% %time%) Archive success">>%my_archive_log% +echo "(%date% %time%) Add lockfile wheels">>%my_archive_log% echo -------------------------------------- >>%my_archive_log% -%target_python_exe% -m pip freeze > %my_archive_log%.packages_versions.txt - -REM Generate changelog and binaries -echo "(%date% %time%) Generate changelog and binaries">>%my_archive_log% set path=%my_original_path% -cd /D %~dp0 -call %my_buildenv%\scripts\env.bat -python.exe -c "from make import *;make_all(%my_release%, '%my_release_level%', pyver='%my_pyver%', basedir=r'%my_basedir%', verbose=True, architecture=%my_arch%, flavor='%my_flavor%', install_options=r'%my_install_options%', find_links=r'%my_find_links%', source_dirs=r'%my_source_dirs%', create_installer='%my_create_installer%', rebuild=False, python_target_release='%my_python_target_release%')" >> %my_archive_log% +@echo on +call %my_WINPYDIRBASE%\scripts\env.bat +@echo on +set WINPYVERLOCK=%WINPYVER2:.=_% +set pylockinclude=%my_root_dir_for_builds%\bd%my_python_target%\bu%addlockfile%\pylock.%addlockfile%-%WINPYARCH%bit-%WINPYVERLOCK%.toml +echo pylockinclude="%pylockinclude%" +if not "Z%addlockfile%Z"=="ZZ" if exist "%pylockinclude%" ( +echo %my_WINPYDIRBASE%\python\scripts\wppm.exe "%pylockinclude%" -ws "%my_find_links%" -wd "%my_WINPYDIRBASE%\wheelhouse\included.wheels">>%my_archive_log% +%my_WINPYDIRBASE%\python\scripts\wppm.exe "%pylockinclude%" -ws "%my_find_links%" -wd "%my_WINPYDIRBASE%\wheelhouse\included.wheels" +) -echo -------------------------------------- >>%my_archive_log% -echo "(%date% %time%) generate pylock.tomle files and requirement_with_hash.txt files">>%my_archive_log% +@echo on +echo wheelhousereq=%wheelhousereq% +set LOCKDIR=%WINPYDIRBASE%\..\ +set pip_lock_includedlocal=%LOCKDIR%pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_includedwheelslocal.toml +set pip_lock_includedweb=%LOCKDIR%pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_includedwheels.toml +set req_lock_includedlocal=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_includedwheelslocal.txt +set req_lock_includedweb=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_includedwheels.txt + +if not "Z%wheelhousereq%Z"=="ZZ" if exist "%wheelhousereq%" ( +echo JOYYYwheelhousereq=%wheelhousereq% +echo z%pip_lock_includedlocal%z=%pip_lock_includedlocal% +rem no winpython in it naturally, with deps +python.exe -m pip lock --no-index --trusted-host=None --find-links=%my_find_links% -c C:\WinP\constraints.txt -r "%wheelhousereq%" -o %pip_lock_includedlocal% +rem generating also classic requirement with hash-256, from obtained pylock.toml +python.exe -c "from winpython import wheelhouse as wh;wh.pylock_to_req(r'%pip_lock_includedlocal%', r'%req_lock_includedlocal%')" + +rem same with frozen web from local +python.exe -m pip lock --no-deps --require-hashes -c C:\WinP\constraints.txt -r "%req_lock_includedlocal%" -o %pip_lock_includedweb% + +echo %my_WINPYDIRBASE%\python\scripts\wppm.exe "%pip_lock_includedweb%" -ws "%my_find_links%" -wd "%my_WINPYDIRBASE%\wheelhouse\included.wheels">>%my_archive_log% +%my_WINPYDIRBASE%\python\scripts\wppm.exe "%pip_lock_includedweb%" -ws "%my_find_links%" -wd "%my_WINPYDIRBASE%\wheelhouse\included.wheels" +) + +echo -------------------------------------- >>%my_archive_log%; +echo "(%date% %time%) generate pylock.toml files and requirement.txt with hash files">>%my_archive_log% echo -------------------------------------- >>%my_archive_log% set path=%my_original_path% @@ -125,17 +151,17 @@ rem generate pip freeze requirements echo %date% %time% set LOCKDIR=%WINPYDIRBASE%\..\ -set WINPYVERLOCK=%WINPYVER:.=_% -set req=%LOCKDIR%requirement.%WINPYVERLOCK%_raw.txt -set wanted_req=%LOCKDIR%requirement.%WINPYVERLOCK%.txt -set pip_lock_web=%LOCKDIR%pylock.%WINPYVERLOCK%.toml -set pip_lock_local=%LOCKDIR%pylock.%WINPYVER%_local.toml -set req_lock_web=%LOCKDIR%requirement_with_hash.%WINPYVERLOCK%.txt -set req_lock_local=%LOCKDIR%requirement_with_hash.%WINPYVERLOCK%_local.txt +set WINPYVERLOCK=%WINPYVER2:.=_% +set req=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_raw.txt +set wanted_req=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%.txt +set pip_lock_web=%LOCKDIR%pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%.toml +set pip_lock_local=%LOCKDIR%pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_local.toml +set req_lock_web=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%.txt +set req_lock_local=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_local.txt -set my_archive_lockfile=%my_archive_dir%\pylock.%WINPYVERLOCK%_%date:/=-%at_%my_time%.toml -set my_archive_lockfile_local=%my_archive_dir%\pylock.%WINPYVERLOCK%_%date:/=-%at_%my_time%.local.toml -set my_changelog_lockfile=%~dp0changelogs\pylock.%WINPYVERLOCK%.toml +set my_archive_lockfile=%my_archive_dir%\pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_%date:/=-%at_%my_time%.toml +set my_archive_lockfile_local=%my_archive_dir%\pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_%date:/=-%at_%my_time%.local.toml +set my_changelog_lockfile=%~dp0changelogs\pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%.toml python.exe -m pip freeze>%req% findstr /v "winpython" %req% > %wanted_req% @@ -157,6 +183,25 @@ fc "%req_lock_web%" "%req_lock_local%" copy/Y %pip_lock_web% %my_archive_lockfile% copy/Y %pip_lock_web% %my_changelog_lockfile% + +REM Archive success +echo -------------------------------------- >>%my_archive_log% +echo "(%date% %time%) Archive success">>%my_archive_log% +echo -------------------------------------- >>%my_archive_log% +set path=%my_original_path% +call %my_WINPYDIRBASE%\scripts\env.bat + +%target_python_exe% -m pip freeze > %my_archive_log%.packages_versions.txt + +REM Generate changelog and binaries +echo "(%date% %time%) Generate changelog and binaries">>%my_archive_log% +set path=%my_original_path% +cd /D %~dp0 +call %my_buildenv%\scripts\env.bat + +python.exe -c "from make import *;make_all(%my_release%, '%my_release_level%', pyver='%my_pyver%', basedir=r'%my_basedir%', verbose=True, architecture=%my_arch%, flavor='%my_flavor%', install_options=r'%my_install_options%', find_links=r'%my_find_links%', source_dirs=r'%my_source_dirs%', create_installer='%my_create_installer%', rebuild=False, python_target_release='%my_python_target_release%')" >> %my_archive_log% + + echo -------------------------------------- >>%my_archive_log% echo "(%date% %time%) END OF CREATION">>%my_archive_log% echo -------------------------------------- >>%my_archive_log% diff --git a/generate_winpython_distros313_wheel.bat b/generate_winpython_distros313_wheel.bat new file mode 100644 index 00000000..9ae54ff3 --- /dev/null +++ b/generate_winpython_distros313_wheel.bat @@ -0,0 +1,40 @@ +rem this replace running manually from spyder the make.py +rem to launch from a winpython module 'make' directory + +set my_original_path=%path% + +set my_root_dir_for_builds=C:\Winp +set my_python_target=313 +set my_pyver=3.13 +set my_flavor=whl +set my_arch=64 + +rem settings delegated to generate_a_winpython_distro.bat +set my_release= +set my_release_level= + +rem list of installers to create separated per dot: False=none, .zip=zip, .7z=.7z, 7zip=auto-extractible 7z +set my_create_installer=7zip.7z.zip +set my_create_installer=.7z + +set my_preclear_build_directory=Yes + +set tmp_reqdir=%my_root_dir_for_builds%\bd%my_python_target% + +set my_requirements=C:\Winp\bd313\dot_requirements.txt +set my_source_dirs=C:\Winp\bd313\packages.win-amd64 + +set my_find_links=C:\Winp\packages.srcreq +set my_toolsdirs=C:\Winp\bdTools\Tools.dot +set my_docsdirs=C:\WinP\bdDocs\docs.dot + +set my_install_options=--no-index --pre --trusted-host=None + +rem set addlockfile=dot + +set wheelhousereq=C:\Winp\bd313\requirements64_whl.txt + + +call %~dp0\generate_a_winpython_distro.bat + +pause diff --git a/make.py b/make.py index 18aa8117..a23d719d 100644 --- a/make.py +++ b/make.py @@ -208,6 +208,7 @@ def _create_initial_batch_scripts(self): # Replacements for batch scripts (PyPy compatibility) executable_name = self.distribution.short_exe if self.distribution else "python.exe" # default to python.exe if distribution is not yet set init_variables = [('WINPYthon_exe', executable_name), ('WINPYthon_subdirectory_name', self.python_directory_name), ('WINPYVER', self.winpython_version_name)] + init_variables += [('WINPYVER2', f"{self.python_full_version}.{self.build_number}"), ('WINPYFLAVOR', self.flavor), ('WINPYARCH', self.architecture_bits)] with open(self.winpython_directory / "scripts" / "env.ini", "w") as f: f.writelines([f'{a}={b}\n' for a, b in init_variables]) @@ -264,7 +265,7 @@ def rebuild_winpython_package(source_directory: Path, target_directory: Path, ar for file in target_directory.glob("winpython-*"): if file.suffix in (".exe", ".whl", ".gz"): file.unlink() - utils.buildflit_wininst(source_directory, copy_to=target_directory, verbose=verbose) + utils.buildflit_wininst(source_directory, copy_to=target_directory, verbose=True) def make_all(build_number: int, release_level: str, pyver: str, architecture: int, basedir: Path, verbose: bool = False, rebuild: bool = True, create_installer: str = "True", install_options=["--no-index"], diff --git a/winpython/__init__.py b/winpython/__init__.py index c8ebca8a..5ab80ad9 100644 --- a/winpython/__init__.py +++ b/winpython/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '16.1.20250524' +__version__ = '16.1.20250525' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' diff --git a/winpython/wheelhouse.py b/winpython/wheelhouse.py index e93e7bc4..e54f9dac 100644 --- a/winpython/wheelhouse.py +++ b/winpython/wheelhouse.py @@ -93,18 +93,20 @@ def run_pip_command(command: List[str], check: bool = True, capture_output=True) except Exception as e: return False, f"Unexpected error: {e}" -def get_wheels(requirements: Path, wheeldir: Path, from_local: Optional[Path] = None +def get_wheels(requirements: Path, wheeldrain: Path, wheelorigin: Optional[Path] = None , only_check: bool = True,post_install: bool = False) -> bool: """Download or check Python wheels based on requirements.""" added = [] - if from_local: - added += ['--no-index', '--trusted-host=None', f'--find-links={from_local}'] + if wheelorigin: + added = ['--no-index', '--trusted-host=None', f'--find-links={wheelorigin}'] pre_checks = [sys.executable, "-m", "pip", "install", "--dry-run", "--no-deps", "--require-hashes", "-r", str(requirements)] + added - instruction = [sys.executable, "-m", "pip", "download", "--no-deps", "--require-hashes", "-r", str(requirements), "--dest", str(wheeldir)] + added + instruction = [sys.executable, "-m", "pip", "download", "--no-deps", "--require-hashes", "-r", str(requirements), "--dest", str(wheeldrain)] + added + if wheeldrain: + added = ['--no-index', '--trusted-host=None', f'--find-links={wheeldrain}'] post_install_cmd = [sys.executable, "-m", "pip", "install", "--no-deps", "--require-hashes", "-r", str(requirements)] + added - # Run pip dry-run, only if a move of directory - if from_local and from_local != wheeldir: + # Run pip dry-run, only if a move of wheels + if wheelorigin and wheelorigin != wheeldrain: success, output = run_pip_command(pre_checks, check=False) if not success: print("❌ Dry-run failed. Here's the output:\n") @@ -135,15 +137,15 @@ def get_wheels(requirements: Path, wheeldir: Path, from_local: Optional[Path] = return True -def get_pylock_wheels(wheelhouse: Path, lockfile: Path, from_local: Optional[Path] = None) -> None: - """Get wheels for a pylock file.""" +def get_pylock_wheels(wheelhouse: Path, lockfile: Path, wheelorigin: Optional[Path] = None, wheeldrain: Optional[Path] = None) -> None: + """Get wheels asked pylock file.""" filename = Path(lockfile).name wheelhouse.mkdir(parents=True, exist_ok=True) trusted_wheelhouse = wheelhouse / "included.wheels" trusted_wheelhouse.mkdir(parents=True, exist_ok=True) filename_lock = wheelhouse / filename - filename_req = wheelhouse / (Path(lockfile).stem.replace('pylock', 'requirement_with_hash') + '.txt') + filename_req = wheelhouse / (Path(lockfile).stem.replace('pylock', 'requirement') + '.txt') pylock_to_req(Path(lockfile), filename_req) @@ -151,27 +153,29 @@ def get_pylock_wheels(wheelhouse: Path, lockfile: Path, from_local: Optional[Pat shutil.copy2(lockfile, filename_lock) # We create a destination for wheels that is specific, so we can check all is there - destination_wheelhouse = wheelhouse / Path(lockfile).name.replace('.toml', '.wheels') + destination_wheelhouse = Path(wheeldrain) if wheeldrain else wheelhouse / Path(lockfile).name.replace('.toml', '.wheels') + destination_wheelhouse.mkdir(parents=True, exist_ok=True) + # there can be an override + + in_trusted = False - if from_local is None: + if wheelorigin is None: # Try from trusted WheelHouse print(f"\n\n*** Checking if we can install from our Local WheelHouse: ***\n {trusted_wheelhouse}\n\n") - in_trusted = get_wheels(filename_req, destination_wheelhouse, from_local=trusted_wheelhouse, only_check=True) + in_trusted = get_wheels(filename_req, destination_wheelhouse, wheelorigin=trusted_wheelhouse, only_check=True) if in_trusted: print(f"\n\n*** We can install from Local WheelHouse: ***\n {trusted_wheelhouse}\n\n") - user_input = input("Do you want to continue and install from {trusted_wheelhouse} ? (yes/no):") - if user_input.lower() == "yes": - in_installed = get_wheels(filename_req, trusted_wheelhouse, from_local=trusted_wheelhouse, only_check=True, post_install=True) + in_installed = get_wheels(filename_req, trusted_wheelhouse, wheelorigin=trusted_wheelhouse, only_check=True, post_install=True) if not in_trusted: - post_install = True if from_local and Path(from_local).is_dir and Path(from_local).samefile(destination_wheelhouse) else False + post_install = True if wheelorigin and Path(wheelorigin).is_dir and Path(wheelorigin).samefile(destination_wheelhouse) else False if post_install: print(f"\n\n*** Installing from Local WheelHouse: ***\n {destination_wheelhouse}\n\n") else: - print(f"\n\n*** Re-Checking if we can install from: {'pypi.org' if not from_local or from_local == '' else from_local}\n\n") + print(f"\n\n*** Re-Checking if we can install from: {'pypi.org' if not wheelorigin or wheelorigin == '' else wheelorigin}\n\n") - in_pylock = get_wheels(filename_req, destination_wheelhouse, from_local=from_local, only_check=False, post_install=post_install) + in_pylock = get_wheels(filename_req, destination_wheelhouse, wheelorigin=wheelorigin, only_check=False, post_install=post_install) if in_pylock: if not post_install: print(f"\n\n*** You can now install from this dedicated WheelHouse: ***\n {destination_wheelhouse}") diff --git a/winpython/wppm.py b/winpython/wppm.py index 258795e3..71296c72 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -15,7 +15,7 @@ from pathlib import Path from argparse import ArgumentParser, RawTextHelpFormatter from winpython import utils, piptree, associate - +from winpython import wheelhouse as wh # Workaround for installing PyVISA on Windows from source: os.environ["HOME"] = os.environ["USERPROFILE"] @@ -24,7 +24,7 @@ class Package: def __init__(self, fname: str, suggested_summary: str = None): self.fname = fname self.description = piptree.sum_up(suggested_summary) if suggested_summary else "" - self.name, self.version = None, None + self.name, self.version = fname, '?.?.?' if fname.lower().endswith((".zip", ".tar.gz", ".whl")): bname = Path(self.fname).name # e.g., "sqlite_bro-1.0.0..." infos = utils.get_source_package_infos(bname) # get name, version @@ -47,6 +47,7 @@ def __init__(self, target: str = None, verbose: bool = False): self.version, self.architecture = utils.get_python_infos(self.target) self.python_exe = utils.get_python_executable(self.target) self.short_exe = Path(self.python_exe).name + self.wheelhouse = Path(self.target).parent / "wheelhouse" def create_file(self, package, name, dstdir, contents): """Generate data file -- path is relative to distribution root dir""" @@ -91,7 +92,8 @@ def patch_all_shebang(self, to_movable: bool = True, max_exe_size: int = 999999, def install(self, package: Package, install_options: list[str] = None): """Install package in distribution.""" - if package.fname.endswith((".whl", ".tar.gz", ".zip")): # Check extension with tuple + if package.fname.endswith((".whl", ".tar.gz", ".zip")) or ( + ' ' not in package.fname and ';' not in package.fname and len(package.fname) >1): # Check extension with tuple self.install_bdist_direct(package, install_options=install_options) self.handle_specific_packages(package) # minimal post-install actions @@ -239,7 +241,8 @@ def main(test=False): # parser.add_argument( "--unregister_forall", action="store_true", help="un-Register distribution for all users") parser.add_argument("--fix", action="store_true", help="make WinPython fix") parser.add_argument("--movable", action="store_true", help="make WinPython movable") - parser.add_argument("-wh", "--wheelhouse", default=None, type=str, help="wheelhouse location to search for wheels: wppm pylock.toml -wh directory_of_wheels") + parser.add_argument("-ws", dest="wheelsource", default=None, type=str, help="location to search wheels: wppm pylock.toml -ws source_of_wheels") + parser.add_argument("-wd", dest="wheeldrain" , default=None, type=str, help="location of found wheels: wppm pylock.toml -wd destination_of_wheels") parser.add_argument("-ls", "--list", action="store_true", help="list installed packages matching the given [optional] package expression: wppm -ls, wppm -ls pand") parser.add_argument("-lsa", dest="all", action="store_true",help=f"list details of package names matching given regular expression: wppm -lsa pandas -l1") parser.add_argument("-p",dest="pipdown",action="store_true",help="show Package dependencies of the given package[option]: wppm -p pandas[test]") @@ -326,7 +329,7 @@ def main(test=False): sys.exit() if not args.install and not args.uninstall: args.install = True - if not Path(args.fname).is_file() and args.install: + if not Path(args.fname).is_file() and not args.install: if args.fname == "": parser.print_help() sys.exit() @@ -335,18 +338,18 @@ def main(test=False): else: try: filename = Path(args.fname).name + install_from_wheelhouse = ["--no-index", "--trusted-host=None", f"--find-links={dist.wheelhouse / 'included.wheels'}"] if filename.split('.')[0] == "pylock" and filename.split('.')[-1] == 'toml': print(' a lock file !', args.fname, dist.target) - from winpython import wheelhouse as wh - wh.get_pylock_wheels(Path(dist.target).parent/ "WheelHouse", Path(args.fname), args.wheelhouse) - sys.exit() + wh.get_pylock_wheels(dist.wheelhouse, Path(args.fname), args.wheelsource, args.wheeldrain) + sys.exit() if args.uninstall: package = dist.find_package(args.fname) dist.uninstall(package) elif args.install: package = Package(args.fname) if args.install: - dist.install(package) + dist.install(package, install_options=install_from_wheelhouse) except NotImplementedError: raise RuntimeError("Package is not (yet) supported by WPPM") else: From aae61bac9abb8d6d70a00bdeb340c1c359a2ede0 Mon Sep 17 00:00:00 2001 From: stonebig Date: Tue, 27 May 2025 21:22:16 +0200 Subject: [PATCH 04/37] sometimes --- winpython/wheelhouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winpython/wheelhouse.py b/winpython/wheelhouse.py index e54f9dac..1653aded 100644 --- a/winpython/wheelhouse.py +++ b/winpython/wheelhouse.py @@ -166,7 +166,7 @@ def get_pylock_wheels(wheelhouse: Path, lockfile: Path, wheelorigin: Optional[Pa in_trusted = get_wheels(filename_req, destination_wheelhouse, wheelorigin=trusted_wheelhouse, only_check=True) if in_trusted: print(f"\n\n*** We can install from Local WheelHouse: ***\n {trusted_wheelhouse}\n\n") - in_installed = get_wheels(filename_req, trusted_wheelhouse, wheelorigin=trusted_wheelhouse, only_check=True, post_install=True) + in_installed = get_wheels(filename_req, trusted_wheelhouse, wheelorigin=trusted_wheelhouse, only_check=False, post_install=True) if not in_trusted: post_install = True if wheelorigin and Path(wheelorigin).is_dir and Path(wheelorigin).samefile(destination_wheelhouse) else False From 03e73b94bfb1d1bcac2819ed01c1eaa9f4119b22 Mon Sep 17 00:00:00 2001 From: stonebig Date: Wed, 28 May 2025 18:45:55 +0200 Subject: [PATCH 05/37] build with parametrized compression levels --- generate_winpython_distros313_wheel.bat | 4 ++-- make.py | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/generate_winpython_distros313_wheel.bat b/generate_winpython_distros313_wheel.bat index 9ae54ff3..02edeb99 100644 --- a/generate_winpython_distros313_wheel.bat +++ b/generate_winpython_distros313_wheel.bat @@ -14,8 +14,8 @@ set my_release= set my_release_level= rem list of installers to create separated per dot: False=none, .zip=zip, .7z=.7z, 7zip=auto-extractible 7z -set my_create_installer=7zip.7z.zip -set my_create_installer=.7z +set my_create_installer=7zip-mx5.7z-mx7.zip +set my_create_installer=.7z-mx7 set my_preclear_build_directory=Yes diff --git a/make.py b/make.py index a23d719d..1445cadc 100644 --- a/make.py +++ b/make.py @@ -146,17 +146,18 @@ def architecture_bits(self) -> int: """Returns the architecture (32 or 64 bits) of the distribution.""" return self.distribution.architecture if self.distribution else 64 - def create_installer_7zip(self, installer_type: str = ".exe"): - """Creates a WinPython installer using 7-Zip: ".exe", ".7z", ".zip")""" + def create_installer_7zip(self, installer_type: str = "exe", compression= "mx5"): + """Creates a WinPython installer using 7-Zip: "exe", "7z", "zip")""" self._print_action(f"Creating WinPython installer ({installer_type})") - if installer_type not in [".exe", ".7z", ".zip"]: - raise RuntimeError("installer_type {installer_type} is undefined") + if installer_type not in ["exe", "7z", "zip"]: + return DISTDIR = self.winpython_directory filename_stem = f"Winpython{self.architecture_bits}-{self.python_full_version}.{self.build_number}{self.flavor}{self.release_level}" - fullfilename = DISTDIR.parent / (filename_stem + installer_type) - sfx_option = "-sfx7z.sfx" if installer_type == ".exe" else "" - zip_option = "-tzip" if installer_type == ".zip" else "" - command = f'"{find_7zip_executable()}" {zip_option} -mx5 a "{fullfilename}" "{DISTDIR}" {sfx_option}' + fullfilename = DISTDIR.parent / (filename_stem + "." + installer_type) + sfx_option = "-sfx7z.sfx" if installer_type == "exe" else "" + zip_option = "-tzip" if installer_type == "zip" else "" + compress_level = "mx5" if compression == "" else compression + command = f'"{find_7zip_executable()}" {zip_option} -{compress_level} a "{fullfilename}" "{DISTDIR}" {sfx_option}' print(f'Executing 7-Zip script: "{command}"') try: subprocess.run(command, shell=True, check=True, stderr=sys.stderr, stdout=sys.stderr) @@ -331,9 +332,9 @@ def make_all(build_number: int, release_level: str, pyver: str, architecture: in builder.build(rebuild=rebuild, requirements_files_list=requirements_files_list, winpy_dirname=winpython_dirname) - for installer_type in [".zip", ".7z", ".exe"]: - if installer_type in create_installer.lower().replace("7zip",".exe"): - builder.create_installer_7zip(installer_type) + for commmand in create_installer.lower().replace("7zip",".exe").split('.'): + installer_type, compression = (commmand + "-").split("-")[:2] + builder.create_installer_7zip(installer_type, compression) if __name__ == "__main__": # DO create only one Winpython distribution at a time From 83baeddef72d098e797761838a648c5937e65f37 Mon Sep 17 00:00:00 2001 From: stonebig Date: Wed, 28 May 2025 19:52:04 +0200 Subject: [PATCH 06/37] restore default behavior --- winpython/wppm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/winpython/wppm.py b/winpython/wppm.py index 71296c72..9e676ce9 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -329,12 +329,12 @@ def main(test=False): sys.exit() if not args.install and not args.uninstall: args.install = True - if not Path(args.fname).is_file() and not args.install: + if not Path(args.fname).is_file() and args.install: if args.fname == "": parser.print_help() sys.exit() else: - raise FileNotFoundError(f"File not found: {args.fname}") + args.install = True # for Drag & Drop of .toml (and not wheel) else: try: filename = Path(args.fname).name From 99709de4edd0c48ed6dab6e18a43305f9db06be4 Mon Sep 17 00:00:00 2001 From: stonebig Date: Wed, 28 May 2025 23:49:33 +0200 Subject: [PATCH 07/37] wheelhouse in changelogs --- diff.py | 23 +++++++++++++++++------ make.py | 6 ++++++ winpython/wppm.py | 11 +++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/diff.py b/diff.py index 1fd01586..61b4ef80 100644 --- a/diff.py +++ b/diff.py @@ -31,7 +31,7 @@ def __str__(self): def from_text(self, text): match = re.match(self.PATTERN_OLD, text) or re.match(self.PATTERN, text) if not match: - raise ValueError("Text does not match expected pattern") + raise ValueError("Text does not match expected pattern: "+ text) self.name, self.url, self.version, self.description = match.groups() def to_wiki(self): @@ -45,6 +45,7 @@ class PackageIndex: WINPYTHON_PATTERN = r"\#\# WinPython\-*[0-9b-t]* ([0-9\.a-zA-Z]*)" TOOLS_LINE = "### Tools" PYTHON_PACKAGES_LINE = "### Python packages" + WHEELHOUSE_PACKAGES_LINE = "### WheelHouse packages" HEADER_LINE1 = "Name | Version | Description" HEADER_LINE2 = "-----|---------|------------" @@ -55,6 +56,7 @@ def __init__(self, version, basedir=None, flavor="", architecture=64): self.architecture = architecture self.other_packages = {} self.python_packages = {} + self.wheelhouse_packages = {} self.from_file(basedir) def from_file(self, basedir): @@ -67,24 +69,29 @@ def from_file(self, basedir): def from_text(self, text): version = re.match(self.WINPYTHON_PATTERN + self.flavor, text).groups()[0] assert version == self.version - tools_flag = python_flag = False + tools_flag = python_flag = wheelhouse_flag = False for line in text.splitlines(): if line: if line == self.TOOLS_LINE: - tools_flag, python_flag = True, False + tools_flag, python_flag, wheelhouse_flag = True, False, False continue elif line == self.PYTHON_PACKAGES_LINE: - tools_flag, python_flag = False, True + tools_flag, python_flag, wheelhouse_flag = False, True, False + continue + elif line == self.WHEELHOUSE_PACKAGES_LINE: + tools_flag, python_flag, wheelhouse_flag = False, False, True continue elif line in (self.HEADER_LINE1, self.HEADER_LINE2, "
", "
"): continue - if tools_flag or python_flag: + if tools_flag or python_flag or wheelhouse_flag: package = Package() package.from_text(line) if tools_flag: self.other_packages[package.name] = package - else: + elif python_flag: self.python_packages[package.name] = package + else: + self.wheelhouse_packages[package.name] = package def diff_package_dicts(old_packages, new_packages): """Return difference between package old and package new""" @@ -141,6 +148,10 @@ def compare_package_indexes(version2, version1=None, basedir=None, flavor="", fl if py_text: text += PackageIndex.PYTHON_PACKAGES_LINE + "\r\n\r\n" + py_text + py_text = diff_package_dicts(pi1.wheelhouse_packages, pi2.wheelhouse_packages) + if py_text: + text += PackageIndex.WHEELHOUSE_PACKAGES_LINE + "\r\n\r\n" + py_text + text += "\r\n\r\n* * *\r\n" return text diff --git a/make.py b/make.py index 1445cadc..d66a5213 100644 --- a/make.py +++ b/make.py @@ -120,6 +120,12 @@ def package_index_markdown(self) -> str: -----|---------|------------ {self.distribution.get_installed_packages_markdown()} +### WheelHouse packages + +Name | Version | Description +-----|---------|------------ +{self.distribution.get_wheelhouse_packages_markdown()} + """ diff --git a/winpython/wppm.py b/winpython/wppm.py index 9e676ce9..342f3471 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -76,6 +76,17 @@ def get_installed_packages_markdown(self) -> str: ] return "\n".join(package_lines) + def get_wheelhouse_packages_markdown(self) -> str: + wheeldir = self.wheelhouse / 'included.wheels' + if wheeldir.is_dir(): + package_lines = [ + f"[{name}](https://pypi.org/project/{name}) | {version} | {summary}" + for name, version, summary in wh.list_packages_with_metadata(str(wheeldir)) + #for pkg in sorted(wh.list_packages_with_metadata(str(wheeldir)), key=lambda p: p.name.lower()) + ] + return "\n".join(package_lines) + return "" + def find_package(self, name: str) -> Package | None: """Find installed package by name.""" for pack in self.get_installed_packages(): From 2e7f790deb8ce1405a723dce3daeeb8b467982b1 Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 29 May 2025 09:38:12 +0200 Subject: [PATCH 08/37] fix the fix on historic behavioir --- winpython/wppm.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/winpython/wppm.py b/winpython/wppm.py index 342f3471..d6194829 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -338,14 +338,11 @@ def main(test=False): if args.movable: p = subprocess.Popen(["start", "cmd", "/k",dist.python_exe, "-c" , cmd_mov], shell = True, cwd=dist.target) sys.exit() - if not args.install and not args.uninstall: - args.install = True - if not Path(args.fname).is_file() and args.install: - if args.fname == "": + if not args.install and not args.uninstall and args.fname.endswith(".toml"): + args.install = True # for Drag & Drop of .toml (and not wheel) + if args.fname == "" or (not args.install and not args.uninstall): parser.print_help() sys.exit() - else: - args.install = True # for Drag & Drop of .toml (and not wheel) else: try: filename = Path(args.fname).name From 88ceb637a5d8bed123ea06705a304fc1911ad90a Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 29 May 2025 15:38:25 +0200 Subject: [PATCH 09/37] stronger fix of wheelhouse report --- winpython/piptree.py | 12 ++------- winpython/wheelhouse.py | 56 ++++++++++++++++++++++++++++++++++++++++- winpython/wppm.py | 6 ++--- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/winpython/piptree.py b/winpython/piptree.py index 23a53ff3..a2fa5ee8 100644 --- a/winpython/piptree.py +++ b/winpython/piptree.py @@ -17,6 +17,7 @@ from pip._vendor.packaging.markers import Marker from importlib.metadata import Distribution, distributions from pathlib import Path +from winpython import utils logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -25,15 +26,6 @@ class PipDataError(Exception): """Custom exception for PipData related errors.""" pass -def sum_up(text: str, max_length: int = 144, stop_at: str = ". ") -> str: - """Summarize text to fit within max_length, ending at last complete sentence.""" - summary = (text + os.linesep).splitlines()[0] - if len(summary) <= max_length: - return summary - if stop_at and stop_at in summary[:max_length]: - return summary[:summary.rfind(stop_at, 0, max_length)] + stop_at.rstrip() - return summary[:max_length].rstrip() - class PipData: """Manages package metadata and dependency relationships in a Python environment.""" @@ -287,5 +279,5 @@ def pip_list(self, full: bool = False, max_length: int = 144) -> List[Tuple[str, """ pkgs = sorted(self.distro.items()) if full: - return [(p, d["version"], sum_up(d["summary"], max_length)) for p, d in pkgs] + return [(p, d["version"], utils.sum_up(d["summary"], max_length)) for p, d in pkgs] return [(p, d["version"]) for p, d in pkgs] diff --git a/winpython/wheelhouse.py b/winpython/wheelhouse.py index 1653aded..f0963aed 100644 --- a/winpython/wheelhouse.py +++ b/winpython/wheelhouse.py @@ -2,13 +2,20 @@ """ WheelHouse.py - manage WinPython local WheelHouse. """ - +import os +import re +import tarfile +import zipfile import sys from pathlib import Path from collections import defaultdict import shutil import subprocess from typing import Dict, List, Optional, Tuple +from email import message_from_bytes +from . import utils + +from packaging.utils import canonicalize_name # Use tomllib if available (Python 3.11+), otherwise fall back to tomli try: @@ -183,6 +190,53 @@ def get_pylock_wheels(wheelhouse: Path, lockfile: Path, wheelorigin: Optional[Pa else: print(f"\n\n*** We can't install {filename} ! ***\n\n") +def extract_metadata_from_wheel(filepath: Path) -> Optional[Tuple[str, str, str]]: + "get metadata from a wheel package" + with zipfile.ZipFile(filepath, 'r') as z: + # Locate *.dist-info/METADATA file inside but not in a vendored directory (flit-core) + for name in z.namelist(): + if name.endswith(r'.dist-info/METADATA') and name.split("/")[1] == "METADATA": + with z.open(name) as meta_file: + metadata = message_from_bytes(meta_file.read()) + name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type + version = str(metadata.get('Version', 'unknown')) + summary = utils.sum_up(str(metadata.get('Summary', ''))) + return name, version, summary + return None + +def extract_metadata_from_sdist(filepath: Path) -> Optional[Tuple[str, str, str]]: + "get metadata from a tar.gz or .zip package" + open_func = tarfile.open if filepath.suffixes[-2:] == ['.tar', '.gz'] else zipfile.ZipFile + with open_func(filepath, 'r') as archive: + namelist = archive.getnames() if isinstance(archive, tarfile.TarFile) else archive.namelist() + for name in namelist: + if name.endswith('PKG-INFO'): + content = archive.extractfile(name).read() if isinstance(archive, tarfile.TarFile) else archive.open(name).read() + metadata = message_from_bytes(content) + name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type + version = str(metadata.get('Version', 'unknown')) + summary = utils.sum_up(str(metadata.get('Summary', ''))) + return name, version, summary + return None + +def list_packages_with_metadata(directory: str) -> List[Tuple[str, str, str]]: + "get metadata from a Wheelhouse directory" + results = [] + for file in os.listdir(directory): + path = Path(directory) / file + try: + if path.suffix == '.whl': + meta = extract_metadata_from_wheel(path) + elif path.suffix == '.zip' or path.name.endswith('.tar.gz'): + meta = extract_metadata_from_sdist(path) + else: + continue + if meta: + results.append(meta) + except OSError: #Exception as e: # need to see it + print(f"Skipping {file}: {e}") + return results + def main() -> None: """Main entry point for the script.""" if len(sys.argv) != 2: diff --git a/winpython/wppm.py b/winpython/wppm.py index d6194829..a8c92abf 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -16,6 +16,7 @@ from argparse import ArgumentParser, RawTextHelpFormatter from winpython import utils, piptree, associate from winpython import wheelhouse as wh +from operator import itemgetter # Workaround for installing PyVISA on Windows from source: os.environ["HOME"] = os.environ["USERPROFILE"] @@ -23,7 +24,7 @@ class Package: """Standardize a Package from filename or pip list.""" def __init__(self, fname: str, suggested_summary: str = None): self.fname = fname - self.description = piptree.sum_up(suggested_summary) if suggested_summary else "" + self.description = (utils.sum_up(suggested_summary) if suggested_summary else "").strip() self.name, self.version = fname, '?.?.?' if fname.lower().endswith((".zip", ".tar.gz", ".whl")): bname = Path(self.fname).name # e.g., "sqlite_bro-1.0.0..." @@ -81,8 +82,7 @@ def get_wheelhouse_packages_markdown(self) -> str: if wheeldir.is_dir(): package_lines = [ f"[{name}](https://pypi.org/project/{name}) | {version} | {summary}" - for name, version, summary in wh.list_packages_with_metadata(str(wheeldir)) - #for pkg in sorted(wh.list_packages_with_metadata(str(wheeldir)), key=lambda p: p.name.lower()) + for name, version, summary in sorted(wh.list_packages_with_metadata(str(wheeldir)), key=itemgetter(0 , 1)) # lambda p: p[0].lower()) ] return "\n".join(package_lines) return "" From 82b6794df5ff5a5bfc76107dba6e182f4cbbb93d Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 29 May 2025 15:39:15 +0200 Subject: [PATCH 10/37] fix further --- winpython/utils.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/winpython/utils.py b/winpython/utils.py index 40961ec1..97f38e78 100644 --- a/winpython/utils.py +++ b/winpython/utils.py @@ -71,30 +71,14 @@ def onerror(function, path, excinfo): else: raise -def getFileProperties(fname): - """Read all properties of the given file return them as a dictionary.""" - import win32api - prop_names = ('ProductName', 'ProductVersion', 'FileDescription', 'FileVersion') - props = {'FixedFileInfo': None, 'StringFileInfo': None, 'FileVersion': None} - - try: - fixed_info = win32api.GetFileVersionInfo(fname, '\\') - props['FixedFileInfo'] = fixed_info - props['FileVersion'] = "{}.{}.{}.{}".format( - fixed_info['FileVersionMS'] // 65536, - fixed_info['FileVersionMS'] % 65536, - fixed_info['FileVersionLS'] // 65536, - fixed_info['FileVersionLS'] % 65536 - ) - lang, codepage = win32api.GetFileVersionInfo(fname, '\\VarFileInfo\\Translation')[0] - props['StringFileInfo'] = { - prop_name: win32api.GetFileVersionInfo(fname, f'\\StringFileInfo\\{lang:04X}{codepage:04X}\\{prop_name}') - for prop_name in prop_names - } - except: - pass - - return props +def sum_up(text: str, max_length: int = 144, stop_at: str = ". ") -> str: + """Summarize text to fit within max_length, ending at last complete sentence.""" + summary = (text + os.linesep).splitlines()[0].strip() + if len(summary) <= max_length: + return summary + if stop_at and stop_at in summary[:max_length]: + return summary[:summary.rfind(stop_at, 0, max_length)] + stop_at.strip() + return summary[:max_length].strip() def get_special_folder_path(path_name): """Return special folder path.""" From de0423cefe10f5f1d853706be919e9117c44ac3c Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 29 May 2025 16:11:00 +0200 Subject: [PATCH 11/37] at last... a glitch-less one --- winpython/wheelhouse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/winpython/wheelhouse.py b/winpython/wheelhouse.py index f0963aed..87c6a9ac 100644 --- a/winpython/wheelhouse.py +++ b/winpython/wheelhouse.py @@ -13,6 +13,8 @@ import subprocess from typing import Dict, List, Optional, Tuple from email import message_from_bytes +from email.parser import BytesParser +from email.policy import default from . import utils from packaging.utils import canonicalize_name @@ -197,7 +199,7 @@ def extract_metadata_from_wheel(filepath: Path) -> Optional[Tuple[str, str, str] for name in z.namelist(): if name.endswith(r'.dist-info/METADATA') and name.split("/")[1] == "METADATA": with z.open(name) as meta_file: - metadata = message_from_bytes(meta_file.read()) + metadata = BytesParser(policy=default).parse(meta_file) name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type version = str(metadata.get('Version', 'unknown')) summary = utils.sum_up(str(metadata.get('Summary', ''))) From e9da3e55c5709947cb94157e867c59ccc22dbb68 Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 29 May 2025 18:03:35 +0200 Subject: [PATCH 12/37] more checks, less bagages --- winpython/__init__.py | 2 +- winpython/piptree.py | 2 +- winpython/utils.py | 2 -- winpython/wheelhouse.py | 54 ++++++++++++++++++++++++++++++++--------- winpython/wppm.py | 4 +-- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/winpython/__init__.py b/winpython/__init__.py index 5ab80ad9..f4815474 100644 --- a/winpython/__init__.py +++ b/winpython/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '16.1.20250525' +__version__ = '16.2.20250529' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' diff --git a/winpython/piptree.py b/winpython/piptree.py index a2fa5ee8..1cc1d1ab 100644 --- a/winpython/piptree.py +++ b/winpython/piptree.py @@ -17,7 +17,7 @@ from pip._vendor.packaging.markers import Marker from importlib.metadata import Distribution, distributions from pathlib import Path -from winpython import utils +from . import utils logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/winpython/utils.py b/winpython/utils.py index 97f38e78..b825fbce 100644 --- a/winpython/utils.py +++ b/winpython/utils.py @@ -18,8 +18,6 @@ import re import tarfile import zipfile -import atexit -import winreg # SOURCE_PATTERN defines what an acceptable source package name is SOURCE_PATTERN = r'([a-zA-Z0-9\-\_\.]*)-([0-9\.\_]*[a-z]*[\-]?[0-9]*)(\.zip|\.tar\.gz|\-(py[2-7]*|py[2-7]*\.py[2-7]*)\-none\-any\.whl)' diff --git a/winpython/wheelhouse.py b/winpython/wheelhouse.py index 87c6a9ac..aa095af4 100644 --- a/winpython/wheelhouse.py +++ b/winpython/wheelhouse.py @@ -17,7 +17,7 @@ from email.policy import default from . import utils -from packaging.utils import canonicalize_name +from packaging.utils import canonicalize_name, parse_wheel_filename, parse_sdist_filename # Use tomllib if available (Python 3.11+), otherwise fall back to tomli try: @@ -193,32 +193,64 @@ def get_pylock_wheels(wheelhouse: Path, lockfile: Path, wheelorigin: Optional[Pa print(f"\n\n*** We can't install {filename} ! ***\n\n") def extract_metadata_from_wheel(filepath: Path) -> Optional[Tuple[str, str, str]]: - "get metadata from a wheel package" + "Extract package metadata from a .whl file and validate it matches the filename" + wheel_name = filepath.name + try: + name, version, build, tags = parse_wheel_filename(wheel_name) + filename_name = canonicalize_name(name) + filename_version = str(version) + except Exception as e: + print(f"❌ Could not parse filename: {wheel_name}", e) + return None + with zipfile.ZipFile(filepath, 'r') as z: # Locate *.dist-info/METADATA file inside but not in a vendored directory (flit-core) for name in z.namelist(): if name.endswith(r'.dist-info/METADATA') and name.split("/")[1] == "METADATA": with z.open(name) as meta_file: metadata = BytesParser(policy=default).parse(meta_file) - name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type - version = str(metadata.get('Version', 'unknown')) + meta_name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type + meta_version = str(metadata.get('Version', 'unknown')) summary = utils.sum_up(str(metadata.get('Summary', ''))) - return name, version, summary + # Assert consistency + if meta_name != filename_name or meta_version != filename_version: + print(f"⚠️ Mismatch in {wheel_name}: filename says {filename_name}=={filename_version}, " + f"but METADATA says {meta_name}=={meta_version}") + return None + return meta_name, meta_version , summary return None def extract_metadata_from_sdist(filepath: Path) -> Optional[Tuple[str, str, str]]: "get metadata from a tar.gz or .zip package" open_func = tarfile.open if filepath.suffixes[-2:] == ['.tar', '.gz'] else zipfile.ZipFile + sdist_name = filepath.name + try: + name, version = parse_sdist_filename(sdist_name) + filename_name = canonicalize_name(name) + filename_version = str(version) + except Exception as e: + print(f"❌ Could not parse filename: {sdist_name}", e) + return None + with open_func(filepath, 'r') as archive: namelist = archive.getnames() if isinstance(archive, tarfile.TarFile) else archive.namelist() for name in namelist: if name.endswith('PKG-INFO'): - content = archive.extractfile(name).read() if isinstance(archive, tarfile.TarFile) else archive.open(name).read() - metadata = message_from_bytes(content) - name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type - version = str(metadata.get('Version', 'unknown')) - summary = utils.sum_up(str(metadata.get('Summary', ''))) - return name, version, summary + if isinstance(archive, tarfile.TarFile): + content = archive.extractfile(name) + else: + content = archive.open(name) + if content: + metadata = BytesParser(policy=default).parse(content) + meta_name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type + meta_version = str(metadata.get('Version', 'unknown')) + summary = utils.sum_up(str(metadata.get('Summary', ''))) + # Assert consistency + if meta_name != filename_name or meta_version != filename_version: + print(f"⚠️ Mismatch in {sdist_name}: filename says {filename_name}=={filename_version}, " + f"but METADATA says {meta_name}=={meta_version}") + return None + return meta_name, meta_version, summary return None def list_packages_with_metadata(directory: str) -> List[Tuple[str, str, str]]: diff --git a/winpython/wppm.py b/winpython/wppm.py index a8c92abf..58f5769b 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -14,8 +14,8 @@ import json from pathlib import Path from argparse import ArgumentParser, RawTextHelpFormatter -from winpython import utils, piptree, associate -from winpython import wheelhouse as wh +from . import utils, piptree, associate +from . import wheelhouse as wh from operator import itemgetter # Workaround for installing PyVISA on Windows from source: os.environ["HOME"] = os.environ["USERPROFILE"] From 07207881fefa6fd58c79f1e320751ac28f47c44c Mon Sep 17 00:00:00 2001 From: stonebig Date: Fri, 30 May 2025 11:49:41 +0200 Subject: [PATCH 13/37] change changelog naming for upload readability --- winpython/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winpython/__init__.py b/winpython/__init__.py index f4815474..d2fdc391 100644 --- a/winpython/__init__.py +++ b/winpython/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '16.2.20250529' +__version__ = '16.3.20250530' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' From 6957b34e4bd7bf791c09bdd0588708dec00ddb5a Mon Sep 17 00:00:00 2001 From: stonebig Date: Fri, 30 May 2025 11:50:14 +0200 Subject: [PATCH 14/37] change changelog naming for upload readability --- generate_a_winpython_distro.bat | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/generate_a_winpython_distro.bat b/generate_a_winpython_distro.bat index 51c7dad0..e9c73d9a 100644 --- a/generate_a_winpython_distro.bat +++ b/generate_a_winpython_distro.bat @@ -2,7 +2,7 @@ rem generate_a_winpython_distro.bat: to be launched from a winpython directory, @echo on REM Initialize variables -if "%my_release_level%"=="" set my_release_level=b3 +if "%my_release_level%"=="" set my_release_level=b4 if "%my_create_installer%"=="" set my_create_installer=True rem Set archive directory and log file @@ -159,6 +159,12 @@ set pip_lock_local=%LOCKDIR%pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_loc set req_lock_web=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%.txt set req_lock_local=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_local.txt +set pip_lock_web=%LOCKDIR%pylock.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%%my_release_level%.toml +set pip_lock_local=%LOCKDIR%pylock.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%%my_release_level%_local.toml +set req_lock_web=%LOCKDIR%requirement.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%%my_release_level%.txt +set req_lock_local=%LOCKDIR%requirement.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%_local.txt + + set my_archive_lockfile=%my_archive_dir%\pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_%date:/=-%at_%my_time%.toml set my_archive_lockfile_local=%my_archive_dir%\pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_%date:/=-%at_%my_time%.local.toml set my_changelog_lockfile=%~dp0changelogs\pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%.toml From b5a339a72a8a029e6808474cf830469c3a15a904 Mon Sep 17 00:00:00 2001 From: stonebig Date: Fri, 30 May 2025 11:51:57 +0200 Subject: [PATCH 15/37] move windows menu and register logic to associate.py --- winpython/associate.py | 72 +++++++++++++++++++++++++++++++++++++++--- winpython/utils.py | 63 ------------------------------------ 2 files changed, 67 insertions(+), 68 deletions(-) diff --git a/winpython/associate.py b/winpython/associate.py index 7825ce48..894d8bda 100644 --- a/winpython/associate.py +++ b/winpython/associate.py @@ -10,9 +10,71 @@ from pathlib import Path import importlib.util import winreg -from winpython import utils +from . import utils from argparse import ArgumentParser +def get_special_folder_path(path_name): + """Return special folder path.""" + from win32com.shell import shell, shellcon + try: + csidl = getattr(shellcon, path_name) + return shell.SHGetSpecialFolderPath(0, csidl, False) + except OSError: + print(f"{path_name} is an unknown path ID") + +def get_winpython_start_menu_folder(current=True): + """Return WinPython Start menu shortcuts folder.""" + folder = get_special_folder_path("CSIDL_PROGRAMS") + if not current: + try: + folder = get_special_folder_path("CSIDL_COMMON_PROGRAMS") + except OSError: + pass + return str(Path(folder) / 'WinPython') + +def remove_winpython_start_menu_folder(current=True): + """Remove WinPython Start menu folder -- remove it if it already exists""" + path = get_winpython_start_menu_folder(current=current) + if Path(path).is_dir(): + try: + shutil.rmtree(path, onexc=onerror) + except WindowsError: + print(f"Directory {path} could not be removed", file=sys.stderr) + +def create_winpython_start_menu_folder(current=True): + """Create WinPython Start menu folder.""" + path = get_winpython_start_menu_folder(current=current) + if Path(path).is_dir(): + try: + shutil.rmtree(path, onexc=onerror) + except WindowsError: + print(f"Directory {path} could not be removed", file=sys.stderr) + Path(path).mkdir(parents=True, exist_ok=True) + return path + +def create_shortcut(path, description, filename, arguments="", workdir="", iconpath="", iconindex=0, verbose=True): + """Create Windows shortcut (.lnk file).""" + import pythoncom + from win32com.shell import shell + ilink = pythoncom.CoCreateInstance(shell.CLSID_ShellLink, None, pythoncom.CLSCTX_INPROC_SERVER, shell.IID_IShellLink) + ilink.SetPath(path) + ilink.SetDescription(description) + if arguments: + ilink.SetArguments(arguments) + if workdir: + ilink.SetWorkingDirectory(workdir) + if iconpath or iconindex: + ilink.SetIconLocation(iconpath, iconindex) + # now save it. + ipf = ilink.QueryInterface(pythoncom.IID_IPersistFile) + if not filename.endswith('.lnk'): + filename += '.lnk' + if verbose: + print(f'create menu *{filename}*') + try: + ipf.Save(filename, 0) + except: + print("a fail !") # --- Helper functions for Registry --- @@ -53,7 +115,7 @@ def _has_pywin32(): def _remove_start_menu_folder(target, current=True, has_pywin32=False): "remove menu Folder for target WinPython if pywin32 exists" if has_pywin32: - utils.remove_winpython_start_menu_folder(current=current) + remove_winpython_start_menu_folder(current=current) else: print("Skipping start menu removal as pywin32 package is not installed.") @@ -68,7 +130,7 @@ def _get_shortcut_data(target, current=True, has_pywin32=False): bname, ext = Path(name).stem, Path(name).suffix if ext.lower() == ".exe": # Path for the shortcut file in the start menu folder - shortcut_name = str(Path(utils.create_winpython_start_menu_folder(current=current)) / bname) + '.lnk' + shortcut_name = str(Path(create_winpython_start_menu_folder(current=current)) / bname) + '.lnk' data.append( ( str(Path(wpdir) / name), # Target executable path @@ -180,9 +242,9 @@ def register(target, current=True, reg_type=winreg.REG_SZ, verbose=True): print(f'Creating WinPython menu for all icons in {target.parent}') for path, desc, fname in _get_shortcut_data(target, current=current, has_pywin32=True): try: - utils.create_shortcut(path, desc, fname, verbose=verbose) + create_shortcut(path, desc, fname, verbose=verbose) except Exception as e: - print(f"Error creating shortcut for {desc} at {fname}: {e}", file=sys.stderr) + print(f"Error creating shortcut for {desc} at {fname}: {e}", file=sys.stderr) else: print("Skipping start menu shortcut creation as pywin32 package is needed.") diff --git a/winpython/utils.py b/winpython/utils.py index b825fbce..3e764ca3 100644 --- a/winpython/utils.py +++ b/winpython/utils.py @@ -78,69 +78,6 @@ def sum_up(text: str, max_length: int = 144, stop_at: str = ". ") -> str: return summary[:summary.rfind(stop_at, 0, max_length)] + stop_at.strip() return summary[:max_length].strip() -def get_special_folder_path(path_name): - """Return special folder path.""" - from win32com.shell import shell, shellcon - try: - csidl = getattr(shellcon, path_name) - return shell.SHGetSpecialFolderPath(0, csidl, False) - except OSError: - print(f"{path_name} is an unknown path ID") - -def get_winpython_start_menu_folder(current=True): - """Return WinPython Start menu shortcuts folder.""" - folder = get_special_folder_path("CSIDL_PROGRAMS") - if not current: - try: - folder = get_special_folder_path("CSIDL_COMMON_PROGRAMS") - except OSError: - pass - return str(Path(folder) / 'WinPython') - -def remove_winpython_start_menu_folder(current=True): - """Remove WinPython Start menu folder -- remove it if it already exists""" - path = get_winpython_start_menu_folder(current=current) - if Path(path).is_dir(): - try: - shutil.rmtree(path, onexc=onerror) - except WindowsError: - print(f"Directory {path} could not be removed", file=sys.stderr) - -def create_winpython_start_menu_folder(current=True): - """Create WinPython Start menu folder.""" - path = get_winpython_start_menu_folder(current=current) - if Path(path).is_dir(): - try: - shutil.rmtree(path, onexc=onerror) - except WindowsError: - print(f"Directory {path} could not be removed", file=sys.stderr) - Path(path).mkdir(parents=True, exist_ok=True) - return path - -def create_shortcut(path, description, filename, arguments="", workdir="", iconpath="", iconindex=0, verbose=True): - """Create Windows shortcut (.lnk file).""" - import pythoncom - from win32com.shell import shell - ilink = pythoncom.CoCreateInstance(shell.CLSID_ShellLink, None, pythoncom.CLSCTX_INPROC_SERVER, shell.IID_IShellLink) - ilink.SetPath(path) - ilink.SetDescription(description) - if arguments: - ilink.SetArguments(arguments) - if workdir: - ilink.SetWorkingDirectory(workdir) - if iconpath or iconindex: - ilink.SetIconLocation(iconpath, iconindex) - # now save it. - ipf = ilink.QueryInterface(pythoncom.IID_IPersistFile) - if not filename.endswith('.lnk'): - filename += '.lnk' - if verbose: - print(f'create menu *{filename}*') - try: - ipf.Save(filename, 0) - except: - print("a fail !") - def print_box(text): """Print text in a box""" line0 = "+" + ("-" * (len(text) + 2)) + "+" From f1b88838e45a0aa9371963c8bc154c37667a0958 Mon Sep 17 00:00:00 2001 From: stonebig Date: Fri, 30 May 2025 19:02:00 +0200 Subject: [PATCH 16/37] post-simplification of diff.py after Wheelhouse addition --- diff.py | 190 +++++++++++++++++++++++--------------------------------- 1 file changed, 78 insertions(+), 112 deletions(-) diff --git a/diff.py b/diff.py index 61b4ef80..02317cd0 100644 --- a/diff.py +++ b/diff.py @@ -17,120 +17,94 @@ assert CHANGELOGS_DIR.is_dir() class Package: - # SourceForge Wiki syntax: - PATTERN = r"\[([a-zA-Z\-\:\/\.\_0-9]*)\]\(([^\]\ ]*)\) \| ([^\|]*) \| ([^\|]*)" - # Google Code Wiki syntax: - PATTERN_OLD = r"\[([a-zA-Z\-\:\/\.\_0-9]*) ([^\]\ ]*)\] \| ([^\|]*) \| ([^\|]*)" + PATTERNS = [ + r"\[([\w\-\:\/\.\_]+)\]\(([^)]+)\) \| ([^\|]*) \| ([^\|]*)", # SourceForge + r"\[([\w\-\:\/\.\_]+) ([^\]\ ]+)\] \| ([^\|]*) \| ([^\|]*)" # Google Code + ] - def __init__(self): - self.name = self.version = self.description = self.url = None - - def __str__(self): - return f"{self.name} {self.version}\r\n{self.description}\r\nWebsite: {self.url}" + def __init__(self, text=None): + self.name = self.url = self.version = self.description = None + if text: + self.from_text(text) def from_text(self, text): - match = re.match(self.PATTERN_OLD, text) or re.match(self.PATTERN, text) - if not match: - raise ValueError("Text does not match expected pattern: "+ text) - self.name, self.url, self.version, self.description = match.groups() + for pattern in self.PATTERNS: + match = re.match(pattern, text) + if match: + self.name, self.url, self.version, self.description = match.groups() + return + raise ValueError(f"Unrecognized package line format: {text}") def to_wiki(self): return f" * [{self.name}]({self.url}) {self.version} ({self.description})\r\n" def upgrade_wiki(self, other): - assert self.name.replace("-", "_").lower() == other.name.replace("-", "_").lower() return f" * [{self.name}]({self.url}) {other.version} → {self.version} ({self.description})\r\n" class PackageIndex: - WINPYTHON_PATTERN = r"\#\# WinPython\-*[0-9b-t]* ([0-9\.a-zA-Z]*)" - TOOLS_LINE = "### Tools" - PYTHON_PACKAGES_LINE = "### Python packages" - WHEELHOUSE_PACKAGES_LINE = "### WheelHouse packages" - HEADER_LINE1 = "Name | Version | Description" - HEADER_LINE2 = "-----|---------|------------" + HEADERS = {"tools": "### Tools", "python": "### Python packages", "wheelhouse": "### WheelHouse packages"} + BLANKS = ["Name | Version | Description", "-----|---------|------------", "", "
", "
"] def __init__(self, version, basedir=None, flavor="", architecture=64): self.version = version self.flavor = flavor self.basedir = basedir self.architecture = architecture - self.other_packages = {} - self.python_packages = {} - self.wheelhouse_packages = {} - self.from_file(basedir) - - def from_file(self, basedir): - fname = CHANGELOGS_DIR / f"WinPython{self.flavor}-{self.architecture}bit-{self.version}.md" - if not fname.exists(): - raise FileNotFoundError(f"Changelog file not found: {fname}") - with open(fname, "r", encoding=utils.guess_encoding(fname)[0]) as fdesc: - self.from_text(fdesc.read()) + self.packages = {"tools": {}, "python": {}, "wheelhouse": {}} + self._load_index() - def from_text(self, text): - version = re.match(self.WINPYTHON_PATTERN + self.flavor, text).groups()[0] - assert version == self.version - tools_flag = python_flag = wheelhouse_flag = False + def _load_index(self): + filename = CHANGELOGS_DIR / f"WinPython{self.flavor}-{self.architecture}bit-{self.version}.md" + if not filename.exists(): + raise FileNotFoundError(f"Changelog not found: {filename}") + + with open(filename, "r", encoding=utils.guess_encoding(filename)[0]) as f: + self._parse_index(f.read()) + + def _parse_index(self, text): + current = None for line in text.splitlines(): - if line: - if line == self.TOOLS_LINE: - tools_flag, python_flag, wheelhouse_flag = True, False, False - continue - elif line == self.PYTHON_PACKAGES_LINE: - tools_flag, python_flag, wheelhouse_flag = False, True, False - continue - elif line == self.WHEELHOUSE_PACKAGES_LINE: - tools_flag, python_flag, wheelhouse_flag = False, False, True - continue - elif line in (self.HEADER_LINE1, self.HEADER_LINE2, "
", "
"): - continue - if tools_flag or python_flag or wheelhouse_flag: - package = Package() - package.from_text(line) - if tools_flag: - self.other_packages[package.name] = package - elif python_flag: - self.python_packages[package.name] = package - else: - self.wheelhouse_packages[package.name] = package - -def diff_package_dicts(old_packages, new_packages): + if line in self.HEADERS.values(): + current = [k for k, v in self.HEADERS.items() if v == line][0] + continue + if line.strip() in self.BLANKS: + continue + if current: + pkg = Package(line) + self.packages[current][pkg.name] = pkg + +def compare_packages(old, new): """Return difference between package old and package new""" # wheel replace '-' per '_' in key - old = {k.replace("-", "_").lower(): v for k, v in old_packages.items()} - new = {k.replace("-", "_").lower(): v for k, v in new_packages.items()} - text = "" - - if new_keys := sorted(set(new) - set(old)): - text += "New packages:\r\n\r\n" + "".join(new[k].to_wiki() for k in new_keys) + "\r\n" - - if upgraded := [new[k].upgrade_wiki(old[k]) for k in sorted(set(old) & set(new)) if old[k].version != new[k].version]: - text += "Upgraded packages:\r\n\r\n" + f"{''.join(upgraded)}" + "\r\n" - - if removed_keys := sorted(set(old) - set(new)): - text += "Removed packages:\r\n\r\n" + "".join(old[k].to_wiki() for k in removed_keys) + "\r\n" - return text - -def find_closer_version(version1, basedir=None, flavor="", architecture=64): + def normalize(d): return {k.replace("-", "_").lower(): v for k, v in d.items()} + old, new = normalize(old), normalize(new) + output = "" + + added = [new[k].to_wiki() for k in new if k not in old] + upgraded = [new[k].upgrade_wiki(old[k]) for k in new if k in old and new[k].version != old[k].version] + removed = [old[k].to_wiki() for k in old if k not in new] + + if added: + output += "New packages:\r\n\r\n" + "".join(added) + "\r\n" + if upgraded: + output += "Upgraded packages:\r\n\r\n" + "".join(upgraded) + "\r\n" + if removed: + output += "Removed packages:\r\n\r\n" + "".join(removed) + "\r\n" + return output + +def find_previous_version(target_version, basedir=None, flavor="", architecture=64): """Find version which is the closest to `version`""" - builddir = Path(basedir) / f"bu{flavor}" - pattern = re.compile(rf"WinPython{flavor}-{architecture}bit-([0-9\.]*)\.(txt|md)") - versions = [pattern.match(name).groups()[0] for name in os.listdir(builddir) if pattern.match(name)] - - if version1 not in versions: - raise ValueError(f"Unknown version {version1}") - - version_below = '0.0.0.0' - for v in versions: - if version.parse(version_below) < version.parse(v) and version.parse(v) < version.parse(version1): - version_below = v + build_dir = Path(basedir) / f"bu{flavor}" + pattern = re.compile(rf"WinPython{flavor}-{architecture}bit-([0-9\.]+)\.(txt|md)") + versions = [pattern.match(f).group(1) for f in os.listdir(build_dir) if pattern.match(f)] + versions = [v for v in versions if version.parse(v) < version.parse(target_version)] + return max(versions, key=version.parse, default=target_version) - return version_below if version_below != '0.0.0.0' else version1 +def compare_package_indexes(version2, version1=None, basedir=None, flavor="", flavor1=None, architecture=64): + version1 = version1 or find_previous_version(version2, basedir, flavor, architecture) + flavor1 = flavor1 or flavor -def compare_package_indexes(version2, version1=None, basedir=None, flavor="", flavor1=None,architecture=64): - """Compare two package index Wiki pages""" - version1 = version1 if version1 else find_closer_version(version2, basedir, flavor, architecture) - flavor1 = flavor1 if flavor1 else flavor pi1 = PackageIndex(version1, basedir, flavor1, architecture) pi2 = PackageIndex(version2, basedir, flavor, architecture) @@ -140,37 +114,29 @@ def compare_package_indexes(version2, version1=None, basedir=None, flavor="", fl "
\r\n\r\n" ) - tools_text = diff_package_dicts(pi1.other_packages, pi2.other_packages) - if tools_text: - text += PackageIndex.TOOLS_LINE + "\r\n\r\n" + tools_text + for key in PackageIndex.HEADERS: + diff = compare_packages(pi1.packages[key], pi2.packages[key]) + if diff: + text += f"{PackageIndex.HEADERS[key]}\r\n\r\n{diff}" - py_text = diff_package_dicts(pi1.python_packages, pi2.python_packages) - if py_text: - text += PackageIndex.PYTHON_PACKAGES_LINE + "\r\n\r\n" + py_text + return text + "\r\n
\r\n* * *\r\n" - py_text = diff_package_dicts(pi1.wheelhouse_packages, pi2.wheelhouse_packages) - if py_text: - text += PackageIndex.WHEELHOUSE_PACKAGES_LINE + "\r\n\r\n" + py_text - - text += "\r\n\r\n* * *\r\n" - return text - -def _copy_all_changelogs(version, basedir, flavor="", architecture=64): +def copy_changelogs(version, basedir, flavor="", architecture=64): basever = ".".join(version.split(".")[:2]) - pattern = re.compile(rf"WinPython{flavor}-{architecture}bit-{basever}([0-9\.]*)\.(txt|md)") - for name in os.listdir(CHANGELOGS_DIR): - if pattern.match(name): - shutil.copyfile(CHANGELOGS_DIR / name, Path(basedir) / f"bu{flavor}" / name) + pattern = re.compile(rf"WinPython{flavor}-{architecture}bit-{basever}[0-9\.]*\.(txt|md)") + dest = Path(basedir) / f"bu{flavor}" + for fname in os.listdir(CHANGELOGS_DIR): + if pattern.match(fname): + shutil.copyfile(CHANGELOGS_DIR / fname, dest / fname) def write_changelog(version2, version1=None, basedir=None, flavor="", architecture=64): """Write changelog between version1 and version2 of WinPython""" - _copy_all_changelogs(version2, basedir, flavor, architecture) + copy_changelogs(version2, basedir, flavor, architecture) print("comparing_package_indexes", version2, basedir, flavor, architecture) - changelog_text = compare_package_indexes(version2, version1, basedir, flavor, architecture=architecture) + changelog = compare_package_indexes(version2, version1, basedir, flavor, architecture=architecture) output_file = Path(basedir) / f"bu{flavor}" / f"WinPython{flavor}-{architecture}bit-{version2}_History.md" - - with open(output_file, "w", encoding="utf-8") as fdesc: - fdesc.write(changelog_text) + with open(output_file, "w", encoding="utf-8") as f: + f.write(changelog) # Copy to winpython/changelogs shutil.copyfile(output_file, CHANGELOGS_DIR / output_file.name) From d65b8e2c7b3805c4bc205f0e13d5dd47f7c7c67f Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 31 May 2025 00:11:02 +0200 Subject: [PATCH 17/37] move diff in winpython for BUILD simplification --- make.py | 6 ++--- diff.py => winpython/diff.py | 52 +++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 29 deletions(-) rename diff.py => winpython/diff.py (71%) diff --git a/make.py b/make.py index d66a5213..f4706dc2 100644 --- a/make.py +++ b/make.py @@ -12,9 +12,7 @@ import subprocess import sys from pathlib import Path -from winpython import wppm, utils -# Local import -import diff +from winpython import wppm, utils, diff # Define constant paths for clarity CHANGELOGS_DIRECTORY = Path(__file__).parent / "changelogs" @@ -265,7 +263,7 @@ def build(self, rebuild: bool = True, requirements_files_list=None, winpy_dirnam self._print_action("Writing changelog") shutil.copyfile(output_markdown_filename, str(Path(CHANGELOGS_DIRECTORY) / Path(output_markdown_filename).name)) - diff.write_changelog(self.winpyver2, None, self.base_directory, self.flavor, self.distribution.architecture) + diff.write_changelog(self.winpyver2, None, CHANGELOGS_DIRECTORY, self.flavor, self.distribution.architecture, basedir=self.winpython_directory.parent) def rebuild_winpython_package(source_directory: Path, target_directory: Path, architecture: int = 64, verbose: bool = False): """Rebuilds the winpython package from source using flit.""" diff --git a/diff.py b/winpython/diff.py similarity index 71% rename from diff.py rename to winpython/diff.py index 02317cd0..594cbe36 100644 --- a/diff.py +++ b/winpython/diff.py @@ -11,9 +11,9 @@ import re import shutil from packaging import version -from winpython import utils +from . import utils -CHANGELOGS_DIR = Path(__file__).parent / "changelogs" +CHANGELOGS_DIR = Path(__file__).parent.parent / "changelogs" assert CHANGELOGS_DIR.is_dir() class Package: @@ -45,16 +45,16 @@ class PackageIndex: HEADERS = {"tools": "### Tools", "python": "### Python packages", "wheelhouse": "### WheelHouse packages"} BLANKS = ["Name | Version | Description", "-----|---------|------------", "", "
", "
"] - def __init__(self, version, basedir=None, flavor="", architecture=64): + def __init__(self, version, searchdir=None, flavor="", architecture=64): self.version = version self.flavor = flavor - self.basedir = basedir + self.searchdir = searchdir self.architecture = architecture self.packages = {"tools": {}, "python": {}, "wheelhouse": {}} self._load_index() def _load_index(self): - filename = CHANGELOGS_DIR / f"WinPython{self.flavor}-{self.architecture}bit-{self.version}.md" + filename = self.searchdir / f"WinPython{self.flavor}-{self.architecture}bit-{self.version}.md" if not filename.exists(): raise FileNotFoundError(f"Changelog not found: {filename}") @@ -93,20 +93,20 @@ def normalize(d): return {k.replace("-", "_").lower(): v for k, v in d.items()} output += "Removed packages:\r\n\r\n" + "".join(removed) + "\r\n" return output -def find_previous_version(target_version, basedir=None, flavor="", architecture=64): +def find_previous_version(target_version, searchdir=None, flavor="", architecture=64): """Find version which is the closest to `version`""" - build_dir = Path(basedir) / f"bu{flavor}" + search_dir = Path(searchdir) if searchdir else CHANGELOGS_DIR pattern = re.compile(rf"WinPython{flavor}-{architecture}bit-([0-9\.]+)\.(txt|md)") - versions = [pattern.match(f).group(1) for f in os.listdir(build_dir) if pattern.match(f)] + versions = [pattern.match(f).group(1) for f in os.listdir(search_dir) if pattern.match(f)] versions = [v for v in versions if version.parse(v) < version.parse(target_version)] return max(versions, key=version.parse, default=target_version) -def compare_package_indexes(version2, version1=None, basedir=None, flavor="", flavor1=None, architecture=64): - version1 = version1 or find_previous_version(version2, basedir, flavor, architecture) +def compare_package_indexes(version2, version1=None, searchdir=None, flavor="", flavor1=None, architecture=64): + version1 = version1 or find_previous_version(version2, searchdir, flavor, architecture) flavor1 = flavor1 or flavor - pi1 = PackageIndex(version1, basedir, flavor1, architecture) - pi2 = PackageIndex(version2, basedir, flavor, architecture) + pi1 = PackageIndex(version1, searchdir, flavor1, architecture) + pi2 = PackageIndex(version2, searchdir, flavor, architecture) text = ( f"## History of changes for WinPython-{architecture}bit {version2 + flavor}\r\n\r\n" @@ -121,25 +121,27 @@ def compare_package_indexes(version2, version1=None, basedir=None, flavor="", fl return text + "\r\n\r\n* * *\r\n" -def copy_changelogs(version, basedir, flavor="", architecture=64): +def copy_changelogs(version, searchdir, flavor="", architecture=64, basedir=None): basever = ".".join(version.split(".")[:2]) pattern = re.compile(rf"WinPython{flavor}-{architecture}bit-{basever}[0-9\.]*\.(txt|md)") - dest = Path(basedir) / f"bu{flavor}" - for fname in os.listdir(CHANGELOGS_DIR): + dest = Path(basedir) + for fname in os.listdir(searchdir): if pattern.match(fname): - shutil.copyfile(CHANGELOGS_DIR / fname, dest / fname) + shutil.copyfile(searchdir / fname, dest / fname) -def write_changelog(version2, version1=None, basedir=None, flavor="", architecture=64): +def write_changelog(version2, version1=None, searchdir=None, flavor="", architecture=64, basedir=None): """Write changelog between version1 and version2 of WinPython""" - copy_changelogs(version2, basedir, flavor, architecture) - print("comparing_package_indexes", version2, basedir, flavor, architecture) - changelog = compare_package_indexes(version2, version1, basedir, flavor, architecture=architecture) - output_file = Path(basedir) / f"bu{flavor}" / f"WinPython{flavor}-{architecture}bit-{version2}_History.md" + if basedir: + copy_changelogs(version2, searchdir, flavor, architecture, basedir) + print("comparing_package_indexes", version2, searchdir, flavor, architecture) + changelog = compare_package_indexes(version2, version1, searchdir, flavor, architecture=architecture) + output_file = searchdir / f"WinPython{flavor}-{architecture}bit-{version2}_History.md" with open(output_file, "w", encoding="utf-8") as f: f.write(changelog) - # Copy to winpython/changelogs - shutil.copyfile(output_file, CHANGELOGS_DIR / output_file.name) + # Copy to winpython/changelogs back to basedir + if basedir: + shutil.copyfile(output_file, basedir / output_file.name) if __name__ == "__main__": - print(compare_package_indexes("3.7.4.0", "3.7.2.0", r"C:\WinP\bd37", "Zero", architecture=32)) - write_changelog("3.7.4.0", "3.7.2.0", r"C:\WinP\bd37", "Ps2", architecture=64) + print(compare_package_indexes("3.7.4.0", "3.7.2.0", r"C:\WinP\bd37\budot", "Zero", architecture=32)) + write_changelog("3.7.4.0", "3.7.2.0", r"C:\WinP\bd37\budot", "Ps2", architecture=64) From c465cfb7d6ca301876b3db6b43f10befbbfc9082 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 31 May 2025 14:22:38 +0200 Subject: [PATCH 18/37] markdown reportlog pushed down to wppm --- make.py | 33 +++++++-------------------------- winpython/wppm.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/make.py b/make.py index f4706dc2..ec75c078 100644 --- a/make.py +++ b/make.py @@ -100,32 +100,13 @@ def _get_python_zip_file(self) -> Path: @property def package_index_markdown(self) -> str: """Generates a Markdown formatted package index page.""" - return f"""## WinPython {self.winpyver2 + self.flavor} - -The following packages are included in WinPython-{self.architecture_bits}bit v{self.winpyver2 + self.flavor} {self.release_level}. - -
- -### Tools - -Name | Version | Description ------|---------|------------ -{utils.get_installed_tools_markdown(utils.get_python_executable(self.python_executable_directory))} - -### Python packages - -Name | Version | Description ------|---------|------------ -{self.distribution.get_installed_packages_markdown()} - -### WheelHouse packages - -Name | Version | Description ------|---------|------------ -{self.distribution.get_wheelhouse_packages_markdown()} - -
-""" + return self.distribution.generate_package_index_markdown( + self.python_executable_directory, + self.winpyver2, + self.flavor, + self.architecture_bits, + self.release_level + ) @property def winpython_version_name(self) -> str: diff --git a/winpython/wppm.py b/winpython/wppm.py index 58f5769b..6b967e71 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -87,6 +87,38 @@ def get_wheelhouse_packages_markdown(self) -> str: return "\n".join(package_lines) return "" + def generate_package_index_markdown(self, python_executable_directory: str, winpyver2: str, + flavor: str, architecture_bits: int, release_level: str) -> str: + """Generates a Markdown formatted package index page.""" + from winpython import utils # If needed + + return f"""## WinPython {winpyver2 + flavor} + +The following packages are included in WinPython-{architecture_bits}bit v{winpyver2 + flavor} {release_level}. + +
+ +### Tools + +Name | Version | Description +-----|---------|------------ +{utils.get_installed_tools_markdown(utils.get_python_executable(python_executable_directory))} + +### Python packages + +Name | Version | Description +-----|---------|------------ +{self.get_installed_packages_markdown()} + +### WheelHouse packages + +Name | Version | Description +-----|---------|------------ +{self.get_wheelhouse_packages_markdown()} + +
+""" + def find_package(self, name: str) -> Package | None: """Find installed package by name.""" for pack in self.get_installed_packages(): From cd48d21f07e695ae0759a24bbebde3352bd8557a Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 31 May 2025 15:41:35 +0200 Subject: [PATCH 19/37] wppm can auto-report from winpython import wppm;print(wppm.Distribution().generate_package_index_markdown()) --- winpython/wppm.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/winpython/wppm.py b/winpython/wppm.py index 6b967e71..782a51c5 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -87,14 +87,19 @@ def get_wheelhouse_packages_markdown(self) -> str: return "\n".join(package_lines) return "" - def generate_package_index_markdown(self, python_executable_directory: str, winpyver2: str, - flavor: str, architecture_bits: int, release_level: str) -> str: + def generate_package_index_markdown(self, python_executable_directory: str|None = None, winpyver2: str|None = None, + flavor: str|None = None, architecture_bits: int|None = None, release_level: str|None = None) -> str: """Generates a Markdown formatted package index page.""" - from winpython import utils # If needed + my_ver , my_arch = utils.get_python_infos(python_executable_directory or self.target) + # suppose we suite ourself (method will vary over time) + my_winpyver2 = winpyver2 or os.getenv("WINPYVER2","") + my_winpyver2 = my_winpyver2 if my_winpyver2 != "" else my_ver + my_flavor = flavor or os.getenv("WINPYFLAVOR", "") + my_release_level = release_level or os.getenv("WINPYVER", "").replace(my_winpyver2+my_flavor, "") - return f"""## WinPython {winpyver2 + flavor} + return f"""## WinPython {my_winpyver2 + my_flavor} -The following packages are included in WinPython-{architecture_bits}bit v{winpyver2 + flavor} {release_level}. +The following packages are included in WinPython-{my_arch}bit v{my_winpyver2 + my_flavor} {my_release_level}.
From ec7a132041fd46ede4fe3f8f58a44945a8153c30 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sun, 1 Jun 2025 11:05:16 +0200 Subject: [PATCH 20/37] simplify markdown generation --- winpython/utils.py | 15 +++++------- winpython/wppm.py | 57 +++++++++++++++++----------------------------- 2 files changed, 27 insertions(+), 45 deletions(-) diff --git a/winpython/utils.py b/winpython/utils.py index 3e764ca3..2d549f20 100644 --- a/winpython/utils.py +++ b/winpython/utils.py @@ -11,9 +11,7 @@ import stat import shutil import locale -import tempfile import subprocess -import configparser as cp from pathlib import Path import re import tarfile @@ -41,25 +39,24 @@ def get_site_packages_path(path=None): pypy_site_packages = base_dir / 'site-packages' # For PyPy return str(pypy_site_packages if pypy_site_packages.is_dir() else site_packages) -def get_installed_tools_markdown(path=None)-> str: +def get_installed_tools(path=None)-> str: """Generates Markdown for installed tools section in package index.""" tool_lines = [] python_exe = Path(get_python_executable(path)) version = exec_shell_cmd(f'powershell (Get-Item {python_exe}).VersionInfo.FileVersion', python_exe.parent).splitlines()[0] - tool_lines.append(f"[Python](http://www.python.org/) | {version} | Python programming language with standard library") + tool_lines.append(("Python" ,f"http://www.python.org/", version, "Python programming language with standard library")) if (node_exe := python_exe.parent.parent / "n" / "node.exe").exists(): version = exec_shell_cmd(f'powershell (Get-Item {node_exe}).VersionInfo.FileVersion', node_exe.parent).splitlines()[0] - tool_lines.append(f"[Nodejs](https://nodejs.org) | {version} | a JavaScript runtime built on Chrome's V8 JavaScript engine") + tool_lines.append("Nodejs", "https://nodejs.org", version, "a JavaScript runtime built on Chrome's V8 JavaScript engine") if (pandoc_exe := python_exe.parent.parent / "t" / "pandoc.exe").exists(): version = exec_shell_cmd("pandoc -v", pandoc_exe.parent).splitlines()[0].split(" ")[-1] - tool_lines.append(f"[Pandoc](https://pandoc.org) | {version} | an universal document converter") + tool_lines.append("Pandoc", "https://pandoc.org", version, "an universal document converter") if (vscode_exe := python_exe.parent.parent / "t" / "VSCode" / "Code.exe").exists(): version = exec_shell_cmd(f'powershell (Get-Item {vscode_exe}).VersionInfo.FileVersion', vscode_exe.parent).splitlines()[0] - tool_lines.append(f"[VSCode](https://code.visualstudio.com) | {version} | a source-code editor developed by Microsoft") - return "\n".join(tool_lines) - + tool_lines.append("VSCode","https://code.visualstudio.com", version, "a source-code editor developed by Microsoft") + return tool_lines def onerror(function, path, excinfo): """Error handler for `shutil.rmtree`.""" diff --git a/winpython/wppm.py b/winpython/wppm.py index 782a51c5..8b733b68 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -69,24 +69,16 @@ def get_installed_packages(self, update: bool = False) -> list[Package]: pip_list = self.pip.pip_list(full=True) return [Package(f"{i[0].replace('-', '_').lower()}-{i[1]}-py3-none-any.whl", suggested_summary=i[2]) for i in pip_list] - def get_installed_packages_markdown(self) -> str: - """Generates Markdown for installed packages section in package index.""" - package_lines = [ - f"[{pkg.name}]({pkg.url}) | {pkg.version} | {pkg.description}" - for pkg in sorted(self.get_installed_packages(), key=lambda p: p.name.lower()) - ] - return "\n".join(package_lines) - - def get_wheelhouse_packages_markdown(self) -> str: - wheeldir = self.wheelhouse / 'included.wheels' - if wheeldir.is_dir(): - package_lines = [ - f"[{name}](https://pypi.org/project/{name}) | {version} | {summary}" - for name, version, summary in sorted(wh.list_packages_with_metadata(str(wheeldir)), key=itemgetter(0 , 1)) # lambda p: p[0].lower()) - ] - return "\n".join(package_lines) - return "" - + def render_markdown_for_list(self, title, items): + """Generates a Markdown section; name, url, version, summary""" + md = f"### {title}\n\n" + md += "Name | Version | Description\n" + md += "-----|---------|------------\n" + for name, url, version, summary in sorted(items, key=lambda p: (p[0].lower(), p[2])): + md += f"[{name}]({url}) | {version} | {summary} \n" + md += "\n" + return md + def generate_package_index_markdown(self, python_executable_directory: str|None = None, winpyver2: str|None = None, flavor: str|None = None, architecture_bits: int|None = None, release_level: str|None = None) -> str: """Generates a Markdown formatted package index page.""" @@ -97,30 +89,23 @@ def generate_package_index_markdown(self, python_executable_directory: str|None my_flavor = flavor or os.getenv("WINPYFLAVOR", "") my_release_level = release_level or os.getenv("WINPYVER", "").replace(my_winpyver2+my_flavor, "") + tools_list = utils.get_installed_tools(utils.get_python_executable(python_executable_directory)) + package_list = [(pkg.name, pkg.url, pkg.version, pkg.description) for pkg in self.get_installed_packages()] + wheelhouse_list = [] + wheeldir = self.wheelhouse / 'included.wheels' + if wheeldir.is_dir(): + wheelhouse_list = [(name, f"https://pypi.org/project/{name}", version, summary) + for name, version, summary in wh.list_packages_with_metadata(str(wheeldir)) ] + return f"""## WinPython {my_winpyver2 + my_flavor} The following packages are included in WinPython-{my_arch}bit v{my_winpyver2 + my_flavor} {my_release_level}.
-### Tools - -Name | Version | Description ------|---------|------------ -{utils.get_installed_tools_markdown(utils.get_python_executable(python_executable_directory))} - -### Python packages - -Name | Version | Description ------|---------|------------ -{self.get_installed_packages_markdown()} - -### WheelHouse packages - -Name | Version | Description ------|---------|------------ -{self.get_wheelhouse_packages_markdown()} - +{self.render_markdown_for_list("Tools", tools_list)} +{self.render_markdown_for_list("Python packages", package_list)} +{self.render_markdown_for_list("WheelHouse packages", wheelhouse_list)}
""" From eb7548fd244c677a440cbcc0337fa3e59f9835bf Mon Sep 17 00:00:00 2001 From: stonebig Date: Sun, 1 Jun 2025 20:11:13 +0200 Subject: [PATCH 21/37] wppm -ls -ws ... works navigating into a WheelHouse.. --- winpython/packagemetadata.py | 109 +++++++++++++++++++++++++++++++++++ winpython/piptree.py | 13 +++-- winpython/wppm.py | 8 +-- 3 files changed, 121 insertions(+), 9 deletions(-) create mode 100644 winpython/packagemetadata.py diff --git a/winpython/packagemetadata.py b/winpython/packagemetadata.py new file mode 100644 index 00000000..469925d6 --- /dev/null +++ b/winpython/packagemetadata.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +packagemetadata.py - get metadata from designated place +""" +import os +import re +import tarfile +import zipfile +import sys +from pathlib import Path +from collections import defaultdict +import shutil +import subprocess +from typing import Dict, List, Optional, Tuple +from . import utils + +from packaging.utils import canonicalize_name, parse_wheel_filename, parse_sdist_filename +# --- Abstract metadata accessor --- + +class PackageMetadata: + """A minimal abstraction for package metadata.""" + def __init__(self, name, version, requires, summary, description, metadata): + self.name = name + self.version = version + self.requires = requires # List[str] of dependencies + self.summary = summary + self.description = description + self.metadata = metadata + +def get_installed_metadata(path = None) -> List[PackageMetadata]: + # Use importlib.metadata or pkg_resources + import importlib.metadata + pkgs = [] + distro = importlib.metadata.distributions(path = path) if path else importlib.metadata.distributions() + for dist in distro: + name = dist.metadata['Name'] + version = dist.version + summary = dist.metadata.get("Summary", ""), + description = dist.metadata.get("Description", ""), + requires = dist.requires or [] + metadata = dist.metadata + pkgs.append(PackageMetadata(name, version, requires, summary, description, metadata)) + return pkgs + +def get_directory_metadata(directory: str) -> List[PackageMetadata]: + # For each .whl/.tar.gz file in directory, extract metadata + pkgs = [] + for fname in os.listdir(directory): + if fname.endswith('.whl'): + # Extract METADATA from wheel + meta = extract_metadata_from_wheel(os.path.join(directory, fname)) + pkgs.append(meta) + elif fname.endswith('.tar.gz'): + # Extract PKG-INFO from sdist + meta = extract_metadata_from_sdist(os.path.join(directory, fname)) + pkgs.append(meta) + return pkgs + +def extract_metadata_from_wheel(path: str) -> PackageMetadata: + import zipfile + with zipfile.ZipFile(path) as zf: + for name in zf.namelist(): + if name.endswith(r'.dist-info/METADATA') and name.split("/")[1] == "METADATA": + with zf.open(name) as f: + # Parse metadata (simple parsing for Name, Version, Requires-Dist) + return parse_metadata_file(f.read().decode()) + raise ValueError(f"No METADATA found in {path}") + +def extract_metadata_from_sdist(path: str) -> PackageMetadata: + import tarfile + with tarfile.open(path, "r:gz") as tf: + for member in tf.getmembers(): + if member.name.endswith('PKG-INFO'): + f = tf.extractfile(member) + return parse_metadata_file(f.read().decode()) + raise ValueError(f"No PKG-INFO found in {path}") + +def parse_metadata_file(txt: str) -> PackageMetadata: + name = version = summary = description = "" + requires = [] + for line in txt.splitlines(): + if line.startswith('Name: '): + name = line[6:].strip() + elif line.startswith('Version: '): + version = line[9:].strip() + elif line.startswith('Summary: '): + summary = description = line[9:].strip() + elif line.startswith('Requires-Dist: '): + requires.append(line[14:].strip()) + return PackageMetadata(name, version, requires, summary, description, {'Name': name, "Summary": summary, "Description": description}) + +# --- Main dependency tree logic --- + +def build_dependency_tree(pkgs: List[PackageMetadata]): + # Existing logic, but using our PackageMetadata objects + pass + +def main(): + if len(sys.argv) > 1: + # Directory mode + directory = sys.argv[1] + pkgs = get_directory_metadata(directory) + else: + # Installed packages mode + pkgs = get_installed_metadata() + build_dependency_tree(pkgs) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/winpython/piptree.py b/winpython/piptree.py index 1cc1d1ab..e46bd465 100644 --- a/winpython/piptree.py +++ b/winpython/piptree.py @@ -18,6 +18,7 @@ from importlib.metadata import Distribution, distributions from pathlib import Path from . import utils +from . import packagemetadata as pm logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -29,7 +30,7 @@ class PipDataError(Exception): class PipData: """Manages package metadata and dependency relationships in a Python environment.""" - def __init__(self, target: Optional[str] = None): + def __init__(self, target: Optional[str] = None, wheelhouse = None): """ Initialize the PipData instance. @@ -39,7 +40,7 @@ def __init__(self, target: Optional[str] = None): self.raw: Dict[str, Dict] = {} self.environment = self._get_environment() try: - packages = self._get_packages(target or sys.executable) + packages = self._get_packages(target or sys.executable, wheelhouse) self._process_packages(packages) self._populate_reverse_dependencies() except Exception as e: @@ -67,12 +68,14 @@ def _get_environment(self) -> Dict[str, str]: "sys_platform": sys.platform, } - def _get_packages(self, search_path: str) -> List[Distribution]: + def _get_packages(self, search_path: str, wheelhouse) -> List[Distribution]: """Retrieve installed packages from the specified path.""" + if wheelhouse: + return pm.get_directory_metadata(wheelhouse) if sys.executable == search_path: - return Distribution.discover() + return pm.get_installed_metadata() #Distribution.discover() else: - return distributions(path=[str(Path(search_path).parent / 'lib' / 'site-packages')]) + return pm.get_installed_metadata(path=[str(Path(search_path).parent / 'lib' / 'site-packages')]) #distributions(path=[str(Path(search_path).parent / 'lib' / 'site-packages')]) def _process_packages(self, packages: List[Distribution]) -> None: """Process packages metadata and store them in the distro dictionary.""" diff --git a/winpython/wppm.py b/winpython/wppm.py index 8b733b68..5b83988c 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -295,17 +295,17 @@ def main(test=False): if args.registerWinPython and args.unregisterWinPython: raise RuntimeError("Incompatible arguments: --install and --uninstall") if args.pipdown: - pip = piptree.PipData(targetpython) + pip = piptree.PipData(targetpython, args.wheelsource) pack, extra, *other = (args.fname + "[").replace("]", "[").split("[") print(pip.down(pack, extra, args.levels, verbose=args.verbose)) sys.exit() elif args.pipup: - pip = piptree.PipData(targetpython) + pip = piptree.PipData(targetpython, args.wheelsource) pack, extra, *other = (args.fname + "[").replace("]", "[").split("[") print(pip.up(pack, extra, args.levels, verbose=args.verbose)) sys.exit() elif args.list: - pip = piptree.PipData(targetpython) + pip = piptree.PipData(targetpython, args.wheelsource) todo = [l for l in pip.pip_list(full=True) if bool(re.search(args.fname, l[0]))] titles = [['Package', 'Version', 'Summary'], ['_' * max(x, 6) for x in utils.columns_width(todo)]] listed = utils.formatted_list(titles + todo, max_width=70) @@ -313,7 +313,7 @@ def main(test=False): print(*p) sys.exit() elif args.all: - pip = piptree.PipData(targetpython) + pip = piptree.PipData(targetpython, args.wheelsource) todo = [l for l in pip.pip_list(full=True) if bool(re.search(args.fname, l[0]))] for l in todo: # print(pip.distro[l[0]]) From c017a18357ed98a6274b58577a8bd38c342f9036 Mon Sep 17 00:00:00 2001 From: stonebig Date: Tue, 3 Jun 2025 21:23:30 +0200 Subject: [PATCH 22/37] less spaces in the raw changelog --- winpython/diff.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/winpython/diff.py b/winpython/diff.py index 594cbe36..a8e52e1d 100644 --- a/winpython/diff.py +++ b/winpython/diff.py @@ -36,10 +36,10 @@ def from_text(self, text): raise ValueError(f"Unrecognized package line format: {text}") def to_wiki(self): - return f" * [{self.name}]({self.url}) {self.version} ({self.description})\r\n" + return f" * [{self.name}]({self.url}) {self.version} ({self.description})\n" def upgrade_wiki(self, other): - return f" * [{self.name}]({self.url}) {other.version} → {self.version} ({self.description})\r\n" + return f" * [{self.name}]({self.url}) {other.version} → {self.version} ({self.description})\n" class PackageIndex: HEADERS = {"tools": "### Tools", "python": "### Python packages", "wheelhouse": "### WheelHouse packages"} @@ -86,11 +86,11 @@ def normalize(d): return {k.replace("-", "_").lower(): v for k, v in d.items()} removed = [old[k].to_wiki() for k in old if k not in new] if added: - output += "New packages:\r\n\r\n" + "".join(added) + "\r\n" + output += "New packages:\n\n" + "".join(added) + "\n\n" if upgraded: - output += "Upgraded packages:\r\n\r\n" + "".join(upgraded) + "\r\n" + output += "Upgraded packages:\n\n" + "".join(upgraded) + "\n\n" if removed: - output += "Removed packages:\r\n\r\n" + "".join(removed) + "\r\n" + output += "Removed packages:\n\n" + "".join(removed) + "\n\n" return output def find_previous_version(target_version, searchdir=None, flavor="", architecture=64): @@ -110,16 +110,16 @@ def compare_package_indexes(version2, version1=None, searchdir=None, flavor="", text = ( f"## History of changes for WinPython-{architecture}bit {version2 + flavor}\r\n\r\n" - f"The following changes were made to WinPython-{architecture}bit distribution since version {version1 + flavor1}.\r\n\r\n" - "
\r\n\r\n" + f"The following changes were made to WinPython-{architecture}bit distribution since version {version1 + flavor1}.\n\n\n" + "
\n\n" ) for key in PackageIndex.HEADERS: diff = compare_packages(pi1.packages[key], pi2.packages[key]) if diff: - text += f"{PackageIndex.HEADERS[key]}\r\n\r\n{diff}" + text += f"\n{PackageIndex.HEADERS[key]}\n\n{diff}" - return text + "\r\n
\r\n* * *\r\n" + return text + "\n
\n\n* * *\n" def copy_changelogs(version, searchdir, flavor="", architecture=64, basedir=None): basever = ".".join(version.split(".")[:2]) From bdb7288f9d1b9ea32c3a735310bf52b36a48ec6e Mon Sep 17 00:00:00 2001 From: stonebig Date: Tue, 3 Jun 2025 21:26:09 +0200 Subject: [PATCH 23/37] -md markdown and -ws . options for comfort --- winpython/__init__.py | 2 +- winpython/packagemetadata.py | 14 ++----- winpython/wheelhouse.py | 81 ++---------------------------------- winpython/wppm.py | 24 +++++++---- 4 files changed, 24 insertions(+), 97 deletions(-) diff --git a/winpython/__init__.py b/winpython/__init__.py index d2fdc391..468eed57 100644 --- a/winpython/__init__.py +++ b/winpython/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '16.3.20250530' +__version__ = '16.4.20250603' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' diff --git a/winpython/packagemetadata.py b/winpython/packagemetadata.py index 469925d6..4e2a4989 100644 --- a/winpython/packagemetadata.py +++ b/winpython/packagemetadata.py @@ -14,7 +14,6 @@ from typing import Dict, List, Optional, Tuple from . import utils -from packaging.utils import canonicalize_name, parse_wheel_filename, parse_sdist_filename # --- Abstract metadata accessor --- class PackageMetadata: @@ -64,7 +63,7 @@ def extract_metadata_from_wheel(path: str) -> PackageMetadata: with zf.open(name) as f: # Parse metadata (simple parsing for Name, Version, Requires-Dist) return parse_metadata_file(f.read().decode()) - raise ValueError(f"No METADATA found in {path}") + raise ValueError(f"No METADATA found in {path}") def extract_metadata_from_sdist(path: str) -> PackageMetadata: import tarfile @@ -78,6 +77,8 @@ def extract_metadata_from_sdist(path: str) -> PackageMetadata: def parse_metadata_file(txt: str) -> PackageMetadata: name = version = summary = description = "" requires = [] + description_lines = [] + in_description = False for line in txt.splitlines(): if line.startswith('Name: '): name = line[6:].strip() @@ -86,15 +87,9 @@ def parse_metadata_file(txt: str) -> PackageMetadata: elif line.startswith('Summary: '): summary = description = line[9:].strip() elif line.startswith('Requires-Dist: '): - requires.append(line[14:].strip()) + requires.append(line[14:].strip()) return PackageMetadata(name, version, requires, summary, description, {'Name': name, "Summary": summary, "Description": description}) -# --- Main dependency tree logic --- - -def build_dependency_tree(pkgs: List[PackageMetadata]): - # Existing logic, but using our PackageMetadata objects - pass - def main(): if len(sys.argv) > 1: # Directory mode @@ -103,7 +98,6 @@ def main(): else: # Installed packages mode pkgs = get_installed_metadata() - build_dependency_tree(pkgs) if __name__ == "__main__": main() \ No newline at end of file diff --git a/winpython/wheelhouse.py b/winpython/wheelhouse.py index aa095af4..f500dc31 100644 --- a/winpython/wheelhouse.py +++ b/winpython/wheelhouse.py @@ -12,9 +12,7 @@ import shutil import subprocess from typing import Dict, List, Optional, Tuple -from email import message_from_bytes -from email.parser import BytesParser -from email.policy import default +from . import packagemetadata as pm from . import utils from packaging.utils import canonicalize_name, parse_wheel_filename, parse_sdist_filename @@ -192,83 +190,10 @@ def get_pylock_wheels(wheelhouse: Path, lockfile: Path, wheelorigin: Optional[Pa else: print(f"\n\n*** We can't install {filename} ! ***\n\n") -def extract_metadata_from_wheel(filepath: Path) -> Optional[Tuple[str, str, str]]: - "Extract package metadata from a .whl file and validate it matches the filename" - wheel_name = filepath.name - try: - name, version, build, tags = parse_wheel_filename(wheel_name) - filename_name = canonicalize_name(name) - filename_version = str(version) - except Exception as e: - print(f"❌ Could not parse filename: {wheel_name}", e) - return None - - with zipfile.ZipFile(filepath, 'r') as z: - # Locate *.dist-info/METADATA file inside but not in a vendored directory (flit-core) - for name in z.namelist(): - if name.endswith(r'.dist-info/METADATA') and name.split("/")[1] == "METADATA": - with z.open(name) as meta_file: - metadata = BytesParser(policy=default).parse(meta_file) - meta_name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type - meta_version = str(metadata.get('Version', 'unknown')) - summary = utils.sum_up(str(metadata.get('Summary', ''))) - # Assert consistency - if meta_name != filename_name or meta_version != filename_version: - print(f"⚠️ Mismatch in {wheel_name}: filename says {filename_name}=={filename_version}, " - f"but METADATA says {meta_name}=={meta_version}") - return None - return meta_name, meta_version , summary - return None - -def extract_metadata_from_sdist(filepath: Path) -> Optional[Tuple[str, str, str]]: - "get metadata from a tar.gz or .zip package" - open_func = tarfile.open if filepath.suffixes[-2:] == ['.tar', '.gz'] else zipfile.ZipFile - sdist_name = filepath.name - try: - name, version = parse_sdist_filename(sdist_name) - filename_name = canonicalize_name(name) - filename_version = str(version) - except Exception as e: - print(f"❌ Could not parse filename: {sdist_name}", e) - return None - - with open_func(filepath, 'r') as archive: - namelist = archive.getnames() if isinstance(archive, tarfile.TarFile) else archive.namelist() - for name in namelist: - if name.endswith('PKG-INFO'): - if isinstance(archive, tarfile.TarFile): - content = archive.extractfile(name) - else: - content = archive.open(name) - if content: - metadata = BytesParser(policy=default).parse(content) - meta_name = canonicalize_name(str(metadata.get('Name', 'unknown'))) # Avoid Head type - meta_version = str(metadata.get('Version', 'unknown')) - summary = utils.sum_up(str(metadata.get('Summary', ''))) - # Assert consistency - if meta_name != filename_name or meta_version != filename_version: - print(f"⚠️ Mismatch in {sdist_name}: filename says {filename_name}=={filename_version}, " - f"but METADATA says {meta_name}=={meta_version}") - return None - return meta_name, meta_version, summary - return None - def list_packages_with_metadata(directory: str) -> List[Tuple[str, str, str]]: "get metadata from a Wheelhouse directory" - results = [] - for file in os.listdir(directory): - path = Path(directory) / file - try: - if path.suffix == '.whl': - meta = extract_metadata_from_wheel(path) - elif path.suffix == '.zip' or path.name.endswith('.tar.gz'): - meta = extract_metadata_from_sdist(path) - else: - continue - if meta: - results.append(meta) - except OSError: #Exception as e: # need to see it - print(f"Skipping {file}: {e}") + packages = pm.get_directory_metadata(directory) + results = [ (p.name, p.version, p.summary) for p in packages] return results def main() -> None: diff --git a/winpython/wppm.py b/winpython/wppm.py index 5b83988c..01ed2c83 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -274,14 +274,15 @@ def main(test=False): # parser.add_argument( "--unregister_forall", action="store_true", help="un-Register distribution for all users") parser.add_argument("--fix", action="store_true", help="make WinPython fix") parser.add_argument("--movable", action="store_true", help="make WinPython movable") - parser.add_argument("-ws", dest="wheelsource", default=None, type=str, help="location to search wheels: wppm pylock.toml -ws source_of_wheels") - parser.add_argument("-wd", dest="wheeldrain" , default=None, type=str, help="location of found wheels: wppm pylock.toml -wd destination_of_wheels") - parser.add_argument("-ls", "--list", action="store_true", help="list installed packages matching the given [optional] package expression: wppm -ls, wppm -ls pand") - parser.add_argument("-lsa", dest="all", action="store_true",help=f"list details of package names matching given regular expression: wppm -lsa pandas -l1") - parser.add_argument("-p",dest="pipdown",action="store_true",help="show Package dependencies of the given package[option]: wppm -p pandas[test]") - parser.add_argument("-r", dest="pipup", action="store_true", help=f"show Reverse dependancies of the given package[option]: wppm -r pytest[test]") - parser.add_argument("-l", "--levels", type=int, default=2, help="show 'LEVELS' levels of dependencies (with -p, -r), default is 2: wppm -p pandas -l1") - parser.add_argument("-t", "--target", default=sys.prefix, help=f'path to target Python distribution (default: "{sys.prefix}")') + parser.add_argument("-ws", dest="wheelsource", default=None, type=str, help="wheels location, '.' = WheelHouse): wppm pylock.toml -ws source_of_wheels, wppm -ls -ws .") + parser.add_argument("-wd", dest="wheeldrain" , default=None, type=str, help="wheels destination: wppm pylock.toml -wd destination_of_wheels") + parser.add_argument("-ls", "--list", action="store_true", help="list installed packages matching [optional] expression: wppm -ls, wppm -ls pand") + parser.add_argument("-lsa", dest="all", action="store_true",help=f"list details of packages matching [optional] expression: wppm -lsa pandas -l1") + parser.add_argument("-md", dest="markdown", action="store_true",help=f"markdown summary if the installation") + parser.add_argument("-p",dest="pipdown",action="store_true",help="show Package dependencies of the given package[option], [.]=all: wppm -p pandas[.]") + parser.add_argument("-r", dest="pipup", action="store_true", help=f"show Reverse wppmdependancies of the given package[option]: wppm -r pytest[test]") + parser.add_argument("-l", dest="levels", type=int, default=2, help="show 'LEVELS' levels of dependencies (with -p, -r), default is 2: wppm -p pandas -l1") + parser.add_argument("-t", dest="target", default=sys.prefix, help=f'path to target Python distribution (default: "{sys.prefix}")') parser.add_argument("-i", "--install", action="store_true", help="install a given package wheel or pylock file (use pip for more features)") parser.add_argument("-u", "--uninstall", action="store_true", help="uninstall package (use pip for more features)") @@ -290,6 +291,10 @@ def main(test=False): targetpython = None if args.target and args.target != sys.prefix: targetpython = args.target if args.target.lower().endswith('.exe') else str(Path(args.target) / 'python.exe') + if args.wheelsource == ".": # play in default WheelHouse + if utils.is_python_distribution(args.target): + dist = Distribution(args.target) + args.wheelsource = dist.wheelhouse / 'included.wheels' if args.install and args.uninstall: raise RuntimeError("Incompatible arguments: --install and --uninstall") if args.registerWinPython and args.unregisterWinPython: @@ -360,6 +365,9 @@ def main(test=False): if args.movable: p = subprocess.Popen(["start", "cmd", "/k",dist.python_exe, "-c" , cmd_mov], shell = True, cwd=dist.target) sys.exit() + if args.markdown: + print(dist.generate_package_index_markdown()) + sys.exit() if not args.install and not args.uninstall and args.fname.endswith(".toml"): args.install = True # for Drag & Drop of .toml (and not wheel) if args.fname == "" or (not args.install and not args.uninstall): From d4bf8a9a3bdd4901c9f295e223a8710f7533236f Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 5 Jun 2025 21:25:17 +0200 Subject: [PATCH 24/37] move hash.py in winpython --- hash_launch.bat | 6 ++++-- hash.py => winpython/hash.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) rename hash.py => winpython/hash.py (96%) diff --git a/hash_launch.bat b/hash_launch.bat index 714925a4..623765d7 100644 --- a/hash_launch.bat +++ b/hash_launch.bat @@ -1,8 +1,10 @@ -call C:\WPy64-3890\scripts\env.bat +call C:\WinPdev\WPy64-310111\scripts\env.bat cd %~dp0 rem echo %date% %time%>>gdc_counting.txt -python hash.py %* >>hash_counting_%date:/=_%.txt +python -c "import sys;from winpython import hash; hash.print_hashes(sys.argv[1:])" %* >>hash_counting_%date:/=_%.txt +rem python hash.py %* >>hash_counting_%date:/=_%.txt start notepad.exe hash_counting_%date:/=_%.txt + \ No newline at end of file diff --git a/hash.py b/winpython/hash.py similarity index 96% rename from hash.py rename to winpython/hash.py index d16bff0a..e0bdd46a 100644 --- a/hash.py +++ b/winpython/hash.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# -# WinPython hash.py script +""" +hash.py: compute hash of given files into a markdown output +""" # Copyright © 2014-2025+ The Winpython development team https://github.com/winpython/ # Licensed under the terms of the MIT License # (see winpython/__init__.py for details) From b84ef841776325ef19ac186183ea93a174f6b4cb Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 5 Jun 2025 23:48:48 +0200 Subject: [PATCH 25/37] taking gloves, as bitten per this dark art --- winpython/packagemetadata.py | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/winpython/packagemetadata.py b/winpython/packagemetadata.py index 4e2a4989..a51e8331 100644 --- a/winpython/packagemetadata.py +++ b/winpython/packagemetadata.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ packagemetadata.py - get metadata from designated place """ @@ -8,12 +8,11 @@ import zipfile import sys from pathlib import Path -from collections import defaultdict -import shutil -import subprocess from typing import Dict, List, Optional, Tuple from . import utils - +import importlib.metadata +import email +from packaging.utils import canonicalize_name # --- Abstract metadata accessor --- class PackageMetadata: @@ -28,11 +27,10 @@ def __init__(self, name, version, requires, summary, description, metadata): def get_installed_metadata(path = None) -> List[PackageMetadata]: # Use importlib.metadata or pkg_resources - import importlib.metadata pkgs = [] distro = importlib.metadata.distributions(path = path) if path else importlib.metadata.distributions() for dist in distro: - name = dist.metadata['Name'] + name = canonicalize_name(dist.metadata['Name']) version = dist.version summary = dist.metadata.get("Summary", ""), description = dist.metadata.get("Description", ""), @@ -56,7 +54,6 @@ def get_directory_metadata(directory: str) -> List[PackageMetadata]: return pkgs def extract_metadata_from_wheel(path: str) -> PackageMetadata: - import zipfile with zipfile.ZipFile(path) as zf: for name in zf.namelist(): if name.endswith(r'.dist-info/METADATA') and name.split("/")[1] == "METADATA": @@ -66,7 +63,6 @@ def extract_metadata_from_wheel(path: str) -> PackageMetadata: raise ValueError(f"No METADATA found in {path}") def extract_metadata_from_sdist(path: str) -> PackageMetadata: - import tarfile with tarfile.open(path, "r:gz") as tf: for member in tf.getmembers(): if member.name.endswith('PKG-INFO'): @@ -75,20 +71,13 @@ def extract_metadata_from_sdist(path: str) -> PackageMetadata: raise ValueError(f"No PKG-INFO found in {path}") def parse_metadata_file(txt: str) -> PackageMetadata: - name = version = summary = description = "" - requires = [] - description_lines = [] - in_description = False - for line in txt.splitlines(): - if line.startswith('Name: '): - name = line[6:].strip() - elif line.startswith('Version: '): - version = line[9:].strip() - elif line.startswith('Summary: '): - summary = description = line[9:].strip() - elif line.startswith('Requires-Dist: '): - requires.append(line[14:].strip()) - return PackageMetadata(name, version, requires, summary, description, {'Name': name, "Summary": summary, "Description": description}) + meta = email.message_from_string(txt) + name = canonicalize_name(meta.get('Name', '')) + version = meta.get('Version', '') + summary = meta.get('Summary', '') + description = meta.get('Description', '') + requires = meta.get_all('Requires-Dist') or [] + return PackageMetadata(name, version, requires, summary, description, dict(meta.items())) def main(): if len(sys.argv) > 1: From effb981f4e8250f06f1d5648bd0b3c8205f92a9a Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 7 Jun 2025 11:37:57 +0200 Subject: [PATCH 26/37] slight refresh --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index a5f445e9..9a1e05cf 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ This is the `winpython` Python package, not the distribution itself. It includes two main features: WinPython Package Manager (WPPM) - a complementary tool to navigate provided package list or register WinPython. + a complementary tool to navigate provided packages, install packages from included Wheelhouse, or register WinPython. pip is the recommanded way to add or remove packages WinPython build toolchain @@ -29,7 +29,7 @@ WinPython build toolchain Dependencies ------------ -* Python3 >= 3.9 +* Python3 >= 3.10 Requirements @@ -58,6 +58,6 @@ More informations * Downloads: https://sourceforge.net/projects/winpython/files/ or https://github.com/winpython/winpython/releases -* Development, bug reports and feature requests: https://github.com/winpython/winpython +* Development, bug reports, discussions and feature requests: https://github.com/winpython/winpython * Discussions: https://groups.google.com/group/winpython From 42f8e52f1b82d8b89ed2a634be829cdec0778327 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 7 Jun 2025 12:02:09 +0200 Subject: [PATCH 27/37] bug fix after changing markdown generation --- winpython/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/winpython/utils.py b/winpython/utils.py index 2d549f20..0c83cb80 100644 --- a/winpython/utils.py +++ b/winpython/utils.py @@ -47,15 +47,15 @@ def get_installed_tools(path=None)-> str: tool_lines.append(("Python" ,f"http://www.python.org/", version, "Python programming language with standard library")) if (node_exe := python_exe.parent.parent / "n" / "node.exe").exists(): version = exec_shell_cmd(f'powershell (Get-Item {node_exe}).VersionInfo.FileVersion', node_exe.parent).splitlines()[0] - tool_lines.append("Nodejs", "https://nodejs.org", version, "a JavaScript runtime built on Chrome's V8 JavaScript engine") + tool_lines.append(("Nodejs", "https://nodejs.org", version, "a JavaScript runtime built on Chrome's V8 JavaScript engine")) if (pandoc_exe := python_exe.parent.parent / "t" / "pandoc.exe").exists(): version = exec_shell_cmd("pandoc -v", pandoc_exe.parent).splitlines()[0].split(" ")[-1] - tool_lines.append("Pandoc", "https://pandoc.org", version, "an universal document converter") + tool_lines.append(("Pandoc", "https://pandoc.org", version, "an universal document converter")) if (vscode_exe := python_exe.parent.parent / "t" / "VSCode" / "Code.exe").exists(): version = exec_shell_cmd(f'powershell (Get-Item {vscode_exe}).VersionInfo.FileVersion', vscode_exe.parent).splitlines()[0] - tool_lines.append("VSCode","https://code.visualstudio.com", version, "a source-code editor developed by Microsoft") + tool_lines.append(("VSCode","https://code.visualstudio.com", version, "a source-code editor developed by Microsoft")) return tool_lines def onerror(function, path, excinfo): From 0c010cf4f42cd511ce3d8060d47de809c3e37ad5 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 7 Jun 2025 12:24:30 +0200 Subject: [PATCH 28/37] allow to compare two markdown files directly --- winpython/diff.py | 56 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/winpython/diff.py b/winpython/diff.py index a8e52e1d..2b059d9d 100644 --- a/winpython/diff.py +++ b/winpython/diff.py @@ -11,10 +11,11 @@ import re import shutil from packaging import version +import sys + from . import utils CHANGELOGS_DIR = Path(__file__).parent.parent / "changelogs" -assert CHANGELOGS_DIR.is_dir() class Package: PATTERNS = [ @@ -48,7 +49,7 @@ class PackageIndex: def __init__(self, version, searchdir=None, flavor="", architecture=64): self.version = version self.flavor = flavor - self.searchdir = searchdir + self.searchdir = Path(searchdir) if searchdir else CHANGELOGS_DIR self.architecture = architecture self.packages = {"tools": {}, "python": {}, "wheelhouse": {}} self._load_index() @@ -102,6 +103,10 @@ def find_previous_version(target_version, searchdir=None, flavor="", architectur return max(versions, key=version.parse, default=target_version) def compare_package_indexes(version2, version1=None, searchdir=None, flavor="", flavor1=None, architecture=64): + """Comparison by looking versions in a given Changelog directory""" + if not searchdir or (not Path(searchdir).is_dir() and not CHANGELOGS_DIR.is_dir()): + print(f"Error: changelogs directory {CHANGELOGS_DIR} does not exist.") + sys.exit(1) version1 = version1 or find_previous_version(version2, searchdir, flavor, architecture) flavor1 = flavor1 or flavor @@ -140,8 +145,49 @@ def write_changelog(version2, version1=None, searchdir=None, flavor="", architec f.write(changelog) # Copy to winpython/changelogs back to basedir if basedir: - shutil.copyfile(output_file, basedir / output_file.name) + shutil.copyfile(output_file, Path(basedir) / output_file.name) + +def compare_two_markdown_files(file1, file2): + """Compare two arbitrary markdown files with WinPython changelog format.""" + class DummyPackageIndex(PackageIndex): + def __init__(self, filename): + self.packages = {"tools": {}, "python": {}, "wheelhouse": {}} + self._load_index(filename) + + def _load_index(self, filename): + with open(filename, "r", encoding=utils.guess_encoding(filename)[0]) as f: + self._parse_index(f.read()) + + pi1 = DummyPackageIndex(Path(file1)) + pi2 = DummyPackageIndex(Path(file2)) + + text = f"## Differences between {file1} and {file2}\n\n
\n\n" + for key in PackageIndex.HEADERS: + diff = compare_packages(pi1.packages[key], pi2.packages[key]) + if diff: + text += f"\n{PackageIndex.HEADERS[key]}\n\n{diff}" + return text + "\n
\n\n* * *\n" + +def print_usage(): + print("Usage:") + print(" python diff.py file1.md file2.md") + print(" - Compare two markdown changelog files directly.") + print(" python diff.py [searchdir] [flavor] [architecture]") + print(" - Compare WinPython markdown changelogs by version.") if __name__ == "__main__": - print(compare_package_indexes("3.7.4.0", "3.7.2.0", r"C:\WinP\bd37\budot", "Zero", architecture=32)) - write_changelog("3.7.4.0", "3.7.2.0", r"C:\WinP\bd37\budot", "Ps2", architecture=64) + if len(sys.argv) == 3 and all(arg.lower().endswith('.md') for arg in sys.argv[1:]): + # Usage: python diff.py file1.md file2.md + file1, file2 = sys.argv[1], sys.argv[2] + print(compare_two_markdown_files(file1, file2)) + elif len(sys.argv) >= 3: + # Original usage (version comparison) + # Example: python diff.py 3.7.4.0 3.7.2.0 "C:\WinP\bd37\budot" "Zero" 32 + version2 = sys.argv[1] + version1 = sys.argv[2] + searchdir = Path(sys.argv[3]) if len(sys.argv) > 3 else CHANGELOGS_DIR + flavor = sys.argv[4] if len(sys.argv) > 4 else "" + architecture = int(sys.argv[5]) if len(sys.argv) > 5 else 64 + print(compare_package_indexes(version2, version1, searchdir, flavor, architecture=architecture)) + else: + print_usage() \ No newline at end of file From 81dabf7214f5c05b1a25e21b69186ae32ded11ef Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 7 Jun 2025 19:09:57 +0200 Subject: [PATCH 29/37] allows diffing from history, files, strings, and mixed sections --- winpython/diff.py | 216 +++++++++++++++++++++++++--------------------- 1 file changed, 117 insertions(+), 99 deletions(-) diff --git a/winpython/diff.py b/winpython/diff.py index 2b059d9d..12671faf 100644 --- a/winpython/diff.py +++ b/winpython/diff.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- # -# WinPython diff.py script +# WinPython diff.py script (streamlined, with historical and flexible modes) # Copyright © 2013 Pierre Raybaut # Copyright © 2014-2025+ The Winpython development team https://github.com/winpython/ # Licensed under the terms of the MIT License -# (see winpython/__init__.py for details) import os -from pathlib import Path import re +import sys import shutil +from pathlib import Path from packaging import version -import sys - from . import utils CHANGELOGS_DIR = Path(__file__).parent.parent / "changelogs" @@ -22,12 +20,10 @@ class Package: r"\[([\w\-\:\/\.\_]+)\]\(([^)]+)\) \| ([^\|]*) \| ([^\|]*)", # SourceForge r"\[([\w\-\:\/\.\_]+) ([^\]\ ]+)\] \| ([^\|]*) \| ([^\|]*)" # Google Code ] - def __init__(self, text=None): self.name = self.url = self.version = self.description = None if text: self.from_text(text) - def from_text(self, text): for pattern in self.PATTERNS: match = re.match(pattern, text) @@ -36,31 +32,13 @@ def from_text(self, text): return raise ValueError(f"Unrecognized package line format: {text}") - def to_wiki(self): - return f" * [{self.name}]({self.url}) {self.version} ({self.description})\n" - - def upgrade_wiki(self, other): - return f" * [{self.name}]({self.url}) {other.version} → {self.version} ({self.description})\n" - class PackageIndex: HEADERS = {"tools": "### Tools", "python": "### Python packages", "wheelhouse": "### WheelHouse packages"} BLANKS = ["Name | Version | Description", "-----|---------|------------", "", "
", "
"] - def __init__(self, version, searchdir=None, flavor="", architecture=64): - self.version = version - self.flavor = flavor - self.searchdir = Path(searchdir) if searchdir else CHANGELOGS_DIR - self.architecture = architecture - self.packages = {"tools": {}, "python": {}, "wheelhouse": {}} - self._load_index() - - def _load_index(self): - filename = self.searchdir / f"WinPython{self.flavor}-{self.architecture}bit-{self.version}.md" - if not filename.exists(): - raise FileNotFoundError(f"Changelog not found: {filename}") - - with open(filename, "r", encoding=utils.guess_encoding(filename)[0]) as f: - self._parse_index(f.read()) + def __init__(self, content): + self.packages = {k: {} for k in self.HEADERS} + self._parse_index(content) def _parse_index(self, text): current = None @@ -71,123 +49,163 @@ def _parse_index(self, text): if line.strip() in self.BLANKS: continue if current: - pkg = Package(line) - self.packages[current][pkg.name] = pkg + try: + pkg = Package(line) + self.packages[current][pkg.name] = pkg + except Exception: + continue def compare_packages(old, new): - """Return difference between package old and package new""" - - # wheel replace '-' per '_' in key def normalize(d): return {k.replace("-", "_").lower(): v for k, v in d.items()} old, new = normalize(old), normalize(new) + added = [new[k] for k in new if k not in old] + upgraded = [new[k] for k in new if k in old and new[k].version != old[k].version] + removed = [old[k] for k in old if k not in new] output = "" - - added = [new[k].to_wiki() for k in new if k not in old] - upgraded = [new[k].upgrade_wiki(old[k]) for k in new if k in old and new[k].version != old[k].version] - removed = [old[k].to_wiki() for k in old if k not in new] - if added: - output += "New packages:\n\n" + "".join(added) + "\n\n" + output += "\nNew packages:\n" + "".join(f" * {p.name} {p.version} ({p.description})\n" for p in added) if upgraded: - output += "Upgraded packages:\n\n" + "".join(upgraded) + "\n\n" + output += "\nUpgraded packages:\n" + "".join(f" * {p.name} {old[p.name].version} → {p.version} ({p.description})\n" for p in upgraded if p.name in old) if removed: - output += "Removed packages:\n\n" + "".join(removed) + "\n\n" - return output + output += "\nRemoved packages:\n" + "".join(f" * {p.name} {p.version} ({p.description})\n" for p in removed) + return output or "\nNo differences found.\n" + +def compare_markdown_sections(md1, md2, header1="python", header2="python", label1="Input1", label2="Input2"): + pkgs1 = PackageIndex(md1).packages + pkgs2 = PackageIndex(md2).packages + diff = compare_packages(pkgs1[header1], pkgs2[header2]) + # If comparing the same section, use the historical header + if header1 == header2 and header1 in PackageIndex.HEADERS: + title = PackageIndex.HEADERS[header1] + else: + title = f"## {label1} [{header1}] vs {label2} [{header2}]" + return f"{title}\n\n{diff}" + +def compare_markdown_section_pairs(md1, md2, header_pairs, label1="Input1", label2="Input2"): + pkgs1 = PackageIndex(md1).packages + pkgs2 = PackageIndex(md2).packages + text = f"# {label1} vs {label2} section-pairs comparison\n" + for h1, h2 in header_pairs: + diff = compare_packages(pkgs1[h1], pkgs2[h2]) + if diff.strip() and diff != "No differences found.\n": + text += f"\n## {label1} [{h1}] vs {label2} [{h2}]\n\n{diff}\n" + return text + +def compare_files(file1, file2, mode="full", header1=None, header2=None, header_pairs=None): + with open(file1, encoding=utils.guess_encoding(file1)[0]) as f1, \ + open(file2, encoding=utils.guess_encoding(file2)[0]) as f2: + md1, md2 = f1.read(), f2.read() + if mode == "full": + result = "" + for k in PackageIndex.HEADERS: + result += compare_markdown_sections(md1, md2, k, k, file1, file2) + "\n" + return result + elif mode == "section": + return compare_markdown_sections(md1, md2, header1, header2, file1, file2) + elif mode == "pairs": + return compare_markdown_section_pairs(md1, md2, header_pairs, file1, file2) + else: + raise ValueError("Unknown mode.") + +# --- ORIGINAL/HISTORICAL VERSION-TO-VERSION COMPARISON --- def find_previous_version(target_version, searchdir=None, flavor="", architecture=64): - """Find version which is the closest to `version`""" search_dir = Path(searchdir) if searchdir else CHANGELOGS_DIR pattern = re.compile(rf"WinPython{flavor}-{architecture}bit-([0-9\.]+)\.(txt|md)") versions = [pattern.match(f).group(1) for f in os.listdir(search_dir) if pattern.match(f)] versions = [v for v in versions if version.parse(v) < version.parse(target_version)] return max(versions, key=version.parse, default=target_version) +def load_version_markdown(version, searchdir, flavor="", architecture=64): + filename = Path(searchdir) / f"WinPython{flavor}-{architecture}bit-{version}.md" + if not filename.exists(): + raise FileNotFoundError(f"Changelog not found: {filename}") + with open(filename, "r", encoding=utils.guess_encoding(filename)[0]) as f: + return f.read() + def compare_package_indexes(version2, version1=None, searchdir=None, flavor="", flavor1=None, architecture=64): - """Comparison by looking versions in a given Changelog directory""" - if not searchdir or (not Path(searchdir).is_dir() and not CHANGELOGS_DIR.is_dir()): - print(f"Error: changelogs directory {CHANGELOGS_DIR} does not exist.") - sys.exit(1) + searchdir = Path(searchdir) if searchdir else CHANGELOGS_DIR version1 = version1 or find_previous_version(version2, searchdir, flavor, architecture) flavor1 = flavor1 or flavor - - pi1 = PackageIndex(version1, searchdir, flavor1, architecture) - pi2 = PackageIndex(version2, searchdir, flavor, architecture) - - text = ( + md1 = load_version_markdown(version1, searchdir, flavor1, architecture) + md2 = load_version_markdown(version2, searchdir, flavor, architecture) + result = f"# WinPython {architecture}bit {version2}{flavor} vs {version1}{flavor1}\n" + result = ( f"## History of changes for WinPython-{architecture}bit {version2 + flavor}\r\n\r\n" f"The following changes were made to WinPython-{architecture}bit distribution since version {version1 + flavor1}.\n\n\n" "
\n\n" ) - - for key in PackageIndex.HEADERS: - diff = compare_packages(pi1.packages[key], pi2.packages[key]) - if diff: - text += f"\n{PackageIndex.HEADERS[key]}\n\n{diff}" - - return text + "\n
\n\n* * *\n" + for k in PackageIndex.HEADERS: + result += compare_markdown_sections(md1, md2, k, k, version1, version2) + "\n" + return result+ "\n
\n\n* * *\n" def copy_changelogs(version, searchdir, flavor="", architecture=64, basedir=None): - basever = ".".join(version.split(".")[:2]) + """Copy all changelogs for a major.minor version into basedir.""" + basever = ".".join(str(version).split(".")[:2]) pattern = re.compile(rf"WinPython{flavor}-{architecture}bit-{basever}[0-9\.]*\.(txt|md)") dest = Path(basedir) for fname in os.listdir(searchdir): if pattern.match(fname): - shutil.copyfile(searchdir / fname, dest / fname) + shutil.copyfile(Path(searchdir) / fname, dest / fname) def write_changelog(version2, version1=None, searchdir=None, flavor="", architecture=64, basedir=None): - """Write changelog between version1 and version2 of WinPython""" + """Write changelog between version1 and version2 of WinPython.""" + searchdir = Path(searchdir) if searchdir else CHANGELOGS_DIR if basedir: copy_changelogs(version2, searchdir, flavor, architecture, basedir) - print("comparing_package_indexes", version2, searchdir, flavor, architecture) changelog = compare_package_indexes(version2, version1, searchdir, flavor, architecture=architecture) output_file = searchdir / f"WinPython{flavor}-{architecture}bit-{version2}_History.md" with open(output_file, "w", encoding="utf-8") as f: f.write(changelog) - # Copy to winpython/changelogs back to basedir if basedir: shutil.copyfile(output_file, Path(basedir) / output_file.name) -def compare_two_markdown_files(file1, file2): - """Compare two arbitrary markdown files with WinPython changelog format.""" - class DummyPackageIndex(PackageIndex): - def __init__(self, filename): - self.packages = {"tools": {}, "python": {}, "wheelhouse": {}} - self._load_index(filename) - - def _load_index(self, filename): - with open(filename, "r", encoding=utils.guess_encoding(filename)[0]) as f: - self._parse_index(f.read()) - - pi1 = DummyPackageIndex(Path(file1)) - pi2 = DummyPackageIndex(Path(file2)) - - text = f"## Differences between {file1} and {file2}\n\n
\n\n" - for key in PackageIndex.HEADERS: - diff = compare_packages(pi1.packages[key], pi2.packages[key]) - if diff: - text += f"\n{PackageIndex.HEADERS[key]}\n\n{diff}" - return text + "\n
\n\n* * *\n" - def print_usage(): print("Usage:") print(" python diff.py file1.md file2.md") - print(" - Compare two markdown changelog files directly.") + print(" - Compare all sections of two markdown files.") + print(" python diff.py file1.md file2.md --section header1 header2") + print(" - Compare section 'header1' of file1 with section 'header2' of file2.") + print(" python diff.py file1.md file2.md --pairs header1a header2a [header1b header2b ...]") + print(" - Compare pairs of sections. Example: python diff.py f1.md f2.md --pairs python wheelhouse tools tools") print(" python diff.py [searchdir] [flavor] [architecture]") - print(" - Compare WinPython markdown changelogs by version.") + print(" - Compare WinPython markdown changelogs by version (historical mode).") + print(" python diff.py --write-changelog [searchdir] [flavor] [architecture] [basedir]") + print(" - Write changelog between version1 and version2 to file (and optionally copy to basedir).") if __name__ == "__main__": - if len(sys.argv) == 3 and all(arg.lower().endswith('.md') for arg in sys.argv[1:]): - # Usage: python diff.py file1.md file2.md - file1, file2 = sys.argv[1], sys.argv[2] - print(compare_two_markdown_files(file1, file2)) - elif len(sys.argv) >= 3: - # Original usage (version comparison) - # Example: python diff.py 3.7.4.0 3.7.2.0 "C:\WinP\bd37\budot" "Zero" 32 - version2 = sys.argv[1] - version1 = sys.argv[2] - searchdir = Path(sys.argv[3]) if len(sys.argv) > 3 else CHANGELOGS_DIR - flavor = sys.argv[4] if len(sys.argv) > 4 else "" - architecture = int(sys.argv[5]) if len(sys.argv) > 5 else 64 + args = sys.argv + if len(args) >= 3 and all(arg.lower().endswith('.md') for arg in args[1:3]): + file1, file2 = args[1], args[2] + if len(args) == 3: + print(compare_files(file1, file2)) + elif args[3] == "--section" and len(args) >= 6: + h1, h2 = args[4], args[5] + print(compare_files(file1, file2, mode="section", header1=h1, header2=h2)) + elif args[3] == "--pairs" and len(args) > 4 and len(args[4:]) % 2 == 0: + pairs = list(zip(args[4::2], args[5::2])) + print(compare_files(file1, file2, mode="pairs", header_pairs=pairs)) + else: + print_usage() + elif len(args) >= 2 and args[1] == "--write-changelog": + # Usage: --write-changelog [searchdir] [flavor] [architecture] [basedir] + if len(args) < 4: + print_usage() + sys.exit(1) + version2 = args[2] + version1 = args[3] + searchdir = args[4] if len(args) > 4 else CHANGELOGS_DIR + flavor = args[5] if len(args) > 5 else "" + architecture = int(args[6]) if len(args) > 6 else 64 + basedir = args[7] if len(args) > 7 else None + write_changelog(version2, version1, searchdir, flavor, architecture, basedir) + print(f"Changelog written for {version2} vs {version1}.") + elif len(args) >= 3: + version2 = args[1] + version1 = args[2] if len(args) > 2 and not args[2].endswith('.md') else None + searchdir = args[3] if len(args) > 3 else CHANGELOGS_DIR + flavor = args[4] if len(args) > 4 else "" + architecture = int(args[5]) if len(args) > 5 else 64 print(compare_package_indexes(version2, version1, searchdir, flavor, architecture=architecture)) else: print_usage() \ No newline at end of file From cb85466e4d46d45e5af9551db5950538c62bd3ac Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 7 Jun 2025 20:25:39 +0200 Subject: [PATCH 30/37] wppm -md -ws . will compare Installed package to WheelHouse packages --- winpython/wppm.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/winpython/wppm.py b/winpython/wppm.py index 01ed2c83..60f1c692 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -14,7 +14,7 @@ import json from pathlib import Path from argparse import ArgumentParser, RawTextHelpFormatter -from . import utils, piptree, associate +from . import utils, piptree, associate, diff from . import wheelhouse as wh from operator import itemgetter # Workaround for installing PyVISA on Windows from source: @@ -80,10 +80,10 @@ def render_markdown_for_list(self, title, items): return md def generate_package_index_markdown(self, python_executable_directory: str|None = None, winpyver2: str|None = None, - flavor: str|None = None, architecture_bits: int|None = None, release_level: str|None = None) -> str: + flavor: str|None = None, architecture_bits: int|None = None + , release_level: str|None = None, wheeldir: str|None = None) -> str: """Generates a Markdown formatted package index page.""" my_ver , my_arch = utils.get_python_infos(python_executable_directory or self.target) - # suppose we suite ourself (method will vary over time) my_winpyver2 = winpyver2 or os.getenv("WINPYVER2","") my_winpyver2 = my_winpyver2 if my_winpyver2 != "" else my_ver my_flavor = flavor or os.getenv("WINPYFLAVOR", "") @@ -92,10 +92,10 @@ def generate_package_index_markdown(self, python_executable_directory: str|None tools_list = utils.get_installed_tools(utils.get_python_executable(python_executable_directory)) package_list = [(pkg.name, pkg.url, pkg.version, pkg.description) for pkg in self.get_installed_packages()] wheelhouse_list = [] - wheeldir = self.wheelhouse / 'included.wheels' - if wheeldir.is_dir(): + my_wheeldir = Path(wheeldir) if wheeldir else self.wheelhouse / 'included.wheels' + if my_wheeldir.is_dir(): wheelhouse_list = [(name, f"https://pypi.org/project/{name}", version, summary) - for name, version, summary in wh.list_packages_with_metadata(str(wheeldir)) ] + for name, version, summary in wh.list_packages_with_metadata(str(my_wheeldir)) ] return f"""## WinPython {my_winpyver2 + my_flavor} @@ -366,7 +366,12 @@ def main(test=False): p = subprocess.Popen(["start", "cmd", "/k",dist.python_exe, "-c" , cmd_mov], shell = True, cwd=dist.target) sys.exit() if args.markdown: - print(dist.generate_package_index_markdown()) + default = dist.generate_package_index_markdown() + if args.wheelsource: + compare = dist.generate_package_index_markdown(wheeldir = args.wheelsource) + print(diff.compare_markdown_sections(default, compare,'python', 'wheelhouse', 'installed', 'wheelhouse')) + else: + print(default) sys.exit() if not args.install and not args.uninstall and args.fname.endswith(".toml"): args.install = True # for Drag & Drop of .toml (and not wheel) From 25ca4cfc6f7284b5aa78d86f96f158b42c6da6e6 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 7 Jun 2025 20:29:09 +0200 Subject: [PATCH 31/37] wppm support multiple packages install, list or navigation --- winpython/wppm.py | 77 ++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/winpython/wppm.py b/winpython/wppm.py index 60f1c692..308030ce 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -184,7 +184,7 @@ def patch_standard_packages(self, package_name="", to_movable=True): # create movable launchers for previous package installations self.patch_all_shebang(to_movable=to_movable) if package_name.lower() in ("", "spyder"): - # spyder don't goes on internet without I ask + # spyder don't goes on internet without you ask utils.patch_sourcefile( Path(self.target) / "lib" / "site-packages" / "spyder" / "config" / "main.py", "'check_updates_on_startup': True,", @@ -266,19 +266,17 @@ def main(test=False): description="WinPython Package Manager: handle a WinPython Distribution and its packages", formatter_class=RawTextHelpFormatter, ) - parser.add_argument("fname", metavar="package or lockfile", nargs="?", default="", type=str, help="optional package name or package wheel") + parser.add_argument("fname", metavar="package(s) or lockfile", nargs="*", default=[""], type=str, help="optional package names, wheels, or lockfile") parser.add_argument("-v", "--verbose", action="store_true", help="show more details on packages and actions") parser.add_argument( "--register", dest="registerWinPython", action="store_true", help=registerWinPythonHelp) - # parser.add_argument( "--register_forall", action="store_true", help="Register distribution for all users") parser.add_argument("--unregister", dest="unregisterWinPython", action="store_true", help=unregisterWinPythonHelp) - # parser.add_argument( "--unregister_forall", action="store_true", help="un-Register distribution for all users") parser.add_argument("--fix", action="store_true", help="make WinPython fix") parser.add_argument("--movable", action="store_true", help="make WinPython movable") parser.add_argument("-ws", dest="wheelsource", default=None, type=str, help="wheels location, '.' = WheelHouse): wppm pylock.toml -ws source_of_wheels, wppm -ls -ws .") parser.add_argument("-wd", dest="wheeldrain" , default=None, type=str, help="wheels destination: wppm pylock.toml -wd destination_of_wheels") parser.add_argument("-ls", "--list", action="store_true", help="list installed packages matching [optional] expression: wppm -ls, wppm -ls pand") parser.add_argument("-lsa", dest="all", action="store_true",help=f"list details of packages matching [optional] expression: wppm -lsa pandas -l1") - parser.add_argument("-md", dest="markdown", action="store_true",help=f"markdown summary if the installation") + parser.add_argument("-md", dest="markdown", action="store_true",help=f"markdown summary of the installation") parser.add_argument("-p",dest="pipdown",action="store_true",help="show Package dependencies of the given package[option], [.]=all: wppm -p pandas[.]") parser.add_argument("-r", dest="pipup", action="store_true", help=f"show Reverse wppmdependancies of the given package[option]: wppm -r pytest[test]") parser.add_argument("-l", dest="levels", type=int, default=2, help="show 'LEVELS' levels of dependencies (with -p, -r), default is 2: wppm -p pandas -l1") @@ -286,7 +284,6 @@ def main(test=False): parser.add_argument("-i", "--install", action="store_true", help="install a given package wheel or pylock file (use pip for more features)") parser.add_argument("-u", "--uninstall", action="store_true", help="uninstall package (use pip for more features)") - args = parser.parse_args() targetpython = None if args.target and args.target != sys.prefix: @@ -301,17 +298,22 @@ def main(test=False): raise RuntimeError("Incompatible arguments: --install and --uninstall") if args.pipdown: pip = piptree.PipData(targetpython, args.wheelsource) - pack, extra, *other = (args.fname + "[").replace("]", "[").split("[") - print(pip.down(pack, extra, args.levels, verbose=args.verbose)) + for args_fname in args.fname: + pack, extra, *other = (args_fname + "[").replace("]", "[").split("[") + print(pip.down(pack, extra, args.levels, verbose=args.verbose)) sys.exit() elif args.pipup: pip = piptree.PipData(targetpython, args.wheelsource) - pack, extra, *other = (args.fname + "[").replace("]", "[").split("[") - print(pip.up(pack, extra, args.levels, verbose=args.verbose)) + for args_fname in args.fname: + pack, extra, *other = (args_fname + "[").replace("]", "[").split("[") + print(pip.up(pack, extra, args.levels, verbose=args.verbose)) sys.exit() elif args.list: pip = piptree.PipData(targetpython, args.wheelsource) - todo = [l for l in pip.pip_list(full=True) if bool(re.search(args.fname, l[0]))] + todo= [] + for args_fname in args.fname: + todo += [l for l in pip.pip_list(full=True) if bool(re.search(args_fname, l[0]))] + todo = sorted(set(todo)) #, key=lambda p: (p[0].lower(), p[2]) titles = [['Package', 'Version', 'Summary'], ['_' * max(x, 6) for x in utils.columns_width(todo)]] listed = utils.formatted_list(titles + todo, max_width=70) for p in listed: @@ -319,17 +321,17 @@ def main(test=False): sys.exit() elif args.all: pip = piptree.PipData(targetpython, args.wheelsource) - todo = [l for l in pip.pip_list(full=True) if bool(re.search(args.fname, l[0]))] - for l in todo: - # print(pip.distro[l[0]]) - title = f"** Package: {l[0]} **" - print("\n" + "*" * len(title), f"\n{title}", "\n" + "*" * len(title)) - for key, value in pip.raw[l[0]].items(): - rawtext = json.dumps(value, indent=2, ensure_ascii=False) - lines = [l for l in rawtext.split(r"\n") if len(l.strip()) > 2] - if key.lower() != 'description' or args.verbose: - print(f"{key}: ", "\n".join(lines).replace('"', "")) - sys.exit() + for args_fname in args.fname: + todo = [l for l in pip.pip_list(full=True) if bool(re.search(args_fname, l[0]))] + for l in sorted(set(todo)): + title = f"** Package: {l[0]} **" + print("\n" + "*" * len(title), f"\n{title}", "\n" + "*" * len(title)) + for key, value in pip.raw[l[0]].items(): + rawtext = json.dumps(value, indent=2, ensure_ascii=False) + lines = [l for l in rawtext.split(r"\n") if len(l.strip()) > 2] + if key.lower() != 'description' or args.verbose: + print(f"{key}: ", "\n".join(lines).replace('"', "")) + sys.exit() if args.registerWinPython: print(registerWinPythonHelp) if utils.is_python_distribution(args.target): @@ -373,26 +375,27 @@ def main(test=False): else: print(default) sys.exit() - if not args.install and not args.uninstall and args.fname.endswith(".toml"): + if not args.install and not args.uninstall and args.fname[0].endswith(".toml"): args.install = True # for Drag & Drop of .toml (and not wheel) - if args.fname == "" or (not args.install and not args.uninstall): + if args.fname[0] == "" or (not args.install and not args.uninstall): parser.print_help() sys.exit() else: try: - filename = Path(args.fname).name - install_from_wheelhouse = ["--no-index", "--trusted-host=None", f"--find-links={dist.wheelhouse / 'included.wheels'}"] - if filename.split('.')[0] == "pylock" and filename.split('.')[-1] == 'toml': - print(' a lock file !', args.fname, dist.target) - wh.get_pylock_wheels(dist.wheelhouse, Path(args.fname), args.wheelsource, args.wheeldrain) - sys.exit() - if args.uninstall: - package = dist.find_package(args.fname) - dist.uninstall(package) - elif args.install: - package = Package(args.fname) - if args.install: - dist.install(package, install_options=install_from_wheelhouse) + for args_fname in args.fname: + filename = Path(args_fname).name + install_from_wheelhouse = ["--no-index", "--trusted-host=None", f"--find-links={dist.wheelhouse / 'included.wheels'}"] + if filename.split('.')[0] == "pylock" and filename.split('.')[-1] == 'toml': + print(' a lock file !', args_fname, dist.target) + wh.get_pylock_wheels(dist.wheelhouse, Path(args_fname), args.wheelsource, args.wheeldrain) + sys.exit() + if args.uninstall: + package = dist.find_package(args_fname) + dist.uninstall(package) + elif args.install: + package = Package(args_fname) + if args.install: + dist.install(package, install_options=install_from_wheelhouse) except NotImplementedError: raise RuntimeError("Package is not (yet) supported by WPPM") else: From 04b1b2cff3cb2277409b156174a6d7fd4d730535 Mon Sep 17 00:00:00 2001 From: stonebig Date: Wed, 11 Jun 2025 20:14:50 +0200 Subject: [PATCH 32/37] summarize launchers categories --- portable/build_my_launchers.bat | 21 +++++++++++++-------- winpython/__init__.py | 2 +- winpython/piptree.py | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/portable/build_my_launchers.bat b/portable/build_my_launchers.bat index 579a96e6..2585eb5c 100644 --- a/portable/build_my_launchers.bat +++ b/portable/build_my_launchers.bat @@ -9,24 +9,29 @@ rem pick the right ones and rename them in launchers_final set do_launcher=%~dp0launchers_src\build_one_launcher.bat set do_launcher_original=%~dp0launchers_src_original\build_one_launcher.bat -::WINDOWS launchers -rem call %do_launcher% "python.ico" "winidle.bat" "IDLE (Python GUI)" WINDOWS proposed echo displace this pause if you want to re-build more pause rem exit -call %do_launcher% "powershell.ico" "cmd_ps.bat" "WinPython Powershell Prompt" WINDOWS proposed +:: WINDOWS launchers with Drag & Drop +call %do_launcher% "python.ico" "winidle.bat" "IDLE (Python GUI)" WINDOWS proposed call %do_launcher% "spyder.ico" "winspyder.bat" "Spyder" WINDOWS proposed -call %do_launcher% "spyder_reset.ico" "spyder_reset.bat" "Spyder reset" WINDOWS proposed -call %do_launcher% "code.ico" "winvscode.bat" "VS Code" WINDOWS proposed -:: CONSOLE launchers -call %do_launcher% "cmd.ico" "cmd.bat" "WinPython Command Prompt" CONSOLE proposed -call %do_launcher% "python.ico" "winpython.bat" "WinPython Interpreter" CONSOLE proposed +:: CONSOLE launchers with Drag & Drop call %do_launcher% "jupyter.ico" "winipython_notebook.bat" "Jupyter Notebook" CONSOLE proposed call %do_launcher% "jupyter.ico" "winjupyter_lab.bat" "Jupyter Lab" CONSOLE proposed call %do_launcher% "winpython.ico" "wpcp.bat" "WinPython Control Panel" CONSOLE proposed + +:: WINDOWS launchers with no Drag & Drop +call %do_launcher_original% "powershell.ico" "cmd_ps.bat" "WinPython Powershell Prompt" WINDOWS proposed +call %do_launcher_original% "spyder_reset.ico" "spyder_reset.bat" "Spyder reset" WINDOWS proposed +call %do_launcher_original% "code.ico" "winvscode.bat" "VS Code" WINDOWS proposed + +:: CONSOLE launchers with no Drag & Drop +call %do_launcher_original% "cmd.ico" "cmd.bat" "WinPython Command Prompt" CONSOLE proposed +call %do_launcher_original% "python.ico" "winpython.bat" "WinPython Interpreter" CONSOLE proposed + pause diff --git a/winpython/__init__.py b/winpython/__init__.py index 468eed57..d169819a 100644 --- a/winpython/__init__.py +++ b/winpython/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '16.4.20250603' +__version__ = '16.5.20250608' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' diff --git a/winpython/piptree.py b/winpython/piptree.py index e46bd465..1c61501e 100644 --- a/winpython/piptree.py +++ b/winpython/piptree.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Enhanced script to inspect and display Python package dependencies, +piptree.py: inspect and display Python package dependencies, supporting both downward and upward dependency trees. Requires Python 3.8+ due to importlib.metadata. """ From 77a2fe05f4b14976cdb264e19f3fd39eaec33e9c Mon Sep 17 00:00:00 2001 From: stonebig Date: Wed, 11 Jun 2025 20:39:13 +0200 Subject: [PATCH 33/37] tweak output of build --- generate_a_winpython_distro.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate_a_winpython_distro.bat b/generate_a_winpython_distro.bat index e9c73d9a..cbb74682 100644 --- a/generate_a_winpython_distro.bat +++ b/generate_a_winpython_distro.bat @@ -162,7 +162,7 @@ set req_lock_local=%LOCKDIR%requirement.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK set pip_lock_web=%LOCKDIR%pylock.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%%my_release_level%.toml set pip_lock_local=%LOCKDIR%pylock.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%%my_release_level%_local.toml set req_lock_web=%LOCKDIR%requirement.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%%my_release_level%.txt -set req_lock_local=%LOCKDIR%requirement.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%_local.txt +set req_lock_local=%LOCKDIR%requirement.%WINPYARCH%-%WINPYVERLOCK%%my_flavor%%my_release_level%_local.txt set my_archive_lockfile=%my_archive_dir%\pylock.%my_flavor%-%WINPYARCH%bit-%WINPYVERLOCK%_%date:/=-%at_%my_time%.toml From 11e984e7b6ea61ede870625a55a723537e0cc6dc Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 14 Jun 2025 20:22:56 +0200 Subject: [PATCH 34/37] ensure a closer match between wppm -md and make.py markdowns --- winpython/__init__.py | 2 +- winpython/wppm.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/winpython/__init__.py b/winpython/__init__.py index d169819a..0fe7dcc2 100644 --- a/winpython/__init__.py +++ b/winpython/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '16.5.20250608' +__version__ = '16.5.20250614' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' diff --git a/winpython/wppm.py b/winpython/wppm.py index 308030ce..59f14ff8 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -75,7 +75,7 @@ def render_markdown_for_list(self, title, items): md += "Name | Version | Description\n" md += "-----|---------|------------\n" for name, url, version, summary in sorted(items, key=lambda p: (p[0].lower(), p[2])): - md += f"[{name}]({url}) | {version} | {summary} \n" + md += f"[{name}]({url}) | {version} | {summary}\n" md += "\n" return md @@ -94,7 +94,7 @@ def generate_package_index_markdown(self, python_executable_directory: str|None wheelhouse_list = [] my_wheeldir = Path(wheeldir) if wheeldir else self.wheelhouse / 'included.wheels' if my_wheeldir.is_dir(): - wheelhouse_list = [(name, f"https://pypi.org/project/{name}", version, summary) + wheelhouse_list = [(name, f"https://pypi.org/project/{name}", version, utils.sum_up(summary)) for name, version, summary in wh.list_packages_with_metadata(str(my_wheeldir)) ] return f"""## WinPython {my_winpyver2 + my_flavor} From cbe3bbd99115f14e0e9124afbcdd88d28f0ea8aa Mon Sep 17 00:00:00 2001 From: stonebig Date: Fri, 20 Jun 2025 19:32:29 +0200 Subject: [PATCH 35/37] simplify constraints hunting --- winpython/__init__.py | 2 +- winpython/piptree.py | 16 ++++++++++------ winpython/wppm.py | 8 ++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/winpython/__init__.py b/winpython/__init__.py index 0fe7dcc2..b6c1d768 100644 --- a/winpython/__init__.py +++ b/winpython/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '16.5.20250614' +__version__ = '16.6.20250620' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' diff --git a/winpython/piptree.py b/winpython/piptree.py index 1c61501e..970f4edf 100644 --- a/winpython/piptree.py +++ b/winpython/piptree.py @@ -197,10 +197,12 @@ def _get_dependency_tree(self, package_name: str, extra: str = "", version_req: extra + ',' in dependency["req_extra"] + ',' and \ Marker(dependency["req_marker"]).evaluate(environment=environment | {"extra": up_req})): # IA risk error: # dask[array] go upwards as dask[dataframe], so {"extra": up_req} , not {"extra": extra} + #tag downward limiting dependancies + wall = " " if dependency["req_version"].startswith("<") or dependency["req_version"].startswith("==") else "" ret += self._get_dependency_tree( dependency["req_key"], up_req, - f"[requires: {package_name}" + f"[requires{wall}: {package_name}" + (f"[{dependency['req_extra']}]" if dependency["req_extra"] != "" else "") + f'{dependency["req_version"]}]', depth, @@ -242,16 +244,18 @@ def down(self, pp: str = "", extra: str = "", depth: int = 20, indent: int = 5, lines = [l for l in rawtext.split("\n") if len(l.strip()) > 2] return "\n".join(lines).replace('"', "") - def up(self, pp: str, extra: str = "", depth: int = 20, indent: int = 5, version_req: str = "", verbose: bool = False) -> str: + def up(self, ppw: str, extra: str = "", depth: int = 20, indent: int = 5, version_req: str = "", verbose: bool = False) -> str: """Generate upward dependency tree as formatted string.""" + pp = ppw[:-1] if ppw.endswith('!') else ppw + ppend = "!" if ppw.endswith('!') else "" #show only downward limiting dependancies if pp == ".": - results = [self.up(p, extra, depth, indent, version_req, verbose) for p in sorted(self.distro)] + results = [self.up(p + ppend, extra, depth, indent, version_req, verbose) for p in sorted(self.distro)] return '\n'.join(filter(None, results)) if extra == ".": if pp in self.distro: extras = set(self.distro[pp]["provided"]).union(set(self.distro[pp]["provides"])) - results = [self.up(pp, e, depth, indent, version_req, verbose=verbose) for e in sorted(extras)] + results = [self.up(pp + ppend, e, depth, indent, version_req, verbose=verbose) for e in sorted(extras)] return '\n'.join(filter(None, results)) return "" @@ -259,8 +263,8 @@ def up(self, pp: str, extra: str = "", depth: int = 20, indent: int = 5, version return "" rawtext = json.dumps(self._get_dependency_tree(pp, extra, version_req, depth, verbose=verbose, upward=True), indent=indent) - lines = [l for l in rawtext.split("\n") if len(l.strip()) > 2] - return "\n".join(filter(None, lines)).replace('"', "") + lines = [l for l in rawtext.split("\n") if len(l.strip()) > 2 and ( ppend=="" or not "[requires:" in l)] + return "\n".join(filter(None, lines)).replace('"', "").replace('[requires :', '[requires:') def description(self, pp: str) -> None: """Return package description or None if not found.""" diff --git a/winpython/wppm.py b/winpython/wppm.py index 59f14ff8..77a937ac 100644 --- a/winpython/wppm.py +++ b/winpython/wppm.py @@ -278,8 +278,8 @@ def main(test=False): parser.add_argument("-lsa", dest="all", action="store_true",help=f"list details of packages matching [optional] expression: wppm -lsa pandas -l1") parser.add_argument("-md", dest="markdown", action="store_true",help=f"markdown summary of the installation") parser.add_argument("-p",dest="pipdown",action="store_true",help="show Package dependencies of the given package[option], [.]=all: wppm -p pandas[.]") - parser.add_argument("-r", dest="pipup", action="store_true", help=f"show Reverse wppmdependancies of the given package[option]: wppm -r pytest[test]") - parser.add_argument("-l", dest="levels", type=int, default=2, help="show 'LEVELS' levels of dependencies (with -p, -r), default is 2: wppm -p pandas -l1") + parser.add_argument("-r", dest="pipup", action="store_true", help=f"show Reverse (!= constraining) dependancies of the given package[option]: wppm -r pytest![test]") + parser.add_argument("-l", dest="levels", type=int, default=-1, help="show 'LEVELS' levels of dependencies (with -p, -r): wppm -p pandas -l1") parser.add_argument("-t", dest="target", default=sys.prefix, help=f'path to target Python distribution (default: "{sys.prefix}")') parser.add_argument("-i", "--install", action="store_true", help="install a given package wheel or pylock file (use pip for more features)") parser.add_argument("-u", "--uninstall", action="store_true", help="uninstall package (use pip for more features)") @@ -300,13 +300,13 @@ def main(test=False): pip = piptree.PipData(targetpython, args.wheelsource) for args_fname in args.fname: pack, extra, *other = (args_fname + "[").replace("]", "[").split("[") - print(pip.down(pack, extra, args.levels, verbose=args.verbose)) + print(pip.down(pack, extra, args.levels if args.levels>0 else 2, verbose=args.verbose)) sys.exit() elif args.pipup: pip = piptree.PipData(targetpython, args.wheelsource) for args_fname in args.fname: pack, extra, *other = (args_fname + "[").replace("]", "[").split("[") - print(pip.up(pack, extra, args.levels, verbose=args.verbose)) + print(pip.up(pack, extra, args.levels if args.levels>=0 else 1, verbose=args.verbose)) sys.exit() elif args.list: pip = piptree.PipData(targetpython, args.wheelsource) From ddd15c5fdfe475d66f5f1287e7fb95218d2551fc Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 21 Jun 2025 12:12:48 +0200 Subject: [PATCH 36/37] include "~" as a contraining factor and hides internals leak --- winpython/piptree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/winpython/piptree.py b/winpython/piptree.py index 970f4edf..11700945 100644 --- a/winpython/piptree.py +++ b/winpython/piptree.py @@ -198,7 +198,7 @@ def _get_dependency_tree(self, package_name: str, extra: str = "", version_req: Marker(dependency["req_marker"]).evaluate(environment=environment | {"extra": up_req})): # IA risk error: # dask[array] go upwards as dask[dataframe], so {"extra": up_req} , not {"extra": extra} #tag downward limiting dependancies - wall = " " if dependency["req_version"].startswith("<") or dependency["req_version"].startswith("==") else "" + wall = " " if dependency["req_version"][:1] in("<", "~") or dependency["req_version"].startswith("==") else "" ret += self._get_dependency_tree( dependency["req_key"], up_req, @@ -249,7 +249,7 @@ def up(self, ppw: str, extra: str = "", depth: int = 20, indent: int = 5, versio pp = ppw[:-1] if ppw.endswith('!') else ppw ppend = "!" if ppw.endswith('!') else "" #show only downward limiting dependancies if pp == ".": - results = [self.up(p + ppend, extra, depth, indent, version_req, verbose) for p in sorted(self.distro)] + results = [aa:=self.up(p + ppend, extra, depth, indent, version_req, verbose) if '[requires :' in aa else None for p in sorted(self.distro)] return '\n'.join(filter(None, results)) if extra == ".": From af2bbae14693315e8819f44bb8f25782e3655826 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sun, 22 Jun 2025 09:08:05 +0200 Subject: [PATCH 37/37] tweak launcher creation --- portable/launchers_src/build_one_launcher.bat | 3 ++- portable/launchers_src_original/build_one_launcher.bat | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/portable/launchers_src/build_one_launcher.bat b/portable/launchers_src/build_one_launcher.bat index 233f68de..c48374c4 100644 --- a/portable/launchers_src/build_one_launcher.bat +++ b/portable/launchers_src/build_one_launcher.bat @@ -3,6 +3,7 @@ set icon_name=%1 set LAUNCH_TARGET=%2 set launcher_name=%3 set subsystem=%4 +set destination=%5 set icon_name=%icon_name:"=% set LAUNCH_TARGET=%LAUNCH_TARGET:"=% @@ -12,7 +13,7 @@ set subsystem=%subsystem:"=% set ROOT_PATH=%~dp0..\ set SCRIPT_PATH=%~dp0 set TEMPO_PATH=%ROOT_PATH%launchers_temp -set OUTPUT_DIR=%ROOT_PATH%launchers_final +set OUTPUT_DIR=%ROOT_PATH%launchers_%destination% set "ICON_FILE=%ROOT_PATH%icons\%icon_name%" set LAUNCHER_EXE=%OUTPUT_DIR%\%launcher_name%.exe diff --git a/portable/launchers_src_original/build_one_launcher.bat b/portable/launchers_src_original/build_one_launcher.bat index 233f68de..c48374c4 100644 --- a/portable/launchers_src_original/build_one_launcher.bat +++ b/portable/launchers_src_original/build_one_launcher.bat @@ -3,6 +3,7 @@ set icon_name=%1 set LAUNCH_TARGET=%2 set launcher_name=%3 set subsystem=%4 +set destination=%5 set icon_name=%icon_name:"=% set LAUNCH_TARGET=%LAUNCH_TARGET:"=% @@ -12,7 +13,7 @@ set subsystem=%subsystem:"=% set ROOT_PATH=%~dp0..\ set SCRIPT_PATH=%~dp0 set TEMPO_PATH=%ROOT_PATH%launchers_temp -set OUTPUT_DIR=%ROOT_PATH%launchers_final +set OUTPUT_DIR=%ROOT_PATH%launchers_%destination% set "ICON_FILE=%ROOT_PATH%icons\%icon_name%" set LAUNCHER_EXE=%OUTPUT_DIR%\%launcher_name%.exe