From f02a05a43897986504890d6a7616079f61017660 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Sun, 15 Sep 2024 14:47:12 +0200 Subject: [PATCH] Bugfix and enhance "setup_python.py" Bugfix fallback in `get_best_variant()` Check if local python executable is in PATH and warn if not. Don't download if again, if it's not in PATH. Check if "zstd" or "gzip" is installed for decompression and prefer "zstd" over "gzip". Handle different directory structures in archive: with and without `install` directory. --- README.md | 8 +- docs/setup_python.md | 14 ++- manageprojects/__init__.py | 2 +- manageprojects/setup_python.py | 95 +++++++++++++++---- .../tests/docwrite_macros_setup_python.py | 8 +- 5 files changed, 90 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 360540a..ceb1854 100644 --- a/README.md +++ b/README.md @@ -347,6 +347,8 @@ See also git tags: https://github.com/jedie/manageprojects/tags [comment]: <> (✂✂✂ auto generated history start ✂✂✂) +* [v0.19.1](https://github.com/jedie/manageprojects/compare/v0.19.0...v0.19.1) + * 2024-09-15 - Bugfix and enhance "setup_python.py" * [v0.19.0](https://github.com/jedie/manageprojects/compare/v0.18.0...v0.19.0) * 2024-09-15 - NEW: setup_python.py * 2024-09-15 - Update requirements @@ -369,15 +371,15 @@ See also git tags: https://github.com/jedie/manageprojects/tags * 2023-12-30 - Fix typos * [v0.17.1](https://github.com/jedie/manageprojects/compare/v0.17.0...v0.17.1) * 2023-12-29 - Still support Python v3.9 + +
Expand older history entries ... + * [v0.17.0](https://github.com/jedie/manageprojects/compare/v0.16.2...v0.17.0) * 2023-12-21 - Bugfix: Don't loose the "[manageprojects]" content on overwrite-update * 2023-12-21 - typing: Optional -> None * 2023-12-21 - Unify BASE_PATH / PACKAGE_ROOT etc. * 2023-12-21 - Apply manageprojects updates: Skip Python 3.9 support * 2023-12-21 - Update requirements - -
Expand older history entries ... - * [v0.16.2](https://github.com/jedie/manageprojects/compare/v0.16.1...v0.16.2) * 2023-12-16 - Update pre-commit-config * 2023-12-16 - Skip test_readme_history() on CI diff --git a/docs/setup_python.md b/docs/setup_python.md index 5576e1e..449496a 100644 --- a/docs/setup_python.md +++ b/docs/setup_python.md @@ -90,15 +90,12 @@ If the latest Python version is already installed, we skip the download. All downloads will be done with a secure connection (SSL) and server authentication. +We check if we have "zstd" or "gzip" installed for decompression and prefer "zstd" over "gzip". + If the latest Python version is already installed, we skip the download. Download will be done in a temporary directory. -We download the archive file and the hash file for verification: - -* Archive extension: `.tar.zst` -* Hash extension: `.tar.zst.sha256` - We check the file hash after downloading the archive. ## Workflow - 5. Add info JSON @@ -111,6 +108,13 @@ We add a shell script to `~/.local/bin/pythonX.XX` to start the Python interpret We display version information from Python and pip on `stderr`. +There exists two different directory structures: + +* `./python/install/bin/python3` +* `./python/bin/python3` + +We handle both cases and move all contents to the final destination. + The extracted Python will be moved to the final destination in `~/.local/pythonX.XX/`. The script set's the correct `PYTHONHOME` environment variable. diff --git a/manageprojects/__init__.py b/manageprojects/__init__.py index e70d607..f0f357a 100644 --- a/manageprojects/__init__.py +++ b/manageprojects/__init__.py @@ -3,5 +3,5 @@ Manage Python / Django projects """ -__version__ = '0.19.0' +__version__ = '0.19.1' __author__ = 'Jens Diemer ' diff --git a/manageprojects/setup_python.py b/manageprojects/setup_python.py index 87a2e55..e80c799 100644 --- a/manageprojects/setup_python.py +++ b/manageprojects/setup_python.py @@ -37,8 +37,7 @@ GUTHUB_PROJECT = 'indygreg/python-build-standalone' LASTEST_RELEASE_URL = f'https://raw.githubusercontent.com/{GUTHUB_PROJECT}/latest-release/latest-release.json' HASH_NAME = 'sha256' -ARCHIVE_EXTENSION = '.tar.zst' -ARCHIVE_HASH_EXTENSION = f'.tar.zst.{HASH_NAME}' + OPTIMIZATION_PRIORITY = ['pgo+lto', 'pgo', 'lto'] TEMP_PREFIX = 'redist_python_' @@ -221,7 +220,7 @@ def get_best_variant(names): if optimization in name: return name logger.warning('No optimization found in names: %r', names) - return names[0] + return sorted(names)[0] def get_python_version(python_bin: str | Path) -> str | None: @@ -232,6 +231,13 @@ def get_python_version(python_bin: str | Path) -> str | None: return full_version +def check_file_in_path(file_name: str): + if not shutil.which(file_name): + logger.error('Executable %r not found in PATH! (Hint: Add ~/.local/bin to PATH)', file_name) + else: + logger.info('Executable %r found in PATH, ok.', file_name) + + @dataclasses.dataclass class DownloadInfo: url: str @@ -248,9 +254,10 @@ def setup_python( The download will be only done, if the system Python is not the same major version as requested and if the local Python is not up-to-date. """ - logger.info('Requested major Python version: %s', major_version) + final_file_name = f'python{major_version}' + existing_version = None existing_python_bin = None """DocWrite: setup_python.md ## Workflow - 1. Check system Python @@ -271,13 +278,39 @@ def setup_python( continue logger.info('System Python v%s already installed: Return path %r of it.', existing_version, python3bin) + check_file_in_path(final_file_name) return Path(python3bin) else: logger.debug('%s not found, ok.', filename) + local_bin_path = Path.home() / '.local' / 'bin' / final_file_name + # Maybe ~/.local/bin/pythonX.XX is already installed, but ~/.local/bin/ is not in PATH: + if not existing_python_bin and local_bin_path.is_file(): + if existing_version := get_python_version(local_bin_path): + assert existing_version.startswith( + major_version + ), f'{existing_version=} does not start with {major_version=}' + existing_python_bin = local_bin_path + logger.debug('Existing Python version: %s', existing_version) - filters = [ARCHIVE_EXTENSION, *get_platform_parts()] + """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive + We check if we have "zstd" or "gzip" installed for decompression and prefer "zstd" over "gzip".""" + if shutil.which('zstd'): + logger.debug('zstd found, ok.') + compress_program = 'zstd' + compress_extension = 'zst' + elif shutil.which('gzip'): + logger.debug('gzip found, ok.') + compress_program = 'gzip' + compress_extension = 'gz' + else: + raise FileNotFoundError('"zstd" or "gzip" compress program not found!') + + archive_extension = f'.tar.{compress_extension}' + archive_hash_extension = f'.tar.{compress_extension}.{HASH_NAME}' + + filters = [archive_extension, *get_platform_parts()] logger.debug('Use filters: %s', filters) """DocWrite: setup_python.md ## Workflow - 2. Collect latest release data @@ -308,19 +341,15 @@ def setup_python( # Ignore incompatible assets continue - """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive - We download the archive file and the hash file for verification: - DocWriteMacro: manageprojects.tests.docwrite_macros_setup_python.extension_info - """ - if full_name.endswith(ARCHIVE_EXTENSION): - name = removesuffix(full_name, ARCHIVE_EXTENSION) + if full_name.endswith(archive_extension): + name = removesuffix(full_name, archive_extension) archive_infos[name] = DownloadInfo(url=asset['browser_download_url'], size=asset['size']) - elif full_name.endswith(ARCHIVE_HASH_EXTENSION): - name = removesuffix(full_name, ARCHIVE_HASH_EXTENSION) + elif full_name.endswith(archive_hash_extension): + name = removesuffix(full_name, archive_hash_extension) hash_urls[name] = asset['browser_download_url'] - assert archive_infos, f'No "{ARCHIVE_EXTENSION}" found in {assets=}' - assert hash_urls, f'No "{ARCHIVE_HASH_EXTENSION}" found in {assets=}' + assert archive_infos, f'No "{archive_extension}" found in {assets=}' + assert hash_urls, f'No "{archive_hash_extension}" found in {assets=}' assert archive_infos.keys() == hash_urls.keys(), f'{archive_infos.keys()=} != {hash_urls.keys()=}' @@ -342,6 +371,7 @@ def setup_python( if force_update: logger.info('Force update requested: Continue with download ...') else: + check_file_in_path(final_file_name) return Path(existing_python_bin) else: logger.warning('No version found in %r', best_variant) @@ -378,14 +408,34 @@ def setup_python( # Extract .tar.zstd archive file into temporary directory: logger.debug('Extract %s into %s ...', archive_temp_path, temp_path) run( - ['tar', '--use-compress-program=zstd', '--extract', '--file', archive_temp_path, '--directory', temp_path], + [ + 'tar', + f'--use-compress-program={compress_program}', + '--extract', + '--file', + archive_temp_path, + '--directory', + temp_path, + ], check=True, ) src_path = temp_path / 'python' assert_is_dir(src_path) - temp_python_path = src_path / 'install' / 'bin' / 'python3' + """DocWrite: setup_python.md ## Workflow - 6. Setup Python + There exists two different directory structures: + + * `./python/install/bin/python3` + * `./python/bin/python3` + + We handle both cases and move all contents to the final destination. + """ + has_install_dir = (src_path / 'install').is_dir() + if has_install_dir: + temp_python_path = src_path / 'install' / 'bin' / 'python3' + else: + temp_python_path = src_path / 'bin' / 'python3' assert_is_file(temp_python_path) python_version_info = verbose_check_output([str(temp_python_path), '-VV']).strip() @@ -411,21 +461,23 @@ def setup_python( """DocWrite: setup_python.md ## Workflow - 6. Setup Python The extracted Python will be moved to the final destination in `~/.local/pythonX.XX/`.""" - dest_path = Path.home() / '.local' / f'python{major_version}' + dest_path = Path.home() / '.local' / final_file_name logger.debug('Move %s to %s ...', src_path, dest_path) if dest_path.exists(): logger.info('Remove existing %r ...', dest_path) shutil.rmtree(dest_path) shutil.move(src_path, dest_path) - python_home_path = dest_path / 'install' + if has_install_dir: + python_home_path = dest_path / 'install' + else: + python_home_path = dest_path """DocWrite: setup_python.md ## Workflow - 6. Setup Python We add a shell script to `~/.local/bin/pythonX.XX` to start the Python interpreter.""" - bin_path = python_home_path / 'bin' / f'python{major_version}' + bin_path = python_home_path / 'bin' / final_file_name assert_is_file(bin_path) - local_bin_path = Path.home() / '.local' / 'bin' / f'python{major_version}' logger.debug('Create %s ...', local_bin_path) local_bin_path.parent.mkdir(parents=True, exist_ok=True) with local_bin_path.open('w') as f: @@ -446,6 +498,7 @@ def setup_python( print('Pip info:', verbose_check_output([str(local_bin_path), '-m', 'pip', '-VV']), file=sys.stderr) logger.info('Python v%s installed: Return path %r of it.', major_version, local_bin_path) + check_file_in_path(final_file_name) return local_bin_path diff --git a/manageprojects/tests/docwrite_macros_setup_python.py b/manageprojects/tests/docwrite_macros_setup_python.py index 3b2a1eb..ba7ec13 100644 --- a/manageprojects/tests/docwrite_macros_setup_python.py +++ b/manageprojects/tests/docwrite_macros_setup_python.py @@ -7,7 +7,7 @@ from bx_py_utils.path import assert_is_file from manageprojects import setup_python -from manageprojects.setup_python import ARCHIVE_EXTENSION, ARCHIVE_HASH_EXTENSION, LASTEST_RELEASE_URL +from manageprojects.setup_python import LASTEST_RELEASE_URL PROG = Path(setup_python.__file__).name @@ -47,9 +47,3 @@ def optimization_priority(macro_context: MacroContext): yield '' for number, optimization in enumerate(setup_python.OPTIMIZATION_PRIORITY, 1): yield f'{number}. `{optimization}`' - - -def extension_info(macro_context: MacroContext): - yield '' - yield f'* Archive extension: `{ARCHIVE_EXTENSION}`' - yield f'* Hash extension: `{ARCHIVE_HASH_EXTENSION}`'