diff --git a/README.md b/README.md index 360540a..d09e1c3 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ Besides this, `manageprojects` also includes other generic helper for Python pac * `publish_package()` - Build and upload a new release to PyPi, but with many pre-checks. * `format-file` - Format/Check a Python source file with Darker & Co., useful as IDE action. - * `install_python.py` - [Install Python interpreter, if needed, from official Python FTP server, verified.](https://github.com/jedie/manageprojects/blob/main/docs/install_python.md) - * `setup_python.py` - [Download and setup redistributable Python Interpreter, if needed.](https://github.com/jedie/manageprojects/blob/main/docs/setup_python.md) + * `install_python.py` - [One file and no dependencies to install Python, if needed, from official Python FTP server, verified.](https://github.com/jedie/manageprojects/blob/main/docs/install_python.md) + * `setup_python.py` - [One file and no dependencies to download and setup redistributable Python, if needed.](https://github.com/jedie/manageprojects/blob/main/docs/setup_python.md) Read below the `Helper` section. @@ -347,6 +347,8 @@ See also git tags: https://github.com/jedie/manageprojects/tags [comment]: <> (✂✂✂ auto generated history start ✂✂✂) +* [v0.19.1](https://github.com/jedie/manageprojects/compare/v0.19.0...v0.19.1) + * 2024-09-15 - Bugfix and enhance "setup_python.py" * [v0.19.0](https://github.com/jedie/manageprojects/compare/v0.18.0...v0.19.0) * 2024-09-15 - NEW: setup_python.py * 2024-09-15 - Update requirements @@ -369,15 +371,15 @@ See also git tags: https://github.com/jedie/manageprojects/tags * 2023-12-30 - Fix typos * [v0.17.1](https://github.com/jedie/manageprojects/compare/v0.17.0...v0.17.1) * 2023-12-29 - Still support Python v3.9 + +
Expand older history entries ... + * [v0.17.0](https://github.com/jedie/manageprojects/compare/v0.16.2...v0.17.0) * 2023-12-21 - Bugfix: Don't loose the "[manageprojects]" content on overwrite-update * 2023-12-21 - typing: Optional -> None * 2023-12-21 - Unify BASE_PATH / PACKAGE_ROOT etc. * 2023-12-21 - Apply manageprojects updates: Skip Python 3.9 support * 2023-12-21 - Update requirements - -
Expand older history entries ... - * [v0.16.2](https://github.com/jedie/manageprojects/compare/v0.16.1...v0.16.2) * 2023-12-16 - Update pre-commit-config * 2023-12-16 - Skip test_readme_history() on CI diff --git a/docs/setup_python.md b/docs/setup_python.md index 5576e1e..f1bf5b6 100644 --- a/docs/setup_python.md +++ b/docs/setup_python.md @@ -1,7 +1,7 @@ # Boot Redistributable Python -This is a standalone script (no dependencies) to download and setup -https://github.com/indygreg/python-build-standalone/ redistributable Python interpreter. +This is a standalone script (one file and no dependencies) to download and setup +https://github.com/indygreg/python-build-standalone/ redistributable Python. But only if it's needed! Minimal version to used this script is Python v3.9. @@ -90,15 +90,12 @@ If the latest Python version is already installed, we skip the download. All downloads will be done with a secure connection (SSL) and server authentication. +We check if we have "zstd" or "gzip" installed for decompression and prefer "zstd" over "gzip". + If the latest Python version is already installed, we skip the download. Download will be done in a temporary directory. -We download the archive file and the hash file for verification: - -* Archive extension: `.tar.zst` -* Hash extension: `.tar.zst.sha256` - We check the file hash after downloading the archive. ## Workflow - 5. Add info JSON @@ -111,6 +108,13 @@ We add a shell script to `~/.local/bin/pythonX.XX` to start the Python interpret We display version information from Python and pip on `stderr`. +There exists two different directory structures: + +* `./python/install/bin/python3` +* `./python/bin/python3` + +We handle both cases and move all contents to the final destination. + The extracted Python will be moved to the final destination in `~/.local/pythonX.XX/`. The script set's the correct `PYTHONHOME` environment variable. diff --git a/manageprojects/__init__.py b/manageprojects/__init__.py index e70d607..f0f357a 100644 --- a/manageprojects/__init__.py +++ b/manageprojects/__init__.py @@ -3,5 +3,5 @@ Manage Python / Django projects """ -__version__ = '0.19.0' +__version__ = '0.19.1' __author__ = 'Jens Diemer ' diff --git a/manageprojects/setup_python.py b/manageprojects/setup_python.py index 87a2e55..c65f60b 100644 --- a/manageprojects/setup_python.py +++ b/manageprojects/setup_python.py @@ -3,8 +3,8 @@ """ DocWrite: setup_python.md # Boot Redistributable Python - This is a standalone script (no dependencies) to download and setup - https://github.com/indygreg/python-build-standalone/ redistributable Python interpreter. + This is a standalone script (one file and no dependencies) to download and setup + https://github.com/indygreg/python-build-standalone/ redistributable Python. But only if it's needed! """ from __future__ import annotations @@ -37,8 +37,7 @@ GUTHUB_PROJECT = 'indygreg/python-build-standalone' LASTEST_RELEASE_URL = f'https://raw.githubusercontent.com/{GUTHUB_PROJECT}/latest-release/latest-release.json' HASH_NAME = 'sha256' -ARCHIVE_EXTENSION = '.tar.zst' -ARCHIVE_HASH_EXTENSION = f'.tar.zst.{HASH_NAME}' + OPTIMIZATION_PRIORITY = ['pgo+lto', 'pgo', 'lto'] TEMP_PREFIX = 'redist_python_' @@ -221,7 +220,7 @@ def get_best_variant(names): if optimization in name: return name logger.warning('No optimization found in names: %r', names) - return names[0] + return sorted(names)[0] def get_python_version(python_bin: str | Path) -> str | None: @@ -232,6 +231,13 @@ def get_python_version(python_bin: str | Path) -> str | None: return full_version +def check_file_in_path(file_name: str): + if not shutil.which(file_name): + logger.error('Executable %r not found in PATH! (Hint: Add ~/.local/bin to PATH)', file_name) + else: + logger.info('Executable %r found in PATH, ok.', file_name) + + @dataclasses.dataclass class DownloadInfo: url: str @@ -248,9 +254,10 @@ def setup_python( The download will be only done, if the system Python is not the same major version as requested and if the local Python is not up-to-date. """ - logger.info('Requested major Python version: %s', major_version) + final_file_name = f'python{major_version}' + existing_version = None existing_python_bin = None """DocWrite: setup_python.md ## Workflow - 1. Check system Python @@ -271,13 +278,39 @@ def setup_python( continue logger.info('System Python v%s already installed: Return path %r of it.', existing_version, python3bin) + check_file_in_path(final_file_name) return Path(python3bin) else: logger.debug('%s not found, ok.', filename) + local_bin_path = Path.home() / '.local' / 'bin' / final_file_name + # Maybe ~/.local/bin/pythonX.XX is already installed, but ~/.local/bin/ is not in PATH: + if not existing_python_bin and local_bin_path.is_file(): + if existing_version := get_python_version(local_bin_path): + assert existing_version.startswith( + major_version + ), f'{existing_version=} does not start with {major_version=}' + existing_python_bin = local_bin_path + logger.debug('Existing Python version: %s', existing_version) - filters = [ARCHIVE_EXTENSION, *get_platform_parts()] + """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive + We check if we have "zstd" or "gzip" installed for decompression and prefer "zstd" over "gzip".""" + if shutil.which('zstd'): + logger.debug('zstd found, ok.') + compress_program = 'zstd' + compress_extension = 'zst' + elif shutil.which('gzip'): + logger.debug('gzip found, ok.') + compress_program = 'gzip' + compress_extension = 'gz' + else: + raise FileNotFoundError('"zstd" or "gzip" compress program not found!') + + archive_extension = f'.tar.{compress_extension}' + archive_hash_extension = f'.tar.{compress_extension}.{HASH_NAME}' + + filters = [archive_extension, *get_platform_parts()] logger.debug('Use filters: %s', filters) """DocWrite: setup_python.md ## Workflow - 2. Collect latest release data @@ -308,19 +341,15 @@ def setup_python( # Ignore incompatible assets continue - """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive - We download the archive file and the hash file for verification: - DocWriteMacro: manageprojects.tests.docwrite_macros_setup_python.extension_info - """ - if full_name.endswith(ARCHIVE_EXTENSION): - name = removesuffix(full_name, ARCHIVE_EXTENSION) + if full_name.endswith(archive_extension): + name = removesuffix(full_name, archive_extension) archive_infos[name] = DownloadInfo(url=asset['browser_download_url'], size=asset['size']) - elif full_name.endswith(ARCHIVE_HASH_EXTENSION): - name = removesuffix(full_name, ARCHIVE_HASH_EXTENSION) + elif full_name.endswith(archive_hash_extension): + name = removesuffix(full_name, archive_hash_extension) hash_urls[name] = asset['browser_download_url'] - assert archive_infos, f'No "{ARCHIVE_EXTENSION}" found in {assets=}' - assert hash_urls, f'No "{ARCHIVE_HASH_EXTENSION}" found in {assets=}' + assert archive_infos, f'No "{archive_extension}" found in {assets=}' + assert hash_urls, f'No "{archive_hash_extension}" found in {assets=}' assert archive_infos.keys() == hash_urls.keys(), f'{archive_infos.keys()=} != {hash_urls.keys()=}' @@ -342,6 +371,7 @@ def setup_python( if force_update: logger.info('Force update requested: Continue with download ...') else: + check_file_in_path(final_file_name) return Path(existing_python_bin) else: logger.warning('No version found in %r', best_variant) @@ -378,14 +408,34 @@ def setup_python( # Extract .tar.zstd archive file into temporary directory: logger.debug('Extract %s into %s ...', archive_temp_path, temp_path) run( - ['tar', '--use-compress-program=zstd', '--extract', '--file', archive_temp_path, '--directory', temp_path], + [ + 'tar', + f'--use-compress-program={compress_program}', + '--extract', + '--file', + archive_temp_path, + '--directory', + temp_path, + ], check=True, ) src_path = temp_path / 'python' assert_is_dir(src_path) - temp_python_path = src_path / 'install' / 'bin' / 'python3' + """DocWrite: setup_python.md ## Workflow - 6. Setup Python + There exists two different directory structures: + + * `./python/install/bin/python3` + * `./python/bin/python3` + + We handle both cases and move all contents to the final destination. + """ + has_install_dir = (src_path / 'install').is_dir() + if has_install_dir: + temp_python_path = src_path / 'install' / 'bin' / 'python3' + else: + temp_python_path = src_path / 'bin' / 'python3' assert_is_file(temp_python_path) python_version_info = verbose_check_output([str(temp_python_path), '-VV']).strip() @@ -411,21 +461,23 @@ def setup_python( """DocWrite: setup_python.md ## Workflow - 6. Setup Python The extracted Python will be moved to the final destination in `~/.local/pythonX.XX/`.""" - dest_path = Path.home() / '.local' / f'python{major_version}' + dest_path = Path.home() / '.local' / final_file_name logger.debug('Move %s to %s ...', src_path, dest_path) if dest_path.exists(): logger.info('Remove existing %r ...', dest_path) shutil.rmtree(dest_path) shutil.move(src_path, dest_path) - python_home_path = dest_path / 'install' + if has_install_dir: + python_home_path = dest_path / 'install' + else: + python_home_path = dest_path """DocWrite: setup_python.md ## Workflow - 6. Setup Python We add a shell script to `~/.local/bin/pythonX.XX` to start the Python interpreter.""" - bin_path = python_home_path / 'bin' / f'python{major_version}' + bin_path = python_home_path / 'bin' / final_file_name assert_is_file(bin_path) - local_bin_path = Path.home() / '.local' / 'bin' / f'python{major_version}' logger.debug('Create %s ...', local_bin_path) local_bin_path.parent.mkdir(parents=True, exist_ok=True) with local_bin_path.open('w') as f: @@ -446,6 +498,7 @@ def setup_python( print('Pip info:', verbose_check_output([str(local_bin_path), '-m', 'pip', '-VV']), file=sys.stderr) logger.info('Python v%s installed: Return path %r of it.', major_version, local_bin_path) + check_file_in_path(final_file_name) return local_bin_path diff --git a/manageprojects/tests/docwrite_macros_setup_python.py b/manageprojects/tests/docwrite_macros_setup_python.py index 3b2a1eb..ba7ec13 100644 --- a/manageprojects/tests/docwrite_macros_setup_python.py +++ b/manageprojects/tests/docwrite_macros_setup_python.py @@ -7,7 +7,7 @@ from bx_py_utils.path import assert_is_file from manageprojects import setup_python -from manageprojects.setup_python import ARCHIVE_EXTENSION, ARCHIVE_HASH_EXTENSION, LASTEST_RELEASE_URL +from manageprojects.setup_python import LASTEST_RELEASE_URL PROG = Path(setup_python.__file__).name @@ -47,9 +47,3 @@ def optimization_priority(macro_context: MacroContext): yield '' for number, optimization in enumerate(setup_python.OPTIMIZATION_PRIORITY, 1): yield f'{number}. `{optimization}`' - - -def extension_info(macro_context: MacroContext): - yield '' - yield f'* Archive extension: `{ARCHIVE_EXTENSION}`' - yield f'* Hash extension: `{ARCHIVE_HASH_EXTENSION}`'