Merge pull request #137 from jedie/dev

Bugfix and enhance "setup_python.py"

README.md

@@ -15,8 +15,8 @@ Besides this, `manageprojects` also includes other generic helper for Python pac
 
  * `publish_package()` - Build and upload a new release to PyPi, but with many pre-checks.
  * `format-file` - Format/Check a Python source file with Darker & Co., useful as IDE action.
- * `install_python.py` - [Install Python interpreter, if needed, from official Python FTP server, verified.](https://github.com/jedie/manageprojects/blob/main/docs/install_python.md)
- * `setup_python.py` - [Download and setup redistributable Python Interpreter, if needed.](https://github.com/jedie/manageprojects/blob/main/docs/setup_python.md)
+ * `install_python.py` - [One file and no dependencies to install Python, if needed, from official Python FTP server, verified.](https://github.com/jedie/manageprojects/blob/main/docs/install_python.md)
+ * `setup_python.py` - [One file and no dependencies to download and setup redistributable Python, if needed.](https://github.com/jedie/manageprojects/blob/main/docs/setup_python.md)
 
 Read below the `Helper` section.
 
@@ -347,6 +347,8 @@ See also git tags: https://github.com/jedie/manageprojects/tags
 
 [comment]: <> (✂✂✂ auto generated history start ✂✂✂)
 
+* [v0.19.1](https://github.com/jedie/manageprojects/compare/v0.19.0...v0.19.1)
+  * 2024-09-15 - Bugfix and enhance "setup_python.py"
 * [v0.19.0](https://github.com/jedie/manageprojects/compare/v0.18.0...v0.19.0)
   * 2024-09-15 - NEW: setup_python.py
   * 2024-09-15 - Update requirements
@@ -369,15 +371,15 @@ See also git tags: https://github.com/jedie/manageprojects/tags
   * 2023-12-30 - Fix typos
 * [v0.17.1](https://github.com/jedie/manageprojects/compare/v0.17.0...v0.17.1)
   * 2023-12-29 - Still support Python v3.9
+
+<details><summary>Expand older history entries ...</summary>
+
 * [v0.17.0](https://github.com/jedie/manageprojects/compare/v0.16.2...v0.17.0)
   * 2023-12-21 - Bugfix: Don't loose the "[manageprojects]" content on overwrite-update
   * 2023-12-21 - typing: Optional -> None
   * 2023-12-21 - Unify BASE_PATH / PACKAGE_ROOT etc.
   * 2023-12-21 - Apply manageprojects updates: Skip Python 3.9 support
   * 2023-12-21 - Update requirements
-
-<details><summary>Expand older history entries ...</summary>
-
 * [v0.16.2](https://github.com/jedie/manageprojects/compare/v0.16.1...v0.16.2)
   * 2023-12-16 - Update pre-commit-config
   * 2023-12-16 - Skip test_readme_history() on CI

docs/setup_python.md

@@ -1,7 +1,7 @@
 # Boot Redistributable Python
 
-This is a standalone script (no dependencies) to download and setup
-https://github.com/indygreg/python-build-standalone/ redistributable Python interpreter.
+This is a standalone script (one file and no dependencies) to download and setup
+https://github.com/indygreg/python-build-standalone/ redistributable Python.
 But only if it's needed!
 
 Minimal version to used this script is Python v3.9.
@@ -90,15 +90,12 @@ If the latest Python version is already installed, we skip the download.
 
 All downloads will be done with a secure connection (SSL) and server authentication.
 
+We check if we have "zstd" or "gzip" installed for decompression and prefer "zstd" over "gzip".
+
 If the latest Python version is already installed, we skip the download.
 
 Download will be done in a temporary directory.
 
-We download the archive file and the hash file for verification:
-
-* Archive extension: `.tar.zst`
-* Hash extension: `.tar.zst.sha256`
-
 We check the file hash after downloading the archive.
 
 ## Workflow - 5. Add info JSON
@@ -111,6 +108,13 @@ We add a shell script to `~/.local/bin/pythonX.XX` to start the Python interpret
 
 We display version information from Python and pip on `stderr`.
 
+There exists two different directory structures:
+
+* `./python/install/bin/python3`
+* `./python/bin/python3`
+
+We handle both cases and move all contents to the final destination.
+
 The extracted Python will be moved to the final destination in `~/.local/pythonX.XX/`.
 
 The script set's the correct `PYTHONHOME` environment variable.

manageprojects/__init__.py

@@ -3,5 +3,5 @@
     Manage Python / Django projects
 """
 
-__version__ = '0.19.0'
+__version__ = '0.19.1'
 __author__ = 'Jens Diemer <[email protected]>'

manageprojects/setup_python.py

@@ -3,8 +3,8 @@
 """
     DocWrite: setup_python.md # Boot Redistributable Python
 
-    This is a standalone script (no dependencies) to download and setup
-    https://github.com/indygreg/python-build-standalone/ redistributable Python interpreter.
+    This is a standalone script (one file and no dependencies) to download and setup
+    https://github.com/indygreg/python-build-standalone/ redistributable Python.
     But only if it's needed!
 """
 from __future__ import annotations
@@ -37,8 +37,7 @@
 GUTHUB_PROJECT = 'indygreg/python-build-standalone'
 LASTEST_RELEASE_URL = f'https://raw.githubusercontent.com/{GUTHUB_PROJECT}/latest-release/latest-release.json'
 HASH_NAME = 'sha256'
-ARCHIVE_EXTENSION = '.tar.zst'
-ARCHIVE_HASH_EXTENSION = f'.tar.zst.{HASH_NAME}'
+
 
 OPTIMIZATION_PRIORITY = ['pgo+lto', 'pgo', 'lto']
 TEMP_PREFIX = 'redist_python_'
@@ -221,7 +220,7 @@ def get_best_variant(names):
             if optimization in name:
                 return name
     logger.warning('No optimization found in names: %r', names)
-    return names[0]
+    return sorted(names)[0]
 
 
 def get_python_version(python_bin: str | Path) -> str | None:
@@ -232,6 +231,13 @@ def get_python_version(python_bin: str | Path) -> str | None:
         return full_version
 
 
+def check_file_in_path(file_name: str):
+    if not shutil.which(file_name):
+        logger.error('Executable %r not found in PATH! (Hint: Add ~/.local/bin to PATH)', file_name)
+    else:
+        logger.info('Executable %r found in PATH, ok.', file_name)
+
+
 @dataclasses.dataclass
 class DownloadInfo:
     url: str
@@ -248,9 +254,10 @@ def setup_python(
     The download will be only done, if the system Python is not the same major version as requested
     and if the local Python is not up-to-date.
     """
-
     logger.info('Requested major Python version: %s', major_version)
 
+    final_file_name = f'python{major_version}'
+
     existing_version = None
     existing_python_bin = None
     """DocWrite: setup_python.md ## Workflow - 1. Check system Python
@@ -271,13 +278,39 @@ def setup_python(
                     continue
 
                 logger.info('System Python v%s already installed: Return path %r of it.', existing_version, python3bin)
+                check_file_in_path(final_file_name)
                 return Path(python3bin)
         else:
             logger.debug('%s not found, ok.', filename)
 
+    local_bin_path = Path.home() / '.local' / 'bin' / final_file_name
+    # Maybe ~/.local/bin/pythonX.XX is already installed, but ~/.local/bin/ is not in PATH:
+    if not existing_python_bin and local_bin_path.is_file():
+        if existing_version := get_python_version(local_bin_path):
+            assert existing_version.startswith(
+                major_version
+            ), f'{existing_version=} does not start with {major_version=}'
+            existing_python_bin = local_bin_path
+
     logger.debug('Existing Python version: %s', existing_version)
 
-    filters = [ARCHIVE_EXTENSION, *get_platform_parts()]
+    """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive
+    We check if we have "zstd" or "gzip" installed for decompression and prefer "zstd" over "gzip"."""
+    if shutil.which('zstd'):
+        logger.debug('zstd found, ok.')
+        compress_program = 'zstd'
+        compress_extension = 'zst'
+    elif shutil.which('gzip'):
+        logger.debug('gzip found, ok.')
+        compress_program = 'gzip'
+        compress_extension = 'gz'
+    else:
+        raise FileNotFoundError('"zstd" or "gzip" compress program not found!')
+
+    archive_extension = f'.tar.{compress_extension}'
+    archive_hash_extension = f'.tar.{compress_extension}.{HASH_NAME}'
+
+    filters = [archive_extension, *get_platform_parts()]
     logger.debug('Use filters: %s', filters)
 
     """DocWrite: setup_python.md ## Workflow - 2. Collect latest release data
@@ -308,19 +341,15 @@ def setup_python(
             # Ignore incompatible assets
             continue
 
-        """DocWrite: setup_python.md ## Workflow - 4. Download and verify Archive
-        We download the archive file and the hash file for verification:
-        DocWriteMacro: manageprojects.tests.docwrite_macros_setup_python.extension_info
-        """
-        if full_name.endswith(ARCHIVE_EXTENSION):
-            name = removesuffix(full_name, ARCHIVE_EXTENSION)
+        if full_name.endswith(archive_extension):
+            name = removesuffix(full_name, archive_extension)
             archive_infos[name] = DownloadInfo(url=asset['browser_download_url'], size=asset['size'])
-        elif full_name.endswith(ARCHIVE_HASH_EXTENSION):
-            name = removesuffix(full_name, ARCHIVE_HASH_EXTENSION)
+        elif full_name.endswith(archive_hash_extension):
+            name = removesuffix(full_name, archive_hash_extension)
             hash_urls[name] = asset['browser_download_url']
 
-    assert archive_infos, f'No "{ARCHIVE_EXTENSION}" found in {assets=}'
-    assert hash_urls, f'No "{ARCHIVE_HASH_EXTENSION}" found in {assets=}'
+    assert archive_infos, f'No "{archive_extension}" found in {assets=}'
+    assert hash_urls, f'No "{archive_hash_extension}" found in {assets=}'
 
     assert archive_infos.keys() == hash_urls.keys(), f'{archive_infos.keys()=} != {hash_urls.keys()=}'
 
@@ -342,6 +371,7 @@ def setup_python(
                 if force_update:
                     logger.info('Force update requested: Continue with download ...')
                 else:
+                    check_file_in_path(final_file_name)
                     return Path(existing_python_bin)
         else:
             logger.warning('No version found in %r', best_variant)
@@ -378,14 +408,34 @@ def setup_python(
         # Extract .tar.zstd archive file into temporary directory:
         logger.debug('Extract %s into %s ...', archive_temp_path, temp_path)
         run(
-            ['tar', '--use-compress-program=zstd', '--extract', '--file', archive_temp_path, '--directory', temp_path],
+            [
+                'tar',
+                f'--use-compress-program={compress_program}',
+                '--extract',
+                '--file',
+                archive_temp_path,
+                '--directory',
+                temp_path,
+            ],
             check=True,
         )
 
         src_path = temp_path / 'python'
         assert_is_dir(src_path)
 
-        temp_python_path = src_path / 'install' / 'bin' / 'python3'
+        """DocWrite: setup_python.md ## Workflow - 6. Setup Python
+        There exists two different directory structures:
+
+        * `./python/install/bin/python3`
+        * `./python/bin/python3`
+
+        We handle both cases and move all contents to the final destination.
+        """
+        has_install_dir = (src_path / 'install').is_dir()
+        if has_install_dir:
+            temp_python_path = src_path / 'install' / 'bin' / 'python3'
+        else:
+            temp_python_path = src_path / 'bin' / 'python3'
         assert_is_file(temp_python_path)
 
         python_version_info = verbose_check_output([str(temp_python_path), '-VV']).strip()
@@ -411,21 +461,23 @@ def setup_python(
 
         """DocWrite: setup_python.md ## Workflow - 6. Setup Python
         The extracted Python will be moved to the final destination in `~/.local/pythonX.XX/`."""
-        dest_path = Path.home() / '.local' / f'python{major_version}'
+        dest_path = Path.home() / '.local' / final_file_name
         logger.debug('Move %s to %s ...', src_path, dest_path)
         if dest_path.exists():
             logger.info('Remove existing %r ...', dest_path)
             shutil.rmtree(dest_path)
         shutil.move(src_path, dest_path)
 
-    python_home_path = dest_path / 'install'
+    if has_install_dir:
+        python_home_path = dest_path / 'install'
+    else:
+        python_home_path = dest_path
 
     """DocWrite: setup_python.md ## Workflow - 6. Setup Python
     We add a shell script to `~/.local/bin/pythonX.XX` to start the Python interpreter."""
-    bin_path = python_home_path / 'bin' / f'python{major_version}'
+    bin_path = python_home_path / 'bin' / final_file_name
     assert_is_file(bin_path)
 
-    local_bin_path = Path.home() / '.local' / 'bin' / f'python{major_version}'
     logger.debug('Create %s ...', local_bin_path)
     local_bin_path.parent.mkdir(parents=True, exist_ok=True)
     with local_bin_path.open('w') as f:
@@ -446,6 +498,7 @@ def setup_python(
     print('Pip info:', verbose_check_output([str(local_bin_path), '-m', 'pip', '-VV']), file=sys.stderr)
 
     logger.info('Python v%s installed: Return path %r of it.', major_version, local_bin_path)
+    check_file_in_path(final_file_name)
     return local_bin_path
 
 

manageprojects/tests/docwrite_macros_setup_python.py

@@ -7,7 +7,7 @@
 from bx_py_utils.path import assert_is_file
 
 from manageprojects import setup_python
-from manageprojects.setup_python import ARCHIVE_EXTENSION, ARCHIVE_HASH_EXTENSION, LASTEST_RELEASE_URL
+from manageprojects.setup_python import LASTEST_RELEASE_URL
 
 
 PROG = Path(setup_python.__file__).name
@@ -47,9 +47,3 @@ def optimization_priority(macro_context: MacroContext):
     yield ''
     for number, optimization in enumerate(setup_python.OPTIMIZATION_PRIORITY, 1):
         yield f'{number}. `{optimization}`'
-
-
-def extension_info(macro_context: MacroContext):
-    yield ''
-    yield f'* Archive extension: `{ARCHIVE_EXTENSION}`'
-    yield f'* Hash extension: `{ARCHIVE_HASH_EXTENSION}`'