diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index dd7066cd..b8b4fadd 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -5,7 +5,7 @@ name: Upload Python Package on: release: - types: [created, edited, published, released] + types: [published] jobs: deploy: diff --git a/.github/workflows/test_cpac.yml b/.github/workflows/test_cpac.yml index f82c94e1..df4be269 100644 --- a/.github/workflows/test_cpac.yml +++ b/.github/workflows/test_cpac.yml @@ -19,10 +19,10 @@ jobs: strategy: matrix: - platform: [docker, singularity] - tag: [latest, dev-v1.8] + platform: [docker] + tag: [latest, nightly] go: [1.14] - python: [3.6, 3.7, 3.8] + python: [3.7, 3.8, 3.9, "3.10"] singularity: [3.6.4] steps: @@ -39,7 +39,7 @@ jobs: run: python -m pip install --upgrade pip setuptools wheel - name: Set up Singularity if: ${{ matrix.platform == 'singularity' }} - uses: eWaterCycle/setup-singularity@v5 + uses: eWaterCycle/setup-singularity@v7 with: singularity-version: ${{ matrix.singularity }} - name: Prepare Singularity cache and tmp directories @@ -47,14 +47,13 @@ jobs: run: mkdir -p ${SINGULARITY_CACHEDIR} && mkdir -p ${SINGULARITY_TMPDIR} - name: Install dependencies run: | - sudo apt-get install libarchive-dev \ - libffi-dev \ - flawfinder \ - libgpgme11-dev \ - libseccomp-dev \ - squashfs-tools \ - libssl1.1 libssl-dev \ - libuuid1 uuid-dev + sudo apt-get install libffi-dev \ + flawfinder \ + libgpgme11-dev \ + libseccomp-dev \ + squashfs-tools \ + libssl1.1 libssl-dev \ + libuuid1 uuid-dev pip install -r requirements.txt pip install -r requirements-dev.txt pip install coverage coveralls nipype @@ -62,22 +61,22 @@ jobs: run: cd $GITHUB_WORKSPACE && pip install -e . - name: Test cpac, platform and tag specified run: | - coverage run --append -m pytest --doctest-modules --platform ${{ matrix.platform }} --tag ${{ matrix.tag }} . + coverage run --append -m pytest --basetemp=${PWD}/tmp --doctest-modules --platform ${{ matrix.platform }} --tag ${{ matrix.tag }} . coverage report -m - name: Test cpac, platform specified, tag unspecified if: ${{ matrix.tag == 'latest' }} run: | - coverage run --append -m pytest --doctest-modules --platform ${{ matrix.platform }} . + coverage run --append -m pytest --basetemp=${PWD}/tmp --doctest-modules --platform ${{ matrix.platform }} . coverage report -m - name: Test cpac, platform unspecified, tag specified if: ${{ matrix.platform == 'docker' }} run: | - coverage run --append -m pytest --doctest-modules --tag ${{ matrix.tag }} . + coverage run --append -m pytest --basetemp=${PWD}/tmp --doctest-modules --tag ${{ matrix.tag }} . coverage report -m - name: Test cpac, platform and tag unspecified if: ${{ matrix.platform == 'docker' }} && ${{ matrix.tag }} == 'latest' run: | - coverage run --append -m pytest --doctest-modules . + coverage run --append -m pytest --basetemp=${PWD}/tmp --doctest-modules . coverage report -m - name: Report coverage uses: AndreMiras/coveralls-python-action@v20201129 @@ -102,6 +101,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Install cpac run: cd $GITHUB_WORKSPACE && pip install . - name: Configure Git credentials diff --git a/.gitignore b/.gitignore index b457ca9f..5a226a96 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ __pycache__/* .cache/* .*.swp */.ipynb_checkpoints/* +tmp # Project files .ropeproject @@ -49,4 +50,5 @@ MANIFEST .venv*/ # Singularity Images -**/*.simg \ No newline at end of file +**/*.simg +**/*.sif \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8743bf51..4afaa0d2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,19 @@ ========= Changelog ========= +`Version 0.5.0: Parse Resources `_ +================================================================================================ +* 🧮 Evaluates memory usage for specific nodes from ``callback.log`` files +* ✨ Adds ``enter`` command to enter a container's ``BASH`` +* ✨ Adds ``version`` command to give version of in-container C-PAC +* ✨ Adds ``parse-resources`` command to parse resources from ``callback.log`` +* 🐛 Fixes issue where ``--version`` command was not working +* 🐛 Fixes issue where custom pipeline configurations were not binding to temporary container prior to checking for bind paths +* ✅ Updates for tests that were failing +* 📝 Add known issues to usage string +* ⬆ Require Python ≥ 3.7 (for typing annotations) +* 📝 Documents support for Singularity 3 + `Version 0.4.0: Goodbye Singularity Hub `_ ================================================================================================ * 👽 Drop call to now-deprecated Singularity Hub diff --git a/README.rst b/README.rst index 4834b225..23c326c8 100644 --- a/README.rst +++ b/README.rst @@ -14,12 +14,12 @@ C-PAC Python Package is a lightweight Python package that handles interfacing a Dependencies ============ -* `Python `_ ≥3.6 +* `Python `_ ≥3.7 * `pip `_ * At least one of: * `Docker `_ - * `Singularity `_ ≥2.5&≤3.0 + * `Singularity `_ ≥2.5 Usage ===== @@ -32,7 +32,8 @@ Usage usage: cpac [-h] [--version] [-o OPT] [-B CUSTOM_BINDING] [--platform {docker,singularity}] [--image IMAGE] [--tag TAG] [--working_dir PATH] [-v] [-vv] - {run,group,utils,pull,upgrade,crash} ... + {run,utils,version,group,pull,upgrade,enter,bash,shell,parse-resources,parse_resources,crash} + ... cpac: a Python package that simplifies using C-PAC containerized images. @@ -57,8 +58,40 @@ Usage cpac run --help + Known issues: + - Some Docker containers unexpectedly persist after cpac finishes. To clear them, run + 1. `docker ps` to list the containers + For each C-PAC conatainer that persists, run + 2. `docker attach ` + 3. `exit` + - https://github.com/FCP-INDI/cpac/issues + positional arguments: - {run,group,utils,pull,upgrade,crash} + {run,utils,version,group,pull,upgrade,enter,bash,shell,parse-resources,parse_resources,crash} + run Run C-PAC. See + "cpac [--platform {docker,singularity}] [--image IMAGE] [--tag TAG] run --help" + for more information. + utils Run C-PAC commandline utilities. See + "cpac [--platform {docker,singularity}] [--image IMAGE] [--tag TAG] utils --help" + for more information. + version Print the version of C-PAC that cpac is using. + group Run a group level analysis in C-PAC. See + "cpac [--platform {docker,singularity}] [--image IMAGE] [--tag TAG] group --help" + for more information. + pull (upgrade) Upgrade your local C-PAC version to the latest version + by pulling from Docker Hub or other repository. + Use with "--image" and/or "--tag" to specify an image + other than the default "fcpindi/c-pac:latest" to pull. + enter (bash, shell) + Enter a new C-PAC container via BASH. + parse-resources (parse_resources) + When provided with a `callback.log` file, this utility can sort through + the memory `runtime` usage, `estimate`, and associated `efficiency`, to + identify the `n` tasks with the `highest` or `lowest` of each of these + categories. + "parse-resources" is intended to be run outside a C-PAC container. + See "cpac parse-resources --help" for more information. + crash Convert a crash pickle to plain text (C-PAC < 1.8.0). optional arguments: -h, --help show this help message and exit diff --git a/requirements.txt b/requirements.txt index 5b24cf58..969a7576 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ docker >= 4.2.1 +dockerpty docker-pycreds pandas >= 0.23.4 pyyaml @@ -6,4 +7,5 @@ setuptools spython >= 0.0.81 tabulate >= 0.8.6 tornado -websocket-client \ No newline at end of file +websocket-client +rich diff --git a/setup.cfg b/setup.cfg index 8aba9d0c..438824e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ classifiers = Operating System :: OS Independent Programming Language :: Python :: 3 Topic :: Scientific/Engineering :: Bio-Informatics -version = 0.4.0 +version = 0.5.0 [options] zip_safe = False @@ -37,14 +37,16 @@ setup_requires = pyyaml install_requires = docker + dockerpty docker-pycreds pandas >= 0.23.4 spython >= 0.0.81 pyyaml + rich tabulate >= 0.8.6 tornado websocket-client -python_requires = >=3.6 +python_requires = >=3.7 [options.packages.find] where = src diff --git a/src/cpac/__main__.py b/src/cpac/__main__.py index 06097a43..62e166aa 100644 --- a/src/cpac/__main__.py +++ b/src/cpac/__main__.py @@ -11,6 +11,7 @@ from cpac import __version__ from cpac.backends import Backends +from cpac.helpers import cpac_parse_resources as parse_resources, TODOs _logger = logging.getLogger(__name__) @@ -60,7 +61,10 @@ def _parser(): '--data_config_file /configs/data_config.yml \\\n\t\t' '--save_working_dir\n\n' 'Each command can take "--help" to provide additonal ' - 'usage information, e.g.,\n\n\tcpac run --help', + 'usage information, e.g.,\n\n\tcpac run --help\n\n' + 'Known issues:\n' + + '\n'.join([f'- {todo}' for todo in TODOs.values()]) + + '\n- https://github.com/FCP-INDI/cpac/issues', conflict_handler='resolve', formatter_class=argparse.RawTextHelpFormatter ) @@ -68,7 +72,10 @@ def _parser(): parser.add_argument( '--version', action='version', - version='cpac {ver}'.format(ver=__version__) + version='cpac (convenience wrapper) version {ver}\nFor C-PAC version, ' + 'run `cpac version` with any cpac options (e.g., ' + '`--platform`, `--image`, `--tag`) that you would use ' + 'while running'.format(ver=__version__) ) parser.add_argument( @@ -144,13 +151,26 @@ def _parser(): subparsers = parser.add_subparsers(dest='command') run_parser = subparsers.add_parser( - 'run', - add_help=False, + 'run', add_help=False, + help='Run C-PAC. See\n"cpac [--platform {docker,singularity}] ' + '[--image IMAGE] [--tag TAG] run --help"\nfor more ' + 'information.', formatter_class=argparse.ArgumentDefaultsHelpFormatter ) + utils_parser = subparsers.add_parser( + 'utils', add_help=False, + help='Run C-PAC commandline utilities. See\n"cpac [--platform ' + '{docker,singularity}] [--image IMAGE] [--tag TAG] utils ' + '--help"\nfor more information.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + version_parser = subparsers.add_parser( + 'version', add_help=True, + help='Print the version of C-PAC that cpac is using.') + help_call = '--help' in sys.argv or '-h' in sys.argv - run_parser.register('action', 'extend', ExtendAction) # run_parser.add_argument('--address', action='store', type=address) if not help_call: @@ -177,32 +197,40 @@ def _parser(): ) group_parser = subparsers.add_parser( - 'group', - add_help=False, + 'group', add_help=False, + help='Run a group level analysis in C-PAC. See\n"cpac [--platform ' + '{docker,singularity}] [--image IMAGE] [--tag TAG] group ' + '--help"\nfor more information.', formatter_class=argparse.ArgumentDefaultsHelpFormatter ) - group_parser.register('action', 'extend', ExtendAction) - - utils_parser = subparsers.add_parser( - 'utils', - add_help=False, - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - utils_parser.register('action', 'extend', ExtendAction) subparsers.add_parser( - 'pull', - add_help=True, + 'pull', add_help=True, + help='Upgrade your local C-PAC version to the latest version\n' + 'by pulling from Docker Hub or other repository.\nUse with ' + '"--image" and/or "--tag" to specify an image\nother than ' + 'the default "fcpindi/c-pac:latest" to pull.', aliases=['upgrade'], formatter_class=argparse.ArgumentDefaultsHelpFormatter ) + enter_parser = subparsers.add_parser( + 'enter', add_help=True, help='Enter a new C-PAC container via BASH.', + aliases=['bash', 'shell']) + + parse_resources.set_args(subparsers.add_parser( + 'parse-resources', add_help=True, aliases=['parse_resources'], + help='\n'.join([parse_resources.__doc__.split( + parse_resources.__file__.split('/', maxsplit=-1)[-1], + maxsplit=1)[-1].strip().replace( + r'`cpac_parse_resources`', '"parse-resources"'), + 'See "cpac parse-resources --help" for more information.']))) + crash_parser = subparsers.add_parser( - 'crash', - add_help=True, + 'crash', add_help=True, + help='Convert a crash pickle to plain text (C-PAC < 1.8.0).', formatter_class=argparse.ArgumentDefaultsHelpFormatter ) - crash_parser.register('action', 'extend', ExtendAction) crash_parser.add_argument( 'crashfile', @@ -210,6 +238,10 @@ def _parser(): 'Crashfiles in C-PAC >= 1.8.0 are plain text files.' ) + for subparser in [crash_parser, enter_parser, group_parser, run_parser, + utils_parser, version_parser]: + subparser.register('action', 'extend', ExtendAction) + return parser @@ -309,13 +341,20 @@ def main(args): **arg_vars ) - if args.command in ['pull', 'upgrade']: + elif args.command in ['enter', 'version']: + Backends(**arg_vars).run( + run_type=args.command, + flags=args.extra_args, + **arg_vars + ) + + elif args.command in ['pull', 'upgrade']: Backends(**arg_vars).pull( force=True, **arg_vars ) - if args.command in clargs: + elif args.command in clargs: # utils doesn't have '-h' flag for help if args.command == 'utils' and '-h' in arg_vars.get('extra_args', []): arg_vars['extra_args'] = [ @@ -329,12 +368,15 @@ def main(args): **arg_vars ) - if args.command == 'crash': + elif args.command == 'crash': Backends(**arg_vars).read_crash( flags=args.extra_args, **arg_vars ) + elif args.command == 'parse-resources': + parse_resources.main(args) + def run(): '''Function to try Docker first and fall back on Singularity if @@ -353,11 +395,15 @@ def run(): Consumes commandline arguments. Run `cpac --help` for usage string. ''' args = sys.argv[1:] - # reorder args command = None command_index = 0 parser = _parser() + if args and ( + args[0] == '--version' or args[0] == '--help' or args[0 == '-h'] + ): + parse_args(args) + # pylint: disable=protected-access commands = list([cmd for cmd in parser._get_positional_actions( ) if cmd.dest == 'command'][0].choices) options = set(chain.from_iterable([ @@ -380,6 +426,9 @@ def run(): if arg in options: reordered_args.append(args.pop(args.index(arg))) option_value_setting = True + elif any(arg.startswith(f'{option}=') for option in options): + reordered_args.append(args.pop(args.index(arg))) + option_value_setting = True elif option_value_setting: if arg.startswith('-'): option_value_setting = False diff --git a/src/cpac/backends/docker.py b/src/cpac/backends/docker.py index 21c6eddd..98d2f247 100644 --- a/src/cpac/backends/docker.py +++ b/src/cpac/backends/docker.py @@ -1,17 +1,20 @@ -import docker +import os +import docker +import dockerpty from docker.errors import ImageNotFound -from requests.exceptions import ConnectionError -from cpac.backends.platform import Backend, Platform_Meta +from cpac.backends.platform import Backend, PlatformMeta class Docker(Backend): def __init__(self, **kwargs): - super(Docker, self).__init__(**kwargs) - self.platform = Platform_Meta('Docker', '🐳') + super().__init__(**kwargs) + self.container = None + self.platform = PlatformMeta('Docker', '🐳') self._print_loading_with_symbol(self.platform.name) self.client = docker.from_env() + self.platform.version = self.client.version().get('Version', 'unknown') try: self.client.ping() except (docker.errors.APIError, ConnectionError): # pragma: no cover @@ -19,7 +22,6 @@ def __init__(self, **kwargs): f"Could not connect to {self.platform.name}. " "Is Docker running?" ) - self.volumes = {} image = kwargs['image'] if kwargs.get( 'image' @@ -31,7 +33,7 @@ def __init__(self, **kwargs): self.image = ':'.join([image, tag]) self._collect_config(**kwargs) - self.docker_kwargs = {} + self.docker_kwargs = {'init': True} if isinstance(kwargs.get('container_options'), list): for opt in kwargs['container_options']: if '=' in opt or ' ' in opt: @@ -53,11 +55,19 @@ def __init__(self, **kwargs): def _collect_config(self, **kwargs): if kwargs.get('command') not in {'pull', 'upgrade', None}: if isinstance(self.pipeline_config, str): + container_kwargs = {'image': self.image} + if os.path.exists(self.pipeline_config): + container_kwargs['volumes'] = {self.pipeline_config: { + 'bind': self.pipeline_config, + 'mode': 'ro', + }} try: - container = self.client.containers.create(image=self.image) + container = self.client.containers.create( + **container_kwargs) except ImageNotFound: # pragma: no cover self.pull(**kwargs) - container = self.client.containers.create(image=self.image) + container = self.client.containers.create( + **container_kwargs) stream = container.get_archive(path=self.pipeline_config)[0] self.config = b''.join([ l for l in stream # noqa E741 @@ -84,14 +94,14 @@ def _read_crash(self, read_crash_command, **kwargs): def run(self, flags=[], **kwargs): kwargs['command'] = [i for i in [ - kwargs['bids_dir'], - kwargs['output_dir'], - kwargs['level_of_analysis'], + kwargs.get('bids_dir'), + kwargs.get('output_dir'), + kwargs.get('level_of_analysis'), *flags ] if (i is not None and len(i))] - self._execute(**kwargs) + return self._execute(**kwargs) - def clarg(self, clcommand, flags=[], **kwargs): + def clarg(self, clcommand, flags=None, **kwargs): """ Runs a commandline command @@ -99,10 +109,12 @@ def clarg(self, clcommand, flags=[], **kwargs): ---------- clcommand: str - flags: list + flags: list or None kwargs: dict """ + if flags is None: + flags = [] kwargs['command'] = [i for i in [ kwargs.get('bids_dir', kwargs.get('working_dir', '/tmp')), kwargs.get('output_dir', '/outputs'), @@ -114,56 +126,82 @@ def clarg(self, clcommand, flags=[], **kwargs): self._execute(**kwargs) def _execute(self, command, run_type='run', **kwargs): + container_return = None try: self.client.images.get(self.image) except docker.errors.ImageNotFound: # pragma: no cover self.pull(**kwargs) - self._load_logging() + if run_type != 'version': + self._load_logging() + + shared_kwargs = { + 'image': self.image, + 'user': str(self.bindings['uid']), + **self._volumes_to_docker_mounts(), + 'working_dir': kwargs.get('working_dir', os.getcwd()), + **self.docker_kwargs + } if run_type == 'run': self.container = self.client.containers.run( - self.image, + **shared_kwargs, command=command, detach=True, stderr=True, stdout=True, - remove=True, - user=':'.join([ - str(self.bindings['uid']), - str(self.bindings['gid']) - ]), - volumes=self._volumes_to_docker_mounts(), - working_dir=kwargs.get('working_dir', '/tmp'), - **self.docker_kwargs + remove=True ) self._run = DockerRun(self.container) - self.container.stop() + elif run_type == 'version': + return self.get_version() elif run_type == 'exec': self.container = self.client.containers.create( - self.image, + **shared_kwargs, auto_remove=True, entrypoint='/bin/bash', - stdin_open=True, - user=':'.join([ - str(self.bindings['uid']), - str(self.bindings['gid']) - ]), - volumes=self._volumes_to_docker_mounts(), - working_dir=kwargs.get('working_dir', '/tmp'), - **self.docker_kwargs + stdin_open=True ) self.container.start() - return(self.container.exec_run( + container_return = self.container.exec_run( cmd=command, stdout=True, stderr=True, stream=True - )[1]) + )[1] + elif run_type == 'enter': + self.container = self.client.containers.create( + **shared_kwargs, + auto_remove=True, + entrypoint='/bin/bash', + stdin_open=True, + tty=True, + detach=False + ) + dockerpty.start(self.client.api, self.container.id) + return container_return + + def get_response(self, command, **kwargs): + """Method to return the response of running a command in the + Docker container. + + Parameters + ---------- + command : str + + Returns + ------- + str + """ + full_response = [] + for response in self._execute(command, run_type='exec', **kwargs): + full_response.append(response.decode()) + return ''.join(full_response) -class DockerRun(object): +class DockerRun: def __init__(self, container): + # pylint: disable=expression-not-assigned self.container = container [print( l.decode('utf-8'), end='' diff --git a/src/cpac/backends/platform.py b/src/cpac/backends/platform.py index 74932292..a612effc 100644 --- a/src/cpac/backends/platform.py +++ b/src/cpac/backends/platform.py @@ -1,23 +1,56 @@ +"""Base classes for platform-specific implementations""" +import atexit import os -import pandas as pd import pwd import tempfile import textwrap -import yaml from collections import namedtuple from contextlib import redirect_stderr from io import StringIO -from tabulate import tabulate +from typing import overload from warnings import warn +import pandas as pd +import yaml + +from docker import errors as docker_errors +from tabulate import tabulate + from cpac.helpers import cpac_read_crash, get_extra_arg_value -from cpac.utils import Locals_to_bind, PermissionMode +from cpac.helpers.cpac_parse_resources import get_or_create_config +from cpac.utils import LocalsToBind, Volume, Volumes +from cpac import __version__ as cpac_version + + +class CpacVersion: + """Class to hold the version of C-PAC running in the container""" + # pylint: disable=too-few-public-methods + def __init__(self, backend): + self.versions = namedtuple('versions', 'cpac CPAC') + self.versions.cpac = cpac_version + self.versions.CPAC = backend.get_response('cat /code/version').rstrip() + self.platform = backend.platform + + def __str__(self): + return (f'cpac (convenience wrapper) version {self.versions.cpac}\n' + f'C-PAC version {self.versions.CPAC} running on ' + f'{self.platform.name} version {self.platform.version}') + + +class PlatformMeta: + """Class to hold platform metadata""" + # pylint: disable=too-few-public-methods + def __init__(self, name, symbol): + self.name = name + self.symbol = symbol + self.version = 'unknown' -Platform_Meta = namedtuple('Platform_Meta', 'name symbol') + def __str__(self): + return f'{self.symbol} {self.name}' -class Backend(object): +class Backend: def __init__(self, **kwargs): # start with default pipline, but prefer pipeline config over preconfig # over default @@ -35,11 +68,51 @@ def __init__(self, **kwargs): '/code/CPAC/resources/configs', f'pipeline_config_{pipeline_config}.yml' ]) + self.volumes = Volume('/etc/passwd', mode='ro') + tracking_opt_out = '--tracking_opt-out' + if not(tracking_opt_out in kwargs or + tracking_opt_out in kwargs.get('extra_args', [])): + udir = os.path.expanduser('~') + if udir != '/': + tracking_path = get_or_create_config(udir) + self.volumes += Volume(tracking_path) + else: + raise EnvironmentError('Unable to create tracking ' + 'configuration. Please run with ' + '--tracking_opt-out and C-PAC >= ' + '1.8.4') + # initilizing these for overriding on load + self.bindings = {} + self.container = None + self.image = None + self.platform = None + self._run = None + self.uid = 0 + self.username = 'root' + self.working_dir = kwargs.get('working_dir', os.getcwd()) + atexit.register(self._cleanup) - def start(self, pipeline_config, subject_config): - raise NotImplementedError() + def __del__(self): + self._cleanup() + + def read_crash(self, crashfile, flags=None, **kwargs): + """For C-PAC < 1.8.0, this method is used to decode a + crashfile into plain text. Since C-PAC 1.8.0, + crashfiles are stored as plain text. + + Parameters + ---------- + crashfile : str + Path to the crashfile to decode. - def read_crash(self, crashfile, flags=[], **kwargs): + flags : list + + Returns + ------- + None + """ + if flags is None: + flags = [] os.chmod(cpac_read_crash.__file__, 0o775) self._set_crashfile_binding(crashfile) if self.platform.name == 'Singularity': @@ -63,32 +136,22 @@ def read_crash(self, crashfile, flags=[], **kwargs): crash_message += stderr.getvalue() stderr.read() # clear stderr print(crash_message.strip()) - if hasattr(self, 'container'): - self.container.stop() - def _bind_volume(self, local, remote, mode): - local, remote = self._prep_binding(local, remote) - b = {'bind': remote, 'mode': PermissionMode(mode)} - if local in self.volumes: - if remote in [binding['bind'] for binding in self.volumes[local]]: - for i, binding in enumerate(self.volumes[local]): - self.volumes[local][i] = { - 'bind': remote, - 'mode': max([ - self.volumes[local][i]['mode'], b['mode'] - ]) - } - else: - self.volumes[local].append(b) - else: - self.volumes[local] = [b] + def _bind_volume(self, volume: Volume) -> None: + """Binds a volume to the container. + + Parameters + ---------- + volume : Volume + Volume to bind. + """ + self.volumes += self._prep_binding(volume) def _collect_config_binding(self, config, config_key): config_binding = None if isinstance(config, str): if os.path.exists(config): - path = os.path.dirname(config) - self._set_bindings({'custom_binding': [':'.join([path]*2)]}) + self._set_bindings(custom_binding=Volume(config, mode='r')) config = self.clarg( clcommand='python -c "from CPAC.utils.configuration; ' 'import Configuration; ' @@ -98,7 +161,8 @@ def _collect_config_binding(self, config, config_key): pipeline_setup = config.get('pipeline_setup', {}) minimal = pipeline_setup.get('FROM', False) if isinstance(pipeline_setup, dict): - config_binding = pipeline_setup.get(config_key, {}).get('path') + config_binding = Volume(pipeline_setup.get(config_key, {}).get( + 'path')) else: minimal = True if minimal: @@ -111,17 +175,54 @@ def _collect_config_binding(self, config, config_key): ) return config_binding + def clarg(self, clcommand, flags=[], **kwargs): + """ + Runs a commandline command + + Parameters + ---------- + clcommand: str + + flags: list + + kwargs: dict + """ + raise NotImplementedError() + + def _cleanup(self): + if hasattr(self, 'container') and hasattr(self.container, 'stop'): + try: + self.container.stop() + except (docker_errors.APIError, docker_errors.NotFound): + pass + def collect_config_bindings(self, config, **kwargs): + """Function to collect bindings for a given configuration. + + Parameters + ---------- + config : str or dict + Configuration to collect bindings for. + + kwargs : dict + Extra arguments from the commandline. + + kwargs['output_dir'] : str + Output directory for the run. + + kwargs['working_dir'] : str + Working directory for the run. + + Returns + ------- + """ kwargs['output_dir'] = kwargs.get( 'output_dir', os.getcwd() ) - kwargs['working_dir'] = kwargs.get( - 'working_dir', - os.getcwd() - ) + kwargs['working_dir'] = self.working_dir - config_bindings = {} + config_bindings = Volumes() cwd = os.getcwd() for c_b in { ('log_directory', 'log'), @@ -129,7 +230,7 @@ def collect_config_bindings(self, config, **kwargs): ('crash_log_directory', 'log'), ('output_directory', 'outputs', 'output_dir') }: - inner_binding = self._collect_config_binding(config, c_b[0]) + inner_binding = self._collect_config_binding(config, c_b[0]).bind outer_binding = None if inner_binding is not None: if len(c_b) == 3: @@ -145,30 +246,58 @@ def collect_config_bindings(self, config, **kwargs): os.path.join(cwd, 'outputs') ), c_b[1]) if outer_binding is not None and inner_binding is not None: - config_bindings[outer_binding] = inner_binding + config_bindings += Volume(inner_binding) elif outer_binding is not None: - config_bindings[outer_binding] = outer_binding + config_bindings += Volume(outer_binding) else: path = os.path.join(cwd, c_b[1]) - config_bindings[path] = path + config_bindings += Volume(path) kwargs['config_bindings'] = config_bindings return kwargs + def get_response(self, command, **kwargs): + """Method to return the response of running a command in the + container. Implemented in the subclasses. + + Parameters + ---------- + command : str + + Returns + ------- + str + """ + raise NotImplementedError() + + def get_version(self): + """Method to get the version of C-PAC running in container. + + Parameters + ---------- + None + + Returns + ------- + CpacVersion + """ + version = CpacVersion(self) + print(version) + return version + def _load_logging(self): - t = pd.DataFrame([ - (i, j['bind'], j['mode']) for i in self.bindings['volumes'].keys( - ) for j in self.bindings['volumes'][i] - ]) - if not t.empty: - t.columns = ['local', self.platform.name, 'mode'] + table = pd.DataFrame([(volume.local, volume.bind, volume.mode) for + volume in self.bindings['volumes']]) + if not table.empty: + table.columns = ['local', self.platform.name, 'mode'] self._print_loading_with_symbol( - " ".join([ + ' '.join([ self.image, - "with these directory bindings:" + f'as "{self.username} ({self.uid})"', + 'with these directory bindings:' ]) ) print(textwrap.indent( - tabulate(t.applymap( + tabulate(table.applymap( lambda x: ( '\n'.join(textwrap.wrap(x, 42)) ) if isinstance(x, str) else x @@ -180,15 +309,39 @@ def _load_logging(self): "paths.\n" ) - def _prep_binding(self, binding_path_local, binding_path_remote): - binding_path_local = os.path.abspath( - os.path.expanduser(binding_path_local) - ) - os.makedirs(binding_path_local, exist_ok=True) - return( - os.path.realpath(binding_path_local), - os.path.abspath(binding_path_remote) + def _prep_binding(self, volume: Volume, + second_try: bool = False) -> Volume: + """ + Prepares a volume binding for the container. + + Parameters + ---------- + volume : Volume + Volume to bind. + + second_try : bool + Whether this is a second try to bind the volume. + + Returns + ------- + Volume + """ + volume.local = os.path.abspath( + os.path.expanduser(volume.local) ) + if not os.path.exists(volume.local): + try: + os.makedirs(volume.local, exist_ok=True) + except PermissionError as perm: + if second_try: + raise perm + new_local = os.path.join(self.working_dir, + volume.local.lstrip('/')) + print(f'Could not create {volume.local}. Binding ' + f'{volume.bind} to {new_local} instead.') + volume.local = new_local + return self._prep_binding(volume, second_try=True) + return volume def _print_loading_with_symbol(self, message, prefix='Loading'): if prefix is not None: @@ -198,6 +351,24 @@ def _print_loading_with_symbol(self, message, prefix='Loading'): except UnicodeEncodeError: print(message) + @overload + def __setattr__(self, name: str, value: Volume) -> None: + ... + @overload # noqa: E301 + def __setattr__(self, name: str, value: list) -> None: + ... + @overload # noqa: E301 + def __setattr__(self, name: str, value: Volumes) -> None: + ... + def __setattr__(self, name, value): # noqa: E301 + if name == 'volumes': + if isinstance(value, Volumes): + self.__dict__[name] = value + else: + self.__dict__[name] = Volumes(value) + else: + self.__dict__[name] = value + def _set_bindings(self, **kwargs): tag = kwargs.get('tag', None) tag = tag if isinstance(tag, str) else None @@ -206,67 +377,65 @@ def _set_bindings(self, **kwargs): *kwargs.get('extra_args', []), kwargs.get('crashfile', '') ]: if os.path.exists(kwarg): - d = kwarg if os.path.isdir(kwarg) else os.path.dirname(kwarg) - self._bind_volume(d, d, 'r') + self._bind_volume(Volume(kwarg, mode='r')) if 'data_config_file' in kwargs and isinstance( kwargs['data_config_file'], str ) and os.path.exists(kwargs['data_config_file']): - dc_dir = os.path.dirname(kwargs['data_config_file']) - self._bind_volume(dc_dir, dc_dir, 'r') - locals_from_data_config = Locals_to_bind() + self._bind_volume(Volume(kwargs['data_config_file'], mode='r')) + locals_from_data_config = LocalsToBind() locals_from_data_config.from_config_file( kwargs['data_config_file'] ) for local in locals_from_data_config.locals: - self._bind_volume(local, local, 'r') - self._bind_volume(kwargs['output_dir'], kwargs['output_dir'], 'rw') - self._bind_volume(kwargs['working_dir'], kwargs['working_dir'], 'rw') + self._bind_volume(Volume(local, mode='r')) + for dir_type in ['working', 'output']: + self._bind_volume(Volume(kwargs[f'{dir_type}_dir'])) if kwargs.get('custom_binding'): - for d in kwargs['custom_binding']: - self._bind_volume(*d.split(':'), 'rw') - for d in ['bids_dir', 'output_dir']: + for d in kwargs['custom_binding']: # pylint: disable=invalid-name + bind_parts = d.split(':') + if len(bind_parts) == 3: + self._bind_volume(Volume(*bind_parts)) + elif len(bind_parts) == 2: + self._bind_volume(Volume(*bind_parts, mode='rw')) + elif len(bind_parts) == 1: + self._bind_volume(Volume(bind_parts[0])) + else: + raise SyntaxError("I don't know what to do with custom " + "binding {}".format(d)) + for d in ['bids_dir', 'output_dir']: # pylint: disable=invalid-name if d in kwargs and isinstance(kwargs[d], str) and os.path.exists( kwargs[d] ): - self._bind_volume( - kwargs[d], - kwargs[d], - 'rw' if d == 'output_dir' else 'r' - ) + self._bind_volume(Volume( + kwargs[d], mode='rw' if d == 'output_dir' else 'r')) if kwargs.get('config_bindings'): for binding in kwargs['config_bindings']: - self._bind_volume( - binding, - kwargs['config_bindings'][binding], - 'rw' - ) - uid = os.getuid() - self.bindings = { - 'gid': pwd.getpwuid(uid).pw_gid, + self._bind_volume(binding) + self.uid = os.getuid() + pwuid = pwd.getpwuid(self.uid) + self.username = getattr(pwuid, 'pw_name', + getattr(pwuid, 'pw_gecos', str(self.uid))) + self.bindings.update({ 'tag': tag, - 'uid': uid, + 'uid': self.uid, 'volumes': self.volumes - } + }) def _volumes_to_docker_mounts(self): - return([ - '{}:{}:{}'.format( - i, - j['bind'], - j['mode'] - ) for i in self.volumes.keys() for j in self.volumes[i] - ]) + return {'volumes': [str(volume) for volume in self.volumes]} def _set_crashfile_binding(self, crashfile): for ckey in ["/wd/", "/crash/", "/log"]: if ckey in crashfile: - self._bind_volume(crashfile.split(ckey)[0], '/outputs', 'rw') - self._bind_volume(tempfile.TemporaryDirectory().name, '/out', 'rw') - helper_dir = os.path.dirname(cpac_read_crash.__file__) - self._bind_volume(helper_dir, helper_dir, 'ro') + self._bind_volume(Volume( + crashfile.split(ckey)[0], '/outputs', 'rw')) + with tempfile.TemporaryDirectory() as temp_dir: + self._bind_volume(Volume(temp_dir, '/out', 'rw')) + helper = cpac_read_crash.__file__ + self._bind_volume(Volume(helper, mode='ro')) -class Result(object): +class Result: mime = None def __init__(self, name, value): diff --git a/src/cpac/backends/singularity.py b/src/cpac/backends/singularity.py index 71da6e29..ac766ded 100644 --- a/src/cpac/backends/singularity.py +++ b/src/cpac/backends/singularity.py @@ -1,21 +1,22 @@ import os from itertools import chain +from spython.image import Image from spython.main import Client -from subprocess import CalledProcessError -from cpac.backends.platform import Backend, Platform_Meta +from cpac.backends.platform import Backend, PlatformMeta BINDING_MODES = {'ro': 'ro', 'w': 'rw', 'rw': 'rw'} class Singularity(Backend): def __init__(self, **kwargs): - super(Singularity, self).__init__(**kwargs) - self.platform = Platform_Meta('Singularity', 'Ⓢ') + super().__init__(**kwargs) + self.container = None + self.platform = PlatformMeta('Singularity', 'Ⓢ') + self.platform.version = Client.version().split(' ')[-1] self._print_loading_with_symbol(self.platform.name) self.pull(**kwargs, force=False) - self.volumes = {} self.options = list(chain.from_iterable(kwargs[ "container_options" ])) if bool(kwargs.get("container_options")) else [] @@ -33,17 +34,28 @@ def __init__(self, **kwargs): def _bindings_as_option(self): self.options += ( - ['-B', ','.join((chain.from_iterable([[ - ':'.join([b for b in [ - local, - binding['bind'] if - local != binding['bind'] or - BINDING_MODES[str(binding['mode'])] != 'rw' else None, - BINDING_MODES[str(binding['mode'])] if - BINDING_MODES[str(binding['mode'])] != 'rw' else None - ] if b is not None]) for binding in self.volumes[local] - ] for local in self.volumes])))] - ) + ['-B', ','.join([':'.join([ + binding.local, binding.bind, str(binding.mode) + ]) for binding in self.volumes])]) + + def _bindings_from_option(self): + enter_options = {} + if '-B' in self.options: + enter_options['bind'] = self.options[ + self.options.index('-B') + 1] + self.options.remove(enter_options['bind']) + self.options.remove('-B') + enter_options['singularity_options'] = self.options + return enter_options + + def _pull(self, img, force, pull_folder): + '''Tries to pull image gracefully''' + try: + self.image = Client.pull(img, force=force, pull_folder=pull_folder) + except ValueError as value_error: + if 'closed file' in str(value_error): + # pylint: disable=protected-access + self.image = Image(Client._get_filename(img)) def pull(self, force=False, **kwargs): image = kwargs['image'] if kwargs.get( @@ -54,50 +66,88 @@ def pull(self, force=False, **kwargs): if kwargs.get("working_dir") is not None: pwd = kwargs["working_dir"] os.chdir(pwd) + image_path = Client._get_filename( # pylint: disable=protected-access + image if tag is None else ':'.join([image, tag])) if ( not force and - image and isinstance(image, str) and os.path.exists(image) + image and isinstance(image, str) and os.path.exists(image_path) ): - self.image = image + self.image = image_path elif tag and isinstance(tag, str): # pragma: no cover - self.image = Client.pull( + self._pull( f"docker://{image}:{tag}", force=force, pull_folder=pwd ) else: # pragma: no cover try: - self.image = Client.pull( + self._pull( "docker://fcpindi/c-pac:latest", force=force, pull_folder=pwd ) - except Exception: - raise OSError("Could not connect to Singularity") + except Exception as exception: + raise OSError( + "Could not connect to Singularity" + ) from exception + + def get_response(self, command, **kwargs): + """Method to return the response of running a command in the + Singularity container. + + Parameters + ---------- + command : str - def _try_to_stream(self, args, stream_command='run', **kwargs): + Returns + ------- + str + """ + full_response = [] + for response in self._try_to_stream( + args={'command': command}, + stream_command='execute', + silent=True, + **kwargs + ): + full_response.append(response) + return ''.join(full_response) + + def _try_to_stream(self, args, stream_command='run', silent=False, + **kwargs): self._bindings_as_option() - try: - if stream_command == 'run': - for line in Client.run( - Client.instance(self.image), - args=args, - options=self.options, - stream=True, - return_result=True - ): - yield line - elif stream_command == 'execute': - for line in Client.execute( + if stream_command == 'run': + self.container = Client.run( + Client.instance(self.image), + args=args, + options=self.options, + stream=not silent, + return_result=True, + **kwargs) + else: + enter_options = self._bindings_from_option() + if stream_command == 'execute': + self.container = Client.execute( self.image, command=args['command'].split(' '), options=self.options, - stream=True, - quiet=False - ): - yield line - except CalledProcessError: # pragma: no cover - return + stream=not silent, + quiet=silent, + **{kwarg: value for kwarg, value in kwargs.items() if + kwarg in ['contain', 'environ', 'nv', 'sudo', + 'return_result', 'writable']}) + elif stream_command == 'enter': + Client.shell( + self.image, + **enter_options, + **kwargs) + if self.container is not None: + for line in self.container: + yield line + if hasattr(self.container, 'close') and callable( + self.container.close + ): + self.container.close() def _read_crash(self, read_crash_command, **kwargs): return self._try_to_stream( @@ -106,18 +156,28 @@ def _read_crash(self, read_crash_command, **kwargs): **kwargs ) - def run(self, flags=[], **kwargs): + def run(self, flags=None, run_type='run', **kwargs): + # pylint: disable=expression-not-assigned + if flags is None: + flags = [] self._load_logging() - [print(o, end='') for o in self._try_to_stream( - args=' '.join([ - kwargs['bids_dir'], - kwargs['output_dir'], - kwargs['level_of_analysis'], - *flags - ]).strip(' ') - )] + if run_type == 'run': + [print(o, end='') for o in self._try_to_stream( + args=' '.join([ + kwargs['bids_dir'], + kwargs['output_dir'], + kwargs['level_of_analysis'], + *flags + ]).strip(' ') + )] + elif run_type == 'version': + return self.get_version() + else: + [print(o, end='') for o in self._try_to_stream( + args=' '.join(flags).strip(' '), + stream_command=run_type)] - def clarg(self, clcommand, flags=[], **kwargs): + def clarg(self, clcommand, flags=None, **kwargs): """ Runs a commandline command @@ -129,6 +189,9 @@ def clarg(self, clcommand, flags=[], **kwargs): kwargs: dict """ + # pylint: disable=expression-not-assigned + if flags is None: + flags = [] self._load_logging() [print(o, end='') for o in self._try_to_stream( args=' '.join([ diff --git a/src/cpac/helpers/__init__.py b/src/cpac/helpers/__init__.py index 872fdf27..dce15153 100644 --- a/src/cpac/helpers/__init__.py +++ b/src/cpac/helpers/__init__.py @@ -1,5 +1,16 @@ '''Hepler functions for cpac Python package.''' import re +from itertools import chain + +TODOs = {'persisting_containers': 'Some Docker containers unexpectedly ' + 'persist after cpac finishes. To clear ' + 'them, run\n ' + r'1. `docker ps` to list the containers' + '\n For each C-PAC conatainer that ' + 'persists, run\n ' + r'2. `docker attach `' + '\n ' + r'3. `exit`'} def get_extra_arg_value(extra_args, argument): @@ -28,15 +39,13 @@ def get_extra_arg_value(extra_args, argument): ... '--participant_ndx 3'], 'participant_ndx') '3' ''' - pattern = r'^\-*' + argument + r'([=\s]{1}.*)$' + extra_args = list(chain.from_iterable([ + re.split(r'[=\s]', arg) for arg in extra_args])) for index, item in enumerate(extra_args): - if re.match(pattern, item) is not None: - for sep in {'=', ' '}: - if sep in item: - return item.split(sep, 1)[1] - if len(extra_args) > index: - return extra_args[index + 1] + if item.startswith('-') and item.lstrip('-') == argument: + return extra_args[index + 1] + return None -__all__ = ['get_extra_arg_value'] +__all__ = ['get_extra_arg_value', 'TODOs'] diff --git a/src/cpac/helpers/cpac_parse_resources.py b/src/cpac/helpers/cpac_parse_resources.py new file mode 100755 index 00000000..4d5bf377 --- /dev/null +++ b/src/cpac/helpers/cpac_parse_resources.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +r'''cpac_parse_resources.py + +When provided with a `callback.log` file, this utility can sort through +the memory `runtime` usage, `estimate`, and associated `efficiency`, to +identify the `n` tasks with the `highest` or `lowest` of each of these +categories. +`cpac_parse_resources` is intended to be run outside a C-PAC container. +''' +import configparser +import os +import uuid + +from rich.console import Console +from rich.table import Table + +from argparse import ArgumentParser +import pandas as pd +import numpy as np +import json + + +runti = 'runtime_memory_gb' +estim = 'estimated_memory_gb' + +field = {'runtime': runti, + 'estimate': estim, + 'efficiency': 'efficiency'} + + +def display(df): + console = Console() + + table = Table(show_header=True, header_style="bold magenta") + table.add_column("Task ID", style="dim", width=40) + table.add_column("Memory Used") + table.add_column("Memory Estimated") + table.add_column("Memory Efficiency") + + for _, d in df.iterrows(): + tmp = list() + tmp += [d['id']] + tmp += [d[runti]] + tmp += [d[estim]] + tmp += ["{0:.2f} %".format(100*d[runti] * 1.0 / d[estim])] + + tmp = ["{0:.4f}".format(t) if isinstance(t, float) else str(t) + for t in tmp] + table.add_row(*tmp) + del tmp + + console.print(table) + + +def get_or_create_config(udir): + """Function to create a Google Analytics tracking config file. + + Sourced from https://github.com/FCP-INDI/C-PAC/blob/80424468c7f4e59c102f446b05d4154ec1cd4b2d/CPAC/utils/ga.py#L19-L30 + """ # noqa: E501 # pylint: disable=line-too-long + tracking_path = os.path.join(udir, '.cpac') + cparser = configparser.ConfigParser() + if os.path.exists(tracking_path): + cparser.read(tracking_path) + if not cparser.has_section('user'): + cparser.read_dict(dict(user=dict(uid=uuid.uuid1().hex, + track=True))) + with open(tracking_path, 'w+') as fhandle: + cparser.write(fhandle) + return tracking_path + + +def load_runtime_stats(callback): + with open(callback) as fhandle: + logs = [json.loads(log) for log in fhandle.readlines()] + + pruned_logs = [] + for log in logs: + if runti not in log.keys(): + continue + + tmp = {} + for k in ['id', runti, estim]: + tmp[k] = log[k] + tmp['efficiency'] = tmp[runti] / tmp[estim] * 100 + + pruned_logs += [tmp] + del tmp + + return pd.DataFrame.from_dict(pruned_logs) + + +def main(args): + """Main function to parse and display resource usage + + Parameters + ---------- + args : argparse.Namespace + + Returns + ------- + None + """ + usage = load_runtime_stats(args.callback) + filtered_usage = query(usage, args.filter_field, args.filter_group, + args.filter_count) + display(filtered_usage) + + +def query(usage, f, g, c): + order = g == 'lowest' + usage.sort_values(by=field[f], ascending=order, inplace=True) + usage.reset_index(inplace=True, drop=True) + return usage[0:c] + + +def set_args(parser): + """Set up the command line arguments for the script + + Parameters + ---------- + parser : argparse.ArgumentParser + + Returns + ------- + parser : argparse.ArgumentParser + """ + parser.add_argument("callback", + help="callback.log file found in the 'log' " + "directory of the specified derivatives path") + parser.add_argument("--filter_field", "-f", action="store", + choices=['runtime', 'estimate', 'efficiency'], + default='efficiency') + parser.add_argument("--filter_group", "-g", action="store", + choices=['lowest', 'highest'], default='lowest') + parser.add_argument("--filter_count", "-n", action="store", type=int, + default=10) + return parser + + +if __name__ == '__main__': + main(set_args(ArgumentParser(__file__, add_help=True)).parse_args()) diff --git a/src/cpac/helpers/cpac_read_crash.py b/src/cpac/helpers/cpac_read_crash.py index 480f9090..000fb784 100755 --- a/src/cpac/helpers/cpac_read_crash.py +++ b/src/cpac/helpers/cpac_read_crash.py @@ -7,7 +7,6 @@ import os import re - from sys import argv path_regex = re.compile( diff --git a/src/cpac/utils.py b/src/cpac/utils.py deleted file mode 100644 index d02b1993..00000000 --- a/src/cpac/utils.py +++ /dev/null @@ -1,227 +0,0 @@ -import os -import yaml - -from cpac import dist_name -from itertools import permutations -from warnings import warn - - -class Locals_to_bind(): - """ - Class to collect local directories to bind to containers. - """ - def __init__(self): - self.locals = set() - - def __repr__(self): - return(str(self.locals)) - - def from_config_file(self, config_path): - """ - Paramter - -------- - config_path: str - path to data config file - """ - with open(config_path, 'r') as r: - config_dict = yaml.safe_load(r) - self._add_locals(config_dict) - - def _add_locals(self, d): - """ - Parameter - --------- - d: any - object to search for local paths - """ - if isinstance(d, dict): - [self._add_locals(d[k]) for k in d] - elif isinstance(d, list) or isinstance(d, tuple): - [self._add_locals(i) for i in d] - elif isinstance(d, str): - if os.path.exists(d): - if os.path.isdir(d): - self.locals.add(d) - else: - self.locals.add(os.path.dirname(d)) - self._local_common_paths() - - def _local_common_paths(self): - new_locals = set() - stragglers = set() - - def common_path(paths): - x = os.path.commonprefix(list(paths)) - while not x.endswith('/'): - x = x[:-2] - x - return(x) - for i in list(permutations(self.locals, 3)): - c = common_path(i) - if len(c) > 1: - new_locals.add(c) - else: - for f in i: - stragglers.add(f) - self.locals = new_locals | {s for s in stragglers if not any([ - s.startswith(n) for n in new_locals - ])} - - -class PermissionMode(): - """ - Class to overload comparison operators to compare file permissions levels. - - 'rw' > 'w' > 'r' - - Examples - -------- - >>> PermissionMode('ro') > PermissionMode('rw') - False - >>> PermissionMode('rw') > PermissionMode('r') - True - >>> PermissionMode('r') > PermissionMode('r') - False - >>> PermissionMode('ro') >= PermissionMode('rw') - False - >>> PermissionMode('rw') >= PermissionMode('r') - True - >>> PermissionMode('r') >= PermissionMode('r') - True - >>> PermissionMode('ro') < PermissionMode('rw') - True - >>> PermissionMode('rw') < PermissionMode('r') - False - >>> PermissionMode('r') < PermissionMode('r') - False - >>> PermissionMode('ro') <= PermissionMode('rw') - True - >>> PermissionMode('ro') <= PermissionMode('ro') - True - >>> PermissionMode('rw') <= PermissionMode('ro') - False - >>> PermissionMode('ro') == PermissionMode('rw') - False - >>> PermissionMode('ro') == PermissionMode('r') - True - """ - defined_modes = {'rw', 'w', 'r', 'ro'} - - def __init__(self, fs_str): - - self.mode = fs_str.mode if isinstance( - fs_str, - PermissionMode - ) else 'ro' if fs_str == 'r' else fs_str - self.defined = self.mode in PermissionMode.defined_modes - self._warn_if_undefined() - - def __repr__(self): - return(self.mode) - - def __eq__(self, other): - for permission in (self, other): - return self.mode == other.mode - - def __gt__(self, other): - for permission in (self, other): - if(permission._warn_if_undefined()): # pragma: no cover - return(NotImplemented) - - if self.mode == 'rw': - if other.mode in {'w', 'ro'}: - return(True) - elif self.mode == 'w' and other.mode == 'ro': - return(True) - - return(False) - - def __ge__(self, other): - for permission in (self, other): - if(permission._warn_if_undefined()): # pragma: no cover - return(NotImplemented) - - if self.mode == other.mode or self > other: - return(True) - - return(False) - - def __lt__(self, other): - for permission in (self, other): - if(permission._warn_if_undefined()): # pragma: no cover - return(NotImplemented) - - if self.mode == 'ro': - if other.mode in {'w', 'rw'}: - return(True) - elif self.mode == 'ro' and other.mode == 'w': - return(True) - - return(False) - - def __le__(self, other): - for permission in (self, other): - if(permission._warn_if_undefined()): # pragma: no cover - return(NotImplemented) - - if self.mode == other.mode or self < other: - return(True) - - return(False) - - def _warn_if_undefined(self): # pragma: no cover - if not self.defined: - warn( - f'\'{self.mode}\' is not a fully-configured permission ' - f'level in {dist_name}. Configured permission levels are ' - f'''{", ".join([ - f"'{mode}'" for mode in PermissionMode.defined_modes - ])}''', - UserWarning - ) - return(True) - return(False) - - -def ls_newest(directory, extensions): - """ - Function to return the most-recently-modified of a given extension in a - given directory - - Parameters - ---------- - directory: str - - extension: iterable - - Returns - ------- - full_path_to_file: str or None if none found - """ - ls = [ - os.path.join( - directory, - d - ) for d in os.listdir( - directory - ) if any([d.endswith( - extension.lstrip('.').lower() - ) for extension in extensions]) - ] - ls.sort(key=lambda fp: os.stat(fp).st_mtime) - try: - return(ls[-1]) - except IndexError: # pragma: no cover - return(None) - - -def render_crashfile(crash_path): - """ - Parameter - --------- - crash_path: str - - Returns - ------- - str, contents of pickle - """ diff --git a/src/cpac/utils/__init__.py b/src/cpac/utils/__init__.py new file mode 100644 index 00000000..d1f4c39d --- /dev/null +++ b/src/cpac/utils/__init__.py @@ -0,0 +1,5 @@ +from .checks import check_version_at_least +from .utils import LocalsToBind, PermissionMode, Volume, Volumes + +__all__ = ['check_version_at_least', 'LocalsToBind', 'PermissionMode', + 'Volume', 'Volumes'] diff --git a/src/cpac/utils/checks.py b/src/cpac/utils/checks.py new file mode 100644 index 00000000..2d85a018 --- /dev/null +++ b/src/cpac/utils/checks.py @@ -0,0 +1,32 @@ +"""Functions to check things like the in-container C-PAC version.""" +from semver import VersionInfo + +from cpac.backends import Backends + + +def check_version_at_least(min_version, platform, image=None, tag=None): + """Function to check the in-container C-PAC version + + Parameters + ---------- + min_version : str + Semantic version + + platform : str or None + + image : str or None + + tag : str or None + + Returns + ------- + bool + Is the version at least the minimum version? + """ + if platform is None: + platform = 'docker' + arg_vars = {'platform': platform, 'image': image, 'tag': tag, + 'command': 'version'} + return VersionInfo.parse(min_version) <= VersionInfo.parse( + Backends(**arg_vars).run( + run_type='version').versions.CPAC.lstrip('v')) diff --git a/src/cpac/utils/utils.py b/src/cpac/utils/utils.py new file mode 100644 index 00000000..46303398 --- /dev/null +++ b/src/cpac/utils/utils.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import os + +from itertools import permutations +from typing import Iterator, overload +from warnings import warn + +import yaml + +from cpac import dist_name + + +class LocalsToBind: + """Class to collect local directories to bind to containers.""" + def __init__(self): + self.locals = set() + + def __repr__(self): + return str(self) + + def __str__(self): + return str(self.locals) + + def from_config_file(self, config_path): + """ + Paramter + -------- + config_path : str + path to data config file + """ + with open(config_path, 'r') as config_yml: + config_dict = yaml.safe_load(config_yml) + self._add_locals(config_dict) + + def _add_locals(self, local: str) -> None: + """ + Parameter + --------- + local : any + object to search for local paths + """ + # pylint: disable=expression-not-assigned + if isinstance(local, dict): + [self._add_locals(local[k]) for k in local] + elif isinstance(local, (list, tuple)): + [self._add_locals(i) for i in local] + elif isinstance(local, str): + if os.path.exists(local): + self.locals.add(local) + self._local_common_paths() + + def _local_common_paths(self): + new_locals = set() + stragglers = set() + + def common_path(paths): + x = os.path.commonprefix(list(paths)) + while not x.endswith('/'): + x = x[:-2] + x + return(x) + for i in list(permutations(self.locals, 3)): + c = common_path(i) + if len(c) > 1: + new_locals.add(c) + else: + for f in i: + stragglers.add(f) + self.locals = new_locals | {s for s in stragglers if not any([ + s.startswith(n) for n in new_locals + ])} + + +class PermissionMode: + """ + Class to overload comparison operators to compare file permissions levels. + + 'rw' > 'w' > 'r' + + Examples + -------- + >>> PermissionMode('ro') > PermissionMode('rw') + False + >>> PermissionMode('rw') > PermissionMode('r') + True + >>> PermissionMode('r') > PermissionMode('r') + False + >>> PermissionMode('ro') >= PermissionMode('rw') + False + >>> PermissionMode('rw') >= PermissionMode('r') + True + >>> PermissionMode('r') >= PermissionMode('r') + True + >>> PermissionMode('ro') < PermissionMode('rw') + True + >>> PermissionMode('rw') < PermissionMode('r') + False + >>> PermissionMode('r') < PermissionMode('r') + False + >>> PermissionMode('ro') <= PermissionMode('rw') + True + >>> PermissionMode('ro') <= PermissionMode('ro') + True + >>> PermissionMode('rw') <= PermissionMode('ro') + False + >>> PermissionMode('ro') == PermissionMode('rw') + False + >>> PermissionMode('ro') == PermissionMode('r') + True + """ + defined_modes = {'rw', 'w', 'r', 'ro'} + + def __init__(self, fs_str): + self.mode = fs_str.mode if isinstance( + fs_str, + PermissionMode + ) else 'ro' if fs_str == 'r' else fs_str + self.defined = self.mode in PermissionMode.defined_modes + self._warn_if_undefined() + + def __repr__(self): + return self.mode + + def __eq__(self, other): + return self.mode == other.mode + + def __gt__(self, other): + for permission in (self, other): + if permission._warn_if_undefined(): + return NotImplemented + if self.mode == 'rw': + if other.mode in {'w', 'ro'}: + return True + elif self.mode == 'w' and other.mode == 'ro': + return True + return False + + def __ge__(self, other): + for permission in (self, other): + if permission._warn_if_undefined(): + return NotImplemented + if self.mode == other.mode or self > other: + return True + return False + + def __lt__(self, other): + for permission in (self, other): + if permission._warn_if_undefined(): + return NotImplemented + if self.mode == 'ro': + if other.mode in {'w', 'rw'}: + return True + elif self.mode == 'ro' and other.mode == 'w': + return True + return False + + def __le__(self, other): + for permission in (self, other): + if permission._warn_if_undefined(): + return NotImplemented + if self.mode == other.mode or self < other: + return True + return False + + def _warn_if_undefined(self): + if not self.defined: + warn(f'\'{self.mode}\' is not a fully-configured permission ' + f'level in {dist_name}. Configured permission levels are ' + f'''{", ".join([ + f"'{mode}'" for mode in PermissionMode.defined_modes + ])}''', + UserWarning) + return True + return False + + +class Volume: + '''Class to store bind volume information''' + @overload + def __init__(self, local: str, bind: str = None, mode: None = None + ) -> None: + ... + @overload # noqa: E301 + def __init__(self, local: str, bind: str = None, + mode: PermissionMode = None) -> None: + ... + def __init__(self, local, bind=None, mode=None): # noqa: E301 + self.local = local + self.bind = bind if bind is not None else local + if self.bind is not None and not self.bind.startswith('/'): + self.bind = os.path.abspath(self.bind) + if isinstance(mode, PermissionMode): + self.mode = mode + elif mode is not None: + self.mode = PermissionMode(mode) + else: + self.mode = PermissionMode('rw') + + def __repr__(self): + return str(self) + + def __str__(self): + return f'{self.local}:{self.bind}:{self.mode}' + + +class Volumes: + '''Class to store all bind volumes. Prevents duplicate mount points.''' + @overload + def __init__(self, volumes: list = None) -> None: + ... + @overload # noqa: E301 + def __init__(self, volumes: Volume = None) -> None: + ... + def __init__(self, volumes=None): # noqa: E301 + try: + if volumes is None: + self.volumes = {} + elif isinstance(volumes, list): + self.volumes = {volume.local: volume for volume in [ + Volume(volume) for volume in volumes]} + elif isinstance(volumes, Volume): + self.volumes = {volumes.local: volumes} + except AttributeError as attribute_error: + raise TypeError('Volumes must be initialized with a Volume ' + 'object, a list of Volume objects or None' + ) from attribute_error + + @overload + def __add__(self, other: list) -> Volumes: + ... + @overload # noqa: E301 + def __add__(self, other: Volume) -> Volumes: + ... + @overload # noqa: E301 + def __add__(self, other: Volumes) -> Volumes: + ... + def __add__(self, other): # noqa: E301 + '''Add volume + + Parameters + ---------- + other : Volume, list, or Volumes + Volume(s) to add + + Returns + ------- + Volumes + ''' + new_volumes = Volumes(self.volumes.copy()) + if isinstance(other, (list, Volumes)): + for volume in other: + new_volumes += volume + elif isinstance(other, Volume): + new_volumes.volumes.update({other.bind: other}) + return new_volumes + + @overload + def __iadd__(self, other: list) -> Volumes: + ... + @overload # noqa: E301 + def __iadd__(self, other: Volume) -> Volumes: + ... + @overload # noqa: E301 + def __iadd__(self, other: Volumes) -> Volumes: + ... + def __iadd__(self, other): # noqa: E301 + '''Add volume in place + + Parameters + ---------- + other : Volume, list, or Volumes + Volume(s) to add + + Returns + ------- + Volumes + ''' + if isinstance(other, (list, Volumes)): + for volume in other: + self += volume + elif isinstance(other, Volume): + self.volumes.update({other.bind: other}) + return self + + def __isub__(self, bind: str) -> Volumes: + '''Remove volume in place + + Parameters + ---------- + bind : str + key of Volume to remove + + Returns + ------- + Volumes + ''' + if bind in self.volumes: + del self.volumes[bind] + return self + + def __iter__(self) -> Iterator[Volume]: + '''Iterator over volumes''' + return iter(self.volumes.values()) + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return str(list(self.volumes.values())) + + def __sub__(self, bind: str) -> Volumes: + '''Remove volume + + Parameters + ---------- + bind : str + key of Volume to remove + + Returns + ------- + Volumes + ''' + new_volumes = Volumes(self.volumes.copy()) + if bind in new_volumes.volumes: + del new_volumes.volumes[bind] + return new_volumes diff --git a/tests/.pylintrc b/tests/.pylintrc new file mode 100644 index 00000000..1396bdcb --- /dev/null +++ b/tests/.pylintrc @@ -0,0 +1,2 @@ +[MAIN] +ignore-imports=y \ No newline at end of file diff --git a/tests/CONSTANTS.py b/tests/CONSTANTS.py index c7f83503..9018fa7d 100644 --- a/tests/CONSTANTS.py +++ b/tests/CONSTANTS.py @@ -1,6 +1,6 @@ -import os - -from cpac.utils import ls_newest +'''Constants for tests''' +# pylint: disable=invalid-name +TAGS = [None, 'latest', 'nightly'] def args_before_after(argv, args): @@ -22,6 +22,8 @@ def args_before_after(argv, args): after : list f'cpac {argv} {args}'.split(' ') ''' + argv = single_space(argv).strip() + args = single_space(args).strip() if argv.startswith('cpac'): argv = argv.lstrip('cpac').strip() if args is not None and len(args): @@ -49,23 +51,23 @@ def set_commandline_args(platform, tag, sep=' '): ''' args = '' if platform is not None: - if platform.lower() == 'docker': - args = args + PLATFORM_ARGS[0] - elif platform.lower() == 'singularity': - args = args + PLATFORM_ARGS[1] - if sep != ' ': - args = args.replace(' ', sep) + args += f' --platform{sep}{platform.lower()} ' if tag and tag is not None: - args = args + f' --tag{sep}{tag}' + args = args + f' --tag{sep}{tag} ' return args -def SINGULARITY_OPTION(): - singularity_option = ls_newest(os.getcwd(), ['sif', 'simg']) - return(f'--image {singularity_option}' if ( - singularity_option is not None - ) else '--platform singularity') +def single_space(string): + '''Function to remove spaces from a string + Parameters + ---------- + string : str -PLATFORM_ARGS = ['--platform docker', SINGULARITY_OPTION()] -TAGS = [None, 'latest', 'dev-v1.8'] + Returns + ------- + string : str + ''' + while ' ' in string: + string = string.replace(' ', ' ') + return string diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py index 8844b112..9a293853 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,12 +5,27 @@ Read more about conftest.py under: https://pytest.org/latest/plugins.html ''' +import logging +import pytest # pylint: disable=import-error + +LOGGER = logging.getLogger() + + +@pytest.fixture(autouse=True) +def ensure_logging_framework_not_altered(): + before_handlers = list(LOGGER.handlers) + yield + LOGGER.handlers = before_handlers -# import pytest def pytest_addoption(parser): - parser.addoption('--platform', action='store', nargs=1, default=[]) - parser.addoption('--tag', action='store', nargs=1, default=[]) + '''Add command line options for pytest.''' + def add_option(option): + '''Factory function to add option and fixture''' + parser.addoption(f'--{option}', action='store', nargs=1, + default=[None]) + for option in ['platform', 'tag']: + add_option(option) def pytest_generate_tests(metafunc): diff --git a/tests/test_cpac.py b/tests/test_cpac.py index 94af804c..76264732 100644 --- a/tests/test_cpac.py +++ b/tests/test_cpac.py @@ -1,47 +1,52 @@ -import pytest +"""Tests for top-level cpac cli.""" import sys - from contextlib import redirect_stdout from io import StringIO, TextIOWrapper, BytesIO from unittest import mock +import pytest + from cpac.backends import Backends from cpac.__main__ import run -from CONSTANTS import set_commandline_args +from .CONSTANTS import set_commandline_args def test_loading_message(platform, tag): - redirect_out = StringIO() - with redirect_stdout(redirect_out): - loaded = Backends(platform, tag=tag) - with_symbol = ' '.join([ - 'Loading', - loaded.platform.symbol, - loaded.platform.name - ]) - assert with_symbol in redirect_out.getvalue() - - redirect_out = TextIOWrapper( - BytesIO(), encoding='latin-1', errors='strict', write_through=True) - with redirect_stdout(redirect_out): - loaded = Backends(platform, tag=tag) - without_symbol = ' '.join([ - 'Loading', - loaded.platform.name - ]) - assert without_symbol in redirect_out.buffer.getvalue().decode() + """Test loading message""" + if platform is not None: + redirect_out = StringIO() + with redirect_stdout(redirect_out): + loaded = Backends(platform, tag=tag) + with_symbol = ' '.join([ + 'Loading', + loaded.platform.symbol, + loaded.platform.name + ]) + assert with_symbol in redirect_out.getvalue() + + redirect_out = TextIOWrapper( + BytesIO(), encoding='latin-1', errors='strict', write_through=True) + with redirect_stdout(redirect_out): + loaded = Backends(platform, tag=tag) + without_symbol = ' '.join([ + 'Loading', + loaded.platform.name + ]) + # pylint: disable=no-member + assert without_symbol in redirect_out.buffer.getvalue().decode() @pytest.mark.parametrize('argsep', [' ', '=']) -def test_pull(argsep, capsys, platform=None, tag=None): +def test_pull(argsep, capsys, platform, tag): + """Test pull command""" def run_test(argv): + argv = [arg for arg in argv if arg] with mock.patch.object(sys, 'argv', argv): run() captured = capsys.readouterr() checkstring = f':{tag}' if tag is not None else ':latest' - assert( - checkstring in captured.out + captured.err - ) + outstring = captured.out + captured.err + assert checkstring in outstring or 'cached' in outstring args = set_commandline_args(platform, tag, argsep) diff --git a/tests/test_cpac_crash.py b/tests/test_cpac_crash.py index efcfa464..cd3b41a8 100644 --- a/tests/test_cpac_crash.py +++ b/tests/test_cpac_crash.py @@ -1,26 +1,25 @@ import os -import pytest import sys from unittest import mock +import pytest + from cpac.__main__ import run -from CONSTANTS import set_commandline_args +from .CONSTANTS import set_commandline_args @pytest.mark.parametrize('argsep', [' ', '=']) -def test_cpac_crash(argsep, capsys, platform=None, tag=None): +def test_cpac_crash(argsep, capsys, platform, tag): args = set_commandline_args(platform, tag, argsep) crashfile = os.path.join( os.path.dirname(__file__), 'test_data', 'test_pickle.pklz' ) argv = ['cpac', 'crash', crashfile] - argv = ' '.join([ + argv = [arg for arg in ' '.join([ w for w in ['cpac', args, 'crash', crashfile] if len(w) - ]).split(' ') + ]).split(' ') if arg] with mock.patch.object(sys, 'argv', argv): run() captured = capsys.readouterr() - assert( - "MemoryError" in captured.out or "MemoryError" in captured.err - ) + assert "MemoryError" in captured.out or "MemoryError" in captured.err diff --git a/tests/test_cpac_group.py b/tests/test_cpac_group.py index 9c7acc76..86c80fcd 100644 --- a/tests/test_cpac_group.py +++ b/tests/test_cpac_group.py @@ -1,15 +1,17 @@ -import pytest import sys from unittest import mock +import pytest + from cpac.__main__ import run -from CONSTANTS import args_before_after, set_commandline_args +from .CONSTANTS import args_before_after, set_commandline_args @pytest.mark.parametrize('argsep', [' ', '=']) -def test_utils_help(argsep, capsys, platform=None, tag=None): +def test_utils_help(argsep, capsys, platform, tag): def run_test(argv, platform): + argv = [arg for arg in argv if arg] with mock.patch.object(sys, 'argv', argv): run() captured = capsys.readouterr() diff --git a/tests/test_cpac_run.py b/tests/test_cpac_run.py index e6b4be4c..c4b2b5f0 100644 --- a/tests/test_cpac_run.py +++ b/tests/test_cpac_run.py @@ -1,12 +1,14 @@ import os -import pytest import sys from datetime import date from unittest import mock +import pytest + from cpac.__main__ import run -from CONSTANTS import args_before_after, set_commandline_args +from cpac.utils import check_version_at_least +from .CONSTANTS import args_before_after, set_commandline_args MINIMAL_CONFIG = os.path.join( os.path.dirname(__file__), 'test_data', 'minimal.min.yml' @@ -15,8 +17,9 @@ @pytest.mark.parametrize('helpflag', ['--help', '-h']) @pytest.mark.parametrize('argsep', [' ', '=']) -def test_run_help(argsep, capsys, helpflag, platform=None, tag=None): +def test_run_help(argsep, capsys, helpflag, platform, tag): def run_test(argv): + argv = [arg for arg in argv if arg] with mock.patch.object(sys, 'argv', argv): run() captured = capsys.readouterr() @@ -37,17 +40,16 @@ def run_test(argv): @pytest.mark.parametrize('argsep', [' ', '=']) @pytest.mark.parametrize('pipeline_file', [None, MINIMAL_CONFIG]) -def test_run_test_config( - argsep, pipeline_file, tmp_path, platform=None, tag=None -): - def run_test(argv): +def test_run_test_config(argsep, pipeline_file, tmp_path, platform, tag): + """Test 'test_config' run command""" + def run_test(argv, wd): # pylint: disable=invalid-name + argv = [arg for arg in argv if arg] with mock.patch.object(sys, 'argv', argv): run() - assert( - any([date.today().isoformat() in fp for fp in os.listdir(wd)]) - ) + assert any( + date.today().isoformat() in fp for fp in os.listdir(wd)) - wd = tmp_path + wd = tmp_path # pylint: disable=invalid-name args = set_commandline_args(platform, tag, argsep) pipeline = '' if pipeline_file is None else ' '.join([ ' --pipeline_file', @@ -58,12 +60,14 @@ def run_test(argv): f's3://fcp-indi/data/Projects/ABIDE/RawDataBIDS/NYU {wd} ' f'test_config --participant_ndx=2{pipeline}' ) - if len(args): + if check_version_at_least('1.8.4', platform): + argv += ' --tracking_opt-out' + if args: before, after = args_before_after(argv, args) # test with args before command - run_test(before) + run_test(before, wd) # test with args after command - run_test(after) + run_test(after, wd) else: # test without --platform and --tag args - run_test(f'cpac {argv}'.split(' ')) + run_test(f'cpac {argv}'.split(' '), wd) diff --git a/tests/test_cpac_utils.py b/tests/test_cpac_utils.py index 6bcd644b..b2609a07 100644 --- a/tests/test_cpac_utils.py +++ b/tests/test_cpac_utils.py @@ -1,17 +1,20 @@ import os -import pytest import sys from unittest import mock +import pytest + from cpac.__main__ import run -from CONSTANTS import args_before_after, set_commandline_args +from cpac.utils import check_version_at_least +from .CONSTANTS import args_before_after, set_commandline_args @pytest.mark.parametrize('argsep', [' ', '=']) @pytest.mark.parametrize('helpflag', ['--help', '-h']) -def test_utils_help(argsep, capsys, helpflag, platform=None, tag=None): +def test_utils_help(argsep, capsys, helpflag, platform, tag): def run_test(argv, platform): + argv = [arg for arg in argv if arg] with mock.patch.object(sys, 'argv', argv): run() captured = capsys.readouterr() @@ -33,20 +36,22 @@ def run_test(argv, platform): @pytest.mark.parametrize('argsep', [' ', '=']) -def test_utils_new_settings_template( - argsep, tmp_path, platform=None, tag=None -): - wd = tmp_path +def test_utils_new_settings_template(argsep, tmp_path, platform, tag): + """Test 'utils data_config new_settings_template' command""" + wd = tmp_path # pylint: disable=invalid-name def run_test(argv): + argv = [arg for arg in argv if arg] with mock.patch.object(sys, 'argv', argv): run() template_path = os.path.join(wd, 'data_settings.yml') - assert(os.path.exists(template_path)) + assert os.path.exists(template_path) args = set_commandline_args(platform, tag, argsep) argv = f'--working_dir {wd} utils data_config new_settings_template' - if len(args): + if check_version_at_least('1.8.4', platform): + argv += ' --tracking_opt-out' + if args: before, after = args_before_after(argv, args) # test with args before command run_test(before)