From 08948a418c6b5294bb4f0b5a9db59e394de35a64 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 15 Oct 2024 13:21:18 -0400 Subject: [PATCH 01/33] :goal_net: Fall back on workflow name, then empty string if no "subject_id" in `cfg` --- CPAC/pipeline/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CPAC/pipeline/engine.py b/CPAC/pipeline/engine.py index 09be47823a..6488713023 100644 --- a/CPAC/pipeline/engine.py +++ b/CPAC/pipeline/engine.py @@ -954,7 +954,7 @@ def post_process(self, wf, label, connection, json_info, pipe_idx, pipe_x, def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None): excl = [] substring_excl = [] - outputs_logger = getLogger(f'{cfg["subject_id"]}_expectedOutputs') + outputs_logger = getLogger(f'{cfg.get("subject_id", getattr(wf, "name", ""))}_expectedOutputs') expected_outputs = ExpectedOutputs() if add_excl: From 2874287c9d02109465b0038c3b468a9b88c55d17 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 15 Oct 2024 13:26:23 -0400 Subject: [PATCH 02/33] :bug: Set subject ID for longitudinal workflow --- CPAC/longitudinal_pipeline/longitudinal_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index fb12d49ab7..3a7075c02f 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -421,7 +421,7 @@ def anat_longitudinal_wf(subject_id, sub_list, config): ------- None """ - + config["subject_id"] = subject_id # list of lists for every strategy session_id_list = [] session_wfs = {} From 47f1de130cf6c3383e5ca56857efaede14765e78 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 15 Oct 2024 13:45:29 -0400 Subject: [PATCH 03/33] :bookmark: Set version to `1.8.7.post1.dev1` --- CHANGELOG.md | 9 ++++++++- CPAC/info.py | 2 +- CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml | 2 +- CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml | 2 +- .../configs/data_config_S3-BIDS-ADHD200_only2.yml | 2 +- .../configs/data_config_S3-BIDS-NKI-RocklandSample.yml | 2 +- CPAC/resources/configs/data_config_cpac_benchmark.yml | 2 +- CPAC/resources/configs/data_settings_template.yml | 2 +- CPAC/resources/configs/group_config_template.yml | 2 +- CPAC/resources/configs/pipeline_config_abcd-options.yml | 2 +- CPAC/resources/configs/pipeline_config_abcd-prep.yml | 2 +- CPAC/resources/configs/pipeline_config_anat-only.yml | 2 +- .../resources/configs/pipeline_config_benchmark-ANTS.yml | 2 +- .../configs/pipeline_config_benchmark-FNIRT.yml | 2 +- CPAC/resources/configs/pipeline_config_blank.yml | 2 +- CPAC/resources/configs/pipeline_config_ccs-options.yml | 2 +- .../configs/pipeline_config_default-deprecated.yml | 2 +- CPAC/resources/configs/pipeline_config_default.yml | 2 +- .../configs/pipeline_config_fmriprep-ingress.yml | 2 +- .../configs/pipeline_config_fmriprep-options.yml | 2 +- CPAC/resources/configs/pipeline_config_fx-options.yml | 2 +- CPAC/resources/configs/pipeline_config_monkey-ABCD.yml | 2 +- CPAC/resources/configs/pipeline_config_monkey.yml | 2 +- CPAC/resources/configs/pipeline_config_ndmg.yml | 2 +- CPAC/resources/configs/pipeline_config_nhp-macaque.yml | 2 +- CPAC/resources/configs/pipeline_config_preproc.yml | 2 +- CPAC/resources/configs/pipeline_config_rbc-options.yml | 2 +- CPAC/resources/configs/pipeline_config_regtest-1.yml | 2 +- CPAC/resources/configs/pipeline_config_regtest-2.yml | 2 +- CPAC/resources/configs/pipeline_config_regtest-3.yml | 2 +- CPAC/resources/configs/pipeline_config_regtest-4.yml | 2 +- CPAC/resources/configs/pipeline_config_rodent.yml | 2 +- CPAC/resources/configs/system_config.yml | 2 +- .../configs/test_configs/data-test_S3-ADHD200_1.yml | 2 +- .../test_configs/data-test_S3-ADHD200_no-params.yml | 2 +- .../configs/test_configs/data-test_S3-NKI-RS_fmap.yml | 2 +- .../test_configs/data_config_S3_CoRR_5only_mult-scan.yml | 2 +- .../test_configs/data_config_S3_CoRR_5only_mult-sess.yml | 2 +- CPAC/resources/configs/test_configs/pipe-test_ABCD.yml | 2 +- .../configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml | 2 +- .../test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml | 2 +- .../test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml | 2 +- .../configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml | 2 +- .../test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml | 2 +- .../test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml | 2 +- .../pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml | 2 +- .../test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml | 2 +- .../test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml | 2 +- .../configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml | 2 +- CPAC/resources/configs/test_configs/pipe-test_all.yml | 2 +- version | 2 +- 51 files changed, 58 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd2ad749db..0ece254322 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.8.7.post1.dev1] - unreleased + +### Fixed + +- Supplied missing `subject_id` for longitudinal workflow logger and make that field optional for the logger. + ## [1.8.7] - 2024-05-03 ### Added @@ -294,7 +300,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 See [Version 1.8.1 Beta](https://fcp-indi.github.io/docs/user/release_notes/v1.8.1) for release notes for v1.8.1 and [Release Notes](https://fcp-indi.github.io/docs/user/release_notes) for all release notes back to v0.1.1. -[unreleased]: https://github.com/FCP-INDI/C-PAC/compare/v1.8.6...develop +[1.8.7.post1.dev1]: https://github.com/FCP-INDI/C-PAC/compare/v1.8.7...v1.8.7.post1.dev1 +[1.8.7]: https://github.com/FCP-INDI/C-PAC/releases/tag/v1.8.7 [1.8.6]: https://github.com/FCP-INDI/C-PAC/releases/tag/v1.8.6 [1.8.5]: https://github.com/FCP-INDI/C-PAC/releases/tag/v1.8.5 [1.8.4]: https://github.com/FCP-INDI/C-PAC/releases/tag/v1.8.4 diff --git a/CPAC/info.py b/CPAC/info.py index f2fcd5481b..5e8c7c0221 100644 --- a/CPAC/info.py +++ b/CPAC/info.py @@ -43,7 +43,7 @@ _version_major = 1 _version_minor = 8 _version_micro = 7 -_version_extra = 'dev1' +_version_extra = 'post1.dev1' def get_cpac_gitversion(): diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml b/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml index d674131633..a9869a5e15 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml index cae981cd9a..93c3301fc5 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml index 0ef6d3ccf7..4c2a6b22df 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml b/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml index 77a62527e6..d9e2543e6f 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_cpac_benchmark.yml b/CPAC/resources/configs/data_config_cpac_benchmark.yml index 15460d264c..a81596f37d 100644 --- a/CPAC/resources/configs/data_config_cpac_benchmark.yml +++ b/CPAC/resources/configs/data_config_cpac_benchmark.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_settings_template.yml b/CPAC/resources/configs/data_settings_template.yml index 1d6c1b912e..8fee565efa 100644 --- a/CPAC/resources/configs/data_settings_template.yml +++ b/CPAC/resources/configs/data_settings_template.yml @@ -1,5 +1,5 @@ # CPAC Data Settings File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/group_config_template.yml b/CPAC/resources/configs/group_config_template.yml index 996c2386f0..9bbdbd8274 100644 --- a/CPAC/resources/configs/group_config_template.yml +++ b/CPAC/resources/configs/group_config_template.yml @@ -1,5 +1,5 @@ # CPAC Group-Level Analysis Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_abcd-options.yml b/CPAC/resources/configs/pipeline_config_abcd-options.yml index 10ea6bece3..5944d5aaab 100644 --- a/CPAC/resources/configs/pipeline_config_abcd-options.yml +++ b/CPAC/resources/configs/pipeline_config_abcd-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_abcd-prep.yml b/CPAC/resources/configs/pipeline_config_abcd-prep.yml index 7aee4e80ad..27fc0d2b9c 100644 --- a/CPAC/resources/configs/pipeline_config_abcd-prep.yml +++ b/CPAC/resources/configs/pipeline_config_abcd-prep.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_anat-only.yml b/CPAC/resources/configs/pipeline_config_anat-only.yml index 01e902abc0..6124d8a5b9 100644 --- a/CPAC/resources/configs/pipeline_config_anat-only.yml +++ b/CPAC/resources/configs/pipeline_config_anat-only.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml b/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml index f59be35958..e69f4ce085 100644 --- a/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml +++ b/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml b/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml index 8fb774aa80..a4798fe57e 100644 --- a/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml +++ b/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_blank.yml b/CPAC/resources/configs/pipeline_config_blank.yml index f4c049b1b5..214ed481a3 100644 --- a/CPAC/resources/configs/pipeline_config_blank.yml +++ b/CPAC/resources/configs/pipeline_config_blank.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_ccs-options.yml b/CPAC/resources/configs/pipeline_config_ccs-options.yml index 61207908f0..1bde92b32a 100644 --- a/CPAC/resources/configs/pipeline_config_ccs-options.yml +++ b/CPAC/resources/configs/pipeline_config_ccs-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_default-deprecated.yml b/CPAC/resources/configs/pipeline_config_default-deprecated.yml index 9a11649dee..20f33d2c9c 100644 --- a/CPAC/resources/configs/pipeline_config_default-deprecated.yml +++ b/CPAC/resources/configs/pipeline_config_default-deprecated.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_default.yml b/CPAC/resources/configs/pipeline_config_default.yml index 71609ce58d..58b187f8a2 100644 --- a/CPAC/resources/configs/pipeline_config_default.yml +++ b/CPAC/resources/configs/pipeline_config_default.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml b/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml index cb81ba5958..92fb58566a 100644 --- a/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml +++ b/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fmriprep-options.yml b/CPAC/resources/configs/pipeline_config_fmriprep-options.yml index b432770220..4d8389e370 100644 --- a/CPAC/resources/configs/pipeline_config_fmriprep-options.yml +++ b/CPAC/resources/configs/pipeline_config_fmriprep-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fx-options.yml b/CPAC/resources/configs/pipeline_config_fx-options.yml index ade6c66940..cbe21300b7 100644 --- a/CPAC/resources/configs/pipeline_config_fx-options.yml +++ b/CPAC/resources/configs/pipeline_config_fx-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml b/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml index b598e38df3..f2d3045cdb 100644 --- a/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml +++ b/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_monkey.yml b/CPAC/resources/configs/pipeline_config_monkey.yml index 09bf8d48ef..6b20c32e4d 100644 --- a/CPAC/resources/configs/pipeline_config_monkey.yml +++ b/CPAC/resources/configs/pipeline_config_monkey.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_ndmg.yml b/CPAC/resources/configs/pipeline_config_ndmg.yml index 70d27d8f7b..29200f026c 100644 --- a/CPAC/resources/configs/pipeline_config_ndmg.yml +++ b/CPAC/resources/configs/pipeline_config_ndmg.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_nhp-macaque.yml b/CPAC/resources/configs/pipeline_config_nhp-macaque.yml index bd4bb18b92..0d4da110ba 100644 --- a/CPAC/resources/configs/pipeline_config_nhp-macaque.yml +++ b/CPAC/resources/configs/pipeline_config_nhp-macaque.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_preproc.yml b/CPAC/resources/configs/pipeline_config_preproc.yml index 8326a41696..3565902c87 100644 --- a/CPAC/resources/configs/pipeline_config_preproc.yml +++ b/CPAC/resources/configs/pipeline_config_preproc.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_rbc-options.yml b/CPAC/resources/configs/pipeline_config_rbc-options.yml index eb27e51c8e..47cb595fce 100644 --- a/CPAC/resources/configs/pipeline_config_rbc-options.yml +++ b/CPAC/resources/configs/pipeline_config_rbc-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-1.yml b/CPAC/resources/configs/pipeline_config_regtest-1.yml index fccbb36fc1..8197651177 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-1.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-1.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-2.yml b/CPAC/resources/configs/pipeline_config_regtest-2.yml index 08d546bdc2..f4e3362a02 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-2.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-2.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-3.yml b/CPAC/resources/configs/pipeline_config_regtest-3.yml index 77febb731f..45aaa47bc9 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-3.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-3.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-4.yml b/CPAC/resources/configs/pipeline_config_regtest-4.yml index 308d64cefa..d70dd0abce 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-4.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-4.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_rodent.yml b/CPAC/resources/configs/pipeline_config_rodent.yml index f829296a26..1b2d36b278 100644 --- a/CPAC/resources/configs/pipeline_config_rodent.yml +++ b/CPAC/resources/configs/pipeline_config_rodent.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/system_config.yml b/CPAC/resources/configs/system_config.yml index 291adc19a9..ed0490942b 100644 --- a/CPAC/resources/configs/system_config.yml +++ b/CPAC/resources/configs/system_config.yml @@ -1,5 +1,5 @@ # C-PAC System Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml index 97679f1f76..ad8e75d19a 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml index 3d1a4f46ca..65d247f2ce 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml b/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml index 080595f840..6a7e215b2d 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml index 0d428b0340..58e1f0eead 100644 --- a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml +++ b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml index 1d55f02e82..4405a67db4 100644 --- a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml +++ b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml b/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml index 39023ec11d..5a14d628a4 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml index e68f31d827..4cd4db8509 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml index 510e3a7ef9..aa07eff09f 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml index e6940dc729..51f4ff8f8b 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml index 0e9e8f34c5..27cb9a66e2 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml index 3326d3427f..8f66e61ebf 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml index 8f4faae6c1..a968af6506 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml index 8f4faae6c1..a968af6506 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml index 8f4faae6c1..a968af6506 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml index 8f4faae6c1..a968af6506 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml index 8f4faae6c1..a968af6506 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_all.yml b/CPAC/resources/configs/test_configs/pipe-test_all.yml index b6feb9c42c..4144ef4cee 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_all.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_all.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.dev1 +# Version 1.8.7.post1.dev1 # # http://fcp-indi.github.io for more info. # diff --git a/version b/version index f4c717aa49..13344e5308 100644 --- a/version +++ b/version @@ -1 +1 @@ -v1.8.7.dev1 +v1.8.7.post1.dev1 From cc72177e6b967a9b6ffd9a618d40c93ec9bdb5e5 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 15 Oct 2024 14:10:31 -0400 Subject: [PATCH 04/33] :package: Disable variant image builds --- .circleci/main.yml | 24 ++++---- .github/stage_requirements/phase_three.txt | 4 +- .github/workflows/build_and_test.yml | 64 +++++++++++----------- CHANGELOG.md | 4 ++ 4 files changed, 49 insertions(+), 47 deletions(-) diff --git a/.circleci/main.yml b/.circleci/main.yml index ee836c0607..320b817f38 100644 --- a/.circleci/main.yml +++ b/.circleci/main.yml @@ -184,8 +184,8 @@ jobs: key: coverage-docker--{{ .Revision }} # - restore_cache: # key: coverage-singularity--{{ .Revision }} - - restore_cache: - key: coverage-docker-ABCD-HCP-{{ .Revision }} + # - restore_cache: + # key: coverage-docker-ABCD-HCP-{{ .Revision }} # - restore_cache: # key: coverage-singularity-ABCD-HCP-{{ .Revision }} - restore_cache: @@ -280,9 +280,9 @@ workflows: # - "Test in Singularity" - "Test lite variant in Docker" # - "Test lite variant in Singularity" - - "Test ABCD-HCP variant in Docker" + # - "Test ABCD-HCP variant in Docker" # - "Test ABCD-HCP variant in Singularity" - - "Test fMRIPrep-LTS variant in Docker" + # - "Test fMRIPrep-LTS variant in Docker" - push-branch-to-docker-hub: filters: branches: @@ -293,18 +293,18 @@ workflows: variant: - "" - lite - - ABCD-HCP - - fMRIPrep-LTS + # - ABCD-HCP + # - fMRIPrep-LTS requires: - "Combine coverage" - pytest-docker: name: "Test in Docker" - - pytest-docker: - name: "Test ABCD-HCP variant in Docker" - variant: ABCD-HCP - - pytest-docker: - name: "Test fMRIPrep-LTS variant in Docker" - variant: fMRIPrep-LTS + # - pytest-docker: + # name: "Test ABCD-HCP variant in Docker" + # variant: ABCD-HCP + # - pytest-docker: + # name: "Test fMRIPrep-LTS variant in Docker" + # variant: fMRIPrep-LTS - pytest-docker: name: "Test lite variant in Docker" variant: lite diff --git a/.github/stage_requirements/phase_three.txt b/.github/stage_requirements/phase_three.txt index b71c08de24..349b6827fe 100644 --- a/.github/stage_requirements/phase_three.txt +++ b/.github/stage_requirements/phase_three.txt @@ -1,3 +1 @@ -lite -ABCD-HCP -fMRIPrep-LTS \ No newline at end of file +lite \ No newline at end of file diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 62a7a34f4b..44722e426d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -317,25 +317,25 @@ jobs: with: variant: lite - C-PAC-ABCD-HCP: - needs: build-base - uses: ./.github/workflows/build_C-PAC.yml - with: - variant: ABCD-HCP + # C-PAC-ABCD-HCP: + # needs: build-base + # uses: ./.github/workflows/build_C-PAC.yml + # with: + # variant: ABCD-HCP - C-PAC-fMRIPrep-LTS: - needs: build-base - uses: ./.github/workflows/build_C-PAC.yml - with: - variant: fMRIPrep-LTS + # C-PAC-fMRIPrep-LTS: + # needs: build-base + # uses: ./.github/workflows/build_C-PAC.yml + # with: + # variant: fMRIPrep-LTS smoke-tests-participant: name: Run participant-level smoke tests needs: - C-PAC - C-PAC-lite - - C-PAC-ABCD-HCP - - C-PAC-fMRIPrep-LTS + # - C-PAC-ABCD-HCP + # - C-PAC-fMRIPrep-LTS if: github.ref_name == 'develop' || github.ref_name == 'main' uses: ./.github/workflows/smoke_test_participant.yml @@ -358,8 +358,8 @@ jobs: needs: - C-PAC - C-PAC-lite - - C-PAC-ABCD-HCP - - C-PAC-fMRIPrep-LTS + # - C-PAC-ABCD-HCP + # - C-PAC-fMRIPrep-LTS if: github.ref_type == 'branch' runs-on: ubuntu-latest steps: @@ -425,22 +425,22 @@ jobs: variant: 'lite' secrets: inherit - Deploy_to_Docker_Hub-ABCD-HCP: - name: Deploy 'ABCD-HCP' to Docker Hub - needs: - - C-PAC-ABCD-HCP - if: github.ref_type == 'tag' - uses: ./.github/workflows/deploy_to_Docker_Hub.yml - with: - variant: 'ABCD-HCP' - secrets: inherit + # Deploy_to_Docker_Hub-ABCD-HCP: + # name: Deploy 'ABCD-HCP' to Docker Hub + # needs: + # - C-PAC-ABCD-HCP + # if: github.ref_type == 'tag' + # uses: ./.github/workflows/deploy_to_Docker_Hub.yml + # with: + # variant: 'ABCD-HCP' + # secrets: inherit - Deploy_to_Docker_Hub-fMRIPrep-LTS: - name: Deploy 'fMRIPrep-LTS' to Docker Hub - needs: - - C-PAC-fMRIPrep-LTS - if: github.ref_type == 'tag' - uses: ./.github/workflows/deploy_to_Docker_Hub.yml - with: - variant: 'fMRIPrep-LTS' - secrets: inherit + # Deploy_to_Docker_Hub-fMRIPrep-LTS: + # name: Deploy 'fMRIPrep-LTS' to Docker Hub + # needs: + # - C-PAC-fMRIPrep-LTS + # if: github.ref_type == 'tag' + # uses: ./.github/workflows/deploy_to_Docker_Hub.yml + # with: + # variant: 'fMRIPrep-LTS' + # secrets: inherit diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ece254322..404bea075a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.8.7.post1.dev1] - unreleased +### Changed + +- Disabled variant image builds. + ### Fixed - Supplied missing `subject_id` for longitudinal workflow logger and make that field optional for the logger. From d2f960a89b0c8f602a147bd0becf32c954dfa3b1 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 22 Oct 2024 15:14:36 -0400 Subject: [PATCH 05/33] :bug: Pass `get` method through from `Configuration.dict()` --- .../Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile | 2 +- .../C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile | 2 +- .github/Dockerfiles/C-PAC.develop-jammy.Dockerfile | 2 +- .github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile | 2 +- .github/Dockerfiles/base-standard.Dockerfile | 2 +- CPAC/info.py | 2 +- CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml | 2 +- CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml | 2 +- .../configs/data_config_S3-BIDS-ADHD200_only2.yml | 2 +- .../configs/data_config_S3-BIDS-NKI-RocklandSample.yml | 2 +- CPAC/resources/configs/data_config_cpac_benchmark.yml | 2 +- CPAC/resources/configs/data_settings_template.yml | 2 +- CPAC/resources/configs/group_config_template.yml | 2 +- CPAC/resources/configs/pipeline_config_abcd-options.yml | 2 +- CPAC/resources/configs/pipeline_config_abcd-prep.yml | 2 +- CPAC/resources/configs/pipeline_config_anat-only.yml | 2 +- CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml | 2 +- .../resources/configs/pipeline_config_benchmark-FNIRT.yml | 2 +- CPAC/resources/configs/pipeline_config_blank.yml | 2 +- CPAC/resources/configs/pipeline_config_ccs-options.yml | 2 +- .../configs/pipeline_config_default-deprecated.yml | 2 +- CPAC/resources/configs/pipeline_config_default.yml | 2 +- .../configs/pipeline_config_fmriprep-ingress.yml | 2 +- .../configs/pipeline_config_fmriprep-options.yml | 2 +- CPAC/resources/configs/pipeline_config_fx-options.yml | 2 +- CPAC/resources/configs/pipeline_config_monkey-ABCD.yml | 2 +- CPAC/resources/configs/pipeline_config_monkey.yml | 2 +- CPAC/resources/configs/pipeline_config_ndmg.yml | 2 +- CPAC/resources/configs/pipeline_config_nhp-macaque.yml | 2 +- CPAC/resources/configs/pipeline_config_preproc.yml | 2 +- CPAC/resources/configs/pipeline_config_rbc-options.yml | 2 +- CPAC/resources/configs/pipeline_config_regtest-1.yml | 2 +- CPAC/resources/configs/pipeline_config_regtest-2.yml | 2 +- CPAC/resources/configs/pipeline_config_regtest-3.yml | 2 +- CPAC/resources/configs/pipeline_config_regtest-4.yml | 2 +- CPAC/resources/configs/pipeline_config_rodent.yml | 2 +- CPAC/resources/configs/system_config.yml | 2 +- .../configs/test_configs/data-test_S3-ADHD200_1.yml | 2 +- .../test_configs/data-test_S3-ADHD200_no-params.yml | 2 +- .../configs/test_configs/data-test_S3-NKI-RS_fmap.yml | 2 +- .../test_configs/data_config_S3_CoRR_5only_mult-scan.yml | 2 +- .../test_configs/data_config_S3_CoRR_5only_mult-sess.yml | 2 +- CPAC/resources/configs/test_configs/pipe-test_ABCD.yml | 2 +- .../configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml | 2 +- .../test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml | 2 +- .../test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml | 2 +- .../configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml | 2 +- .../configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml | 2 +- .../test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml | 2 +- .../pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml | 2 +- .../test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml | 2 +- .../test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml | 2 +- .../configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml | 2 +- CPAC/resources/configs/test_configs/pipe-test_all.yml | 2 +- CPAC/utils/configuration/configuration.py | 8 ++++++-- Dockerfile | 2 +- variant-ABCD-HCP.Dockerfile | 2 +- variant-fMRIPrep-LTS.Dockerfile | 2 +- variant-lite.Dockerfile | 2 +- version | 2 +- 60 files changed, 65 insertions(+), 61 deletions(-) diff --git a/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile b/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile index 2a5760ebb7..2a165f301d 100644 --- a/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile +++ b/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [ABCD-HCP BIDS fMRI Pipeline](https://github.com/DCAN-Labs/abcd-hcp-pipeline/blob/e480a8f99534f1b05f37bf44c64827384b69b383/Dockerfile)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile b/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile index 0e9cd3d899..096d3e2b0b 100644 --- a/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile +++ b/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [fMRIPrep LTS](https://reproducibility.stanford.edu/fmriprep-lts#long-term-support-lts)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile b/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile index f9ced12d7b..03f04e7cea 100644 --- a/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile +++ b/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -FROM ghcr.io/fcp-indi/c-pac/stage-base:standard-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:standard-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "Full C-PAC image" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile b/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile index 1f6f3a9ec9..4a186031f0 100644 --- a/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile +++ b/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "Full C-PAC image without FreeSurfer" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/.github/Dockerfiles/base-standard.Dockerfile b/.github/Dockerfiles/base-standard.Dockerfile index 2d0d51ab76..aa8f638dbc 100644 --- a/.github/Dockerfiles/base-standard.Dockerfile +++ b/.github/Dockerfiles/base-standard.Dockerfile @@ -16,7 +16,7 @@ # License along with C-PAC. If not, see . FROM ghcr.io/fcp-indi/c-pac/freesurfer:6.0.0-min.neurodocker-jammy as FreeSurfer -FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \ Standard software dependencies for C-PAC standard images" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC diff --git a/CPAC/info.py b/CPAC/info.py index 5e8c7c0221..7167b8a039 100644 --- a/CPAC/info.py +++ b/CPAC/info.py @@ -43,7 +43,7 @@ _version_major = 1 _version_minor = 8 _version_micro = 7 -_version_extra = 'post1.dev1' +_version_extra = 'post1.dev2' def get_cpac_gitversion(): diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml b/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml index a9869a5e15..457e246934 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml index 93c3301fc5..6ccf800c25 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml index 4c2a6b22df..0a756d99f4 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml b/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml index d9e2543e6f..820eeeef3e 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_cpac_benchmark.yml b/CPAC/resources/configs/data_config_cpac_benchmark.yml index a81596f37d..a3491d702b 100644 --- a/CPAC/resources/configs/data_config_cpac_benchmark.yml +++ b/CPAC/resources/configs/data_config_cpac_benchmark.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_settings_template.yml b/CPAC/resources/configs/data_settings_template.yml index 8fee565efa..70414856f3 100644 --- a/CPAC/resources/configs/data_settings_template.yml +++ b/CPAC/resources/configs/data_settings_template.yml @@ -1,5 +1,5 @@ # CPAC Data Settings File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/group_config_template.yml b/CPAC/resources/configs/group_config_template.yml index 9bbdbd8274..0a3ff76984 100644 --- a/CPAC/resources/configs/group_config_template.yml +++ b/CPAC/resources/configs/group_config_template.yml @@ -1,5 +1,5 @@ # CPAC Group-Level Analysis Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_abcd-options.yml b/CPAC/resources/configs/pipeline_config_abcd-options.yml index 5944d5aaab..1ba775612f 100644 --- a/CPAC/resources/configs/pipeline_config_abcd-options.yml +++ b/CPAC/resources/configs/pipeline_config_abcd-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_abcd-prep.yml b/CPAC/resources/configs/pipeline_config_abcd-prep.yml index 27fc0d2b9c..d664397fd6 100644 --- a/CPAC/resources/configs/pipeline_config_abcd-prep.yml +++ b/CPAC/resources/configs/pipeline_config_abcd-prep.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_anat-only.yml b/CPAC/resources/configs/pipeline_config_anat-only.yml index 6124d8a5b9..a935cc330b 100644 --- a/CPAC/resources/configs/pipeline_config_anat-only.yml +++ b/CPAC/resources/configs/pipeline_config_anat-only.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml b/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml index e69f4ce085..d6a2f084ae 100644 --- a/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml +++ b/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml b/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml index a4798fe57e..1af7eb337d 100644 --- a/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml +++ b/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_blank.yml b/CPAC/resources/configs/pipeline_config_blank.yml index 214ed481a3..fc56a42fb4 100644 --- a/CPAC/resources/configs/pipeline_config_blank.yml +++ b/CPAC/resources/configs/pipeline_config_blank.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_ccs-options.yml b/CPAC/resources/configs/pipeline_config_ccs-options.yml index 1bde92b32a..351ec8a18c 100644 --- a/CPAC/resources/configs/pipeline_config_ccs-options.yml +++ b/CPAC/resources/configs/pipeline_config_ccs-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_default-deprecated.yml b/CPAC/resources/configs/pipeline_config_default-deprecated.yml index 20f33d2c9c..f29ad21ca3 100644 --- a/CPAC/resources/configs/pipeline_config_default-deprecated.yml +++ b/CPAC/resources/configs/pipeline_config_default-deprecated.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_default.yml b/CPAC/resources/configs/pipeline_config_default.yml index 58b187f8a2..8dc38545f3 100644 --- a/CPAC/resources/configs/pipeline_config_default.yml +++ b/CPAC/resources/configs/pipeline_config_default.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml b/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml index 92fb58566a..b1219d67ff 100644 --- a/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml +++ b/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fmriprep-options.yml b/CPAC/resources/configs/pipeline_config_fmriprep-options.yml index 4d8389e370..9f62c7dafa 100644 --- a/CPAC/resources/configs/pipeline_config_fmriprep-options.yml +++ b/CPAC/resources/configs/pipeline_config_fmriprep-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fx-options.yml b/CPAC/resources/configs/pipeline_config_fx-options.yml index cbe21300b7..32cb82325e 100644 --- a/CPAC/resources/configs/pipeline_config_fx-options.yml +++ b/CPAC/resources/configs/pipeline_config_fx-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml b/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml index f2d3045cdb..82d19e51f1 100644 --- a/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml +++ b/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_monkey.yml b/CPAC/resources/configs/pipeline_config_monkey.yml index 6b20c32e4d..ee5961b720 100644 --- a/CPAC/resources/configs/pipeline_config_monkey.yml +++ b/CPAC/resources/configs/pipeline_config_monkey.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_ndmg.yml b/CPAC/resources/configs/pipeline_config_ndmg.yml index 29200f026c..0174c3e84d 100644 --- a/CPAC/resources/configs/pipeline_config_ndmg.yml +++ b/CPAC/resources/configs/pipeline_config_ndmg.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_nhp-macaque.yml b/CPAC/resources/configs/pipeline_config_nhp-macaque.yml index 0d4da110ba..c6052fc859 100644 --- a/CPAC/resources/configs/pipeline_config_nhp-macaque.yml +++ b/CPAC/resources/configs/pipeline_config_nhp-macaque.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_preproc.yml b/CPAC/resources/configs/pipeline_config_preproc.yml index 3565902c87..0b9eafe8c2 100644 --- a/CPAC/resources/configs/pipeline_config_preproc.yml +++ b/CPAC/resources/configs/pipeline_config_preproc.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_rbc-options.yml b/CPAC/resources/configs/pipeline_config_rbc-options.yml index 47cb595fce..c6432797d7 100644 --- a/CPAC/resources/configs/pipeline_config_rbc-options.yml +++ b/CPAC/resources/configs/pipeline_config_rbc-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-1.yml b/CPAC/resources/configs/pipeline_config_regtest-1.yml index 8197651177..9ef2b7b825 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-1.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-1.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-2.yml b/CPAC/resources/configs/pipeline_config_regtest-2.yml index f4e3362a02..8561e8fa17 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-2.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-2.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-3.yml b/CPAC/resources/configs/pipeline_config_regtest-3.yml index 45aaa47bc9..d136e154a6 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-3.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-3.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-4.yml b/CPAC/resources/configs/pipeline_config_regtest-4.yml index d70dd0abce..6fdf687c5c 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-4.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-4.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_rodent.yml b/CPAC/resources/configs/pipeline_config_rodent.yml index 1b2d36b278..39c3ff88f7 100644 --- a/CPAC/resources/configs/pipeline_config_rodent.yml +++ b/CPAC/resources/configs/pipeline_config_rodent.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/system_config.yml b/CPAC/resources/configs/system_config.yml index ed0490942b..7a00128ff5 100644 --- a/CPAC/resources/configs/system_config.yml +++ b/CPAC/resources/configs/system_config.yml @@ -1,5 +1,5 @@ # C-PAC System Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml index ad8e75d19a..5e8d6f9320 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml index 65d247f2ce..081cb29165 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml b/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml index 6a7e215b2d..345ca7455d 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml index 58e1f0eead..15c161de2c 100644 --- a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml +++ b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml index 4405a67db4..0149953399 100644 --- a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml +++ b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml b/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml index 5a14d628a4..8a12182a40 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml index 4cd4db8509..ed821b6bcc 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml index aa07eff09f..a21996f40c 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml index 51f4ff8f8b..f6e5b610ad 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml index 27cb9a66e2..3f8481f29a 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml index 8f66e61ebf..514efb3a71 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml index a968af6506..92441194d0 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml index a968af6506..92441194d0 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml index a968af6506..92441194d0 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml index a968af6506..92441194d0 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml index a968af6506..92441194d0 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_all.yml b/CPAC/resources/configs/test_configs/pipe-test_all.yml index 4144ef4cee..43eb774958 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_all.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_all.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev1 +# Version 1.8.7.post1.dev2 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/utils/configuration/configuration.py b/CPAC/utils/configuration/configuration.py index b66ffa39a9..7712fa3b63 100644 --- a/CPAC/utils/configuration/configuration.py +++ b/CPAC/utils/configuration/configuration.py @@ -18,7 +18,7 @@ License along with C-PAC. If not, see .""" import os import re -from typing import Optional +from typing import Any, Optional from warnings import warn import pkg_resources as p from click import BadParameter @@ -244,10 +244,14 @@ def __sub__(self: 'Configuration', other: 'Configuration'): ''' return(dct_diff(self.dict(), other.dict())) - def dict(self): + def dict(self) -> dict[Any, Any]: '''Show contents of a C-PAC configuration as a dict''' return {k: v for k, v in self.__dict__.items() if not callable(v)} + def get(self, key: Any, default: Any = None, /) -> Any: + '''Convenience access to self.dict().get''' + return self.dict().get(key, default) + def keys(self): '''Show toplevel keys of a C-PAC configuration dict''' return self.dict().keys() diff --git a/Dockerfile b/Dockerfile index f9ced12d7b..03f04e7cea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -FROM ghcr.io/fcp-indi/c-pac/stage-base:standard-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:standard-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "Full C-PAC image" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/variant-ABCD-HCP.Dockerfile b/variant-ABCD-HCP.Dockerfile index 2a5760ebb7..2a165f301d 100644 --- a/variant-ABCD-HCP.Dockerfile +++ b/variant-ABCD-HCP.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [ABCD-HCP BIDS fMRI Pipeline](https://github.com/DCAN-Labs/abcd-hcp-pipeline/blob/e480a8f99534f1b05f37bf44c64827384b69b383/Dockerfile)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/variant-fMRIPrep-LTS.Dockerfile b/variant-fMRIPrep-LTS.Dockerfile index 0e9cd3d899..096d3e2b0b 100644 --- a/variant-fMRIPrep-LTS.Dockerfile +++ b/variant-fMRIPrep-LTS.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [fMRIPrep LTS](https://reproducibility.stanford.edu/fmriprep-lts#long-term-support-lts)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/variant-lite.Dockerfile b/variant-lite.Dockerfile index 1f6f3a9ec9..4a186031f0 100644 --- a/variant-lite.Dockerfile +++ b/variant-lite.Dockerfile @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.dev1 +FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev2 LABEL org.opencontainers.image.description "Full C-PAC image without FreeSurfer" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/version b/version index 13344e5308..b659be2ca3 100644 --- a/version +++ b/version @@ -1 +1 @@ -v1.8.7.post1.dev1 +v1.8.7.post1.dev2 From cb5861b0c49ce21482f30a827bc7422fe4f4afb5 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 28 Oct 2024 12:04:25 -0400 Subject: [PATCH 06/33] :necktie: Persist longitudinal resource pool --- CHANGELOG.md | 6 +- CPAC/info.py | 2 +- .../longitudinal_workflow.py | 19 +- .../longitudinal_workflow.py.orig | 1215 +++++++++++++++++ CPAC/pipeline/engine.py | 8 +- .../configs/data_config_S3-BIDS-ABIDE.yml | 2 +- .../configs/data_config_S3-BIDS-ADHD200.yml | 2 +- .../data_config_S3-BIDS-ADHD200_only2.yml | 2 +- ...data_config_S3-BIDS-NKI-RocklandSample.yml | 2 +- .../configs/data_config_cpac_benchmark.yml | 2 +- .../configs/data_settings_template.yml | 2 +- .../configs/group_config_template.yml | 2 +- .../configs/pipeline_config_abcd-options.yml | 2 +- .../configs/pipeline_config_abcd-prep.yml | 2 +- .../configs/pipeline_config_anat-only.yml | 2 +- .../pipeline_config_benchmark-ANTS.yml | 2 +- .../pipeline_config_benchmark-FNIRT.yml | 2 +- .../configs/pipeline_config_blank.yml | 2 +- .../configs/pipeline_config_ccs-options.yml | 2 +- .../pipeline_config_default-deprecated.yml | 2 +- .../configs/pipeline_config_default.yml | 2 +- .../pipeline_config_fmriprep-ingress.yml | 2 +- .../pipeline_config_fmriprep-options.yml | 2 +- .../configs/pipeline_config_fx-options.yml | 2 +- .../configs/pipeline_config_monkey-ABCD.yml | 2 +- .../configs/pipeline_config_monkey.yml | 2 +- .../configs/pipeline_config_ndmg.yml | 2 +- .../configs/pipeline_config_nhp-macaque.yml | 2 +- .../configs/pipeline_config_preproc.yml | 2 +- .../configs/pipeline_config_rbc-options.yml | 2 +- .../configs/pipeline_config_regtest-1.yml | 2 +- .../configs/pipeline_config_regtest-2.yml | 2 +- .../configs/pipeline_config_regtest-3.yml | 2 +- .../configs/pipeline_config_regtest-4.yml | 2 +- .../configs/pipeline_config_rodent.yml | 2 +- CPAC/resources/configs/system_config.yml | 2 +- .../test_configs/data-test_S3-ADHD200_1.yml | 2 +- .../data-test_S3-ADHD200_no-params.yml | 2 +- .../test_configs/data-test_S3-NKI-RS_fmap.yml | 2 +- .../data_config_S3_CoRR_5only_mult-scan.yml | 2 +- .../data_config_S3_CoRR_5only_mult-sess.yml | 2 +- .../configs/test_configs/pipe-test_ABCD.yml | 2 +- .../pipe-test_ANTs-3dSk-AllNuis.yml | 2 +- .../pipe-test_ANTs-3dSk-DistCorr3dSk.yml | 2 +- .../pipe-test_ANTs-3dSk-DistCorrBET.yml | 2 +- .../pipe-test_ANTs-BET-AllNuis.yml | 2 +- .../pipe-test_FNIRT-3dSk-AllNuis.yml | 2 +- .../pipe-test_FNIRT-BET-AllNuis-BASC.yml | 2 +- .../pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml | 2 +- .../pipe-test_FNIRT-BET-AllNuis-ISC.yml | 2 +- .../pipe-test_FNIRT-BET-AllNuis-MDMR.yml | 2 +- .../pipe-test_FNIRT-BET-AllNuis.yml | 2 +- .../configs/test_configs/pipe-test_all.yml | 2 +- version | 2 +- 54 files changed, 1286 insertions(+), 62 deletions(-) create mode 100644 CPAC/longitudinal_pipeline/longitudinal_workflow.py.orig diff --git a/CHANGELOG.md b/CHANGELOG.md index 404bea075a..3d2d06f59f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.8.7.post1.dev1] - unreleased +## [1.8.7.post1] - unreleased ### Changed @@ -22,7 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Supplied missing `subject_id` for longitudinal workflow logger and make that field optional for the logger. +- Graph-building bugs that prevented longitudinal workflows from running. ## [1.8.7] - 2024-05-03 @@ -304,7 +304,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 See [Version 1.8.1 Beta](https://fcp-indi.github.io/docs/user/release_notes/v1.8.1) for release notes for v1.8.1 and [Release Notes](https://fcp-indi.github.io/docs/user/release_notes) for all release notes back to v0.1.1. -[1.8.7.post1.dev1]: https://github.com/FCP-INDI/C-PAC/compare/v1.8.7...v1.8.7.post1.dev1 +[1.8.7.post1]: https://github.com/FCP-INDI/C-PAC/compare/v1.8.7...v1.8.7.post1.dev3 [1.8.7]: https://github.com/FCP-INDI/C-PAC/releases/tag/v1.8.7 [1.8.6]: https://github.com/FCP-INDI/C-PAC/releases/tag/v1.8.6 [1.8.5]: https://github.com/FCP-INDI/C-PAC/releases/tag/v1.8.5 diff --git a/CPAC/info.py b/CPAC/info.py index 7167b8a039..4eb7f43700 100644 --- a/CPAC/info.py +++ b/CPAC/info.py @@ -43,7 +43,7 @@ _version_major = 1 _version_minor = 8 _version_micro = 7 -_version_extra = 'post1.dev2' +_version_extra = 'post1.dev3' def get_cpac_gitversion(): diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 3a7075c02f..7e11d6a7a7 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2020-2022 C-PAC Developers +# Copyright (C) 2020-2024 C-PAC Developers # This file is part of C-PAC. @@ -15,10 +15,12 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -import os import copy -import time +import os import shutil +import time +from typing import Optional + from CPAC.pipeline.nodeblock import nodeblock from nipype import config @@ -41,7 +43,7 @@ from CPAC.pipeline.cpac_pipeline import initialize_nipype_wf, \ connect_pipeline, build_anat_preproc_stack, build_T1w_registration_stack,\ build_segmentation_stack -from CPAC.pipeline.engine import initiate_rpool, ingress_output_dir +from CPAC.pipeline.engine import initiate_rpool, ingress_output_dir, ResourcePool from CPAC.registration import ( create_fsl_flirt_linear_reg, @@ -492,6 +494,8 @@ def anat_longitudinal_wf(subject_id, sub_list, config): strats_head_dct[tag].append(os.path.join(cpac_dir, head_file)) + longitudinal_rpool: Optional[ResourcePool] = None + for strat in strats_brain_dct.keys(): wf = initialize_nipype_wf(config, sub_list[0], @@ -567,6 +571,8 @@ def anat_longitudinal_wf(subject_id, sub_list, config): # once for every strategy! wf.run() + longitudinal_rpool = rpool.copy_rpool() + # now, just write out a copy of the above to each session config.pipeline_setup['pipeline_name'] = orig_pipe_name for session in sub_list: @@ -591,7 +597,7 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf = initialize_nipype_wf(config, sub_list[0]) - wf, rpool = initiate_rpool(wf, config, session) + wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) config.pipeline_setup[ 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' @@ -627,6 +633,7 @@ def anat_longitudinal_wf(subject_id, sub_list, config): rpool.gather_pipes(wf, config, add_excl=excl) wf.run() + longitudinal_rpool = rpool.copy_rpool() # begin single-session stuff again for session in sub_list: @@ -651,7 +658,7 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf = initialize_nipype_wf(config, sub_list[0]) - wf, rpool = initiate_rpool(wf, config, session) + wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) pipeline_blocks = [warp_longitudinal_T1w_to_template, warp_longitudinal_seg_to_T1w] diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py.orig b/CPAC/longitudinal_pipeline/longitudinal_workflow.py.orig new file mode 100644 index 0000000000..73b2a1ff77 --- /dev/null +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py.orig @@ -0,0 +1,1215 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2020-2022 C-PAC Developers + +# This file is part of C-PAC. + +# C-PAC is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. + +# C-PAC is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. + +# You should have received a copy of the GNU Lesser General Public +# License along with C-PAC. If not, see . +import os +<<<<<<< HEAD +import copy +import time +import shutil +from CPAC.pipeline.nodeblock import nodeblock +======= +from typing import Optional +>>>>>>> 4dc13cdc1 (:necktie: Persist longitudinal resource pool) + +from nipype import config +from nipype import logging +from CPAC.pipeline import nipype_pipeline_engine as pe +import nipype.interfaces.afni as afni +import nipype.interfaces.fsl as fsl +import nipype.interfaces.io as nio +from nipype.interfaces.utility import Merge, IdentityInterface +import nipype.interfaces.utility as util + +from indi_aws import aws_utils + +<<<<<<< HEAD +from CPAC.utils.utils import concat_list +from CPAC.utils.interfaces.datasink import DataSink +from CPAC.utils.interfaces.function import Function + +import CPAC + +from CPAC.pipeline.cpac_pipeline import initialize_nipype_wf, \ + connect_pipeline, build_anat_preproc_stack, build_T1w_registration_stack,\ + build_segmentation_stack +from CPAC.pipeline.engine import initiate_rpool, ingress_output_dir + +======= +from CPAC.longitudinal_pipeline.longitudinal_preproc import subject_specific_template +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.cpac_pipeline import ( + build_anat_preproc_stack, + build_segmentation_stack, + build_T1w_registration_stack, + connect_pipeline, + initialize_nipype_wf, +) +from CPAC.pipeline.engine import ingress_output_dir, initiate_rpool, ResourcePool +from CPAC.pipeline.nodeblock import nodeblock +>>>>>>> 4dc13cdc1 (:necktie: Persist longitudinal resource pool) +from CPAC.registration import ( + create_fsl_flirt_linear_reg, + create_fsl_fnirt_nonlinear_reg, + create_wf_calculate_ants_warp +) + +from CPAC.registration.registration import apply_transform + +from CPAC.utils.datasource import ( + resolve_resolution, + create_anat_datasource, + create_check_for_s3_node +) + +from CPAC.longitudinal_pipeline.longitudinal_preproc import ( + subject_specific_template +) + +from CPAC.utils import find_files, function +from CPAC.utils.outputs import Outputs +from CPAC.utils.strategy import Strategy +from CPAC.utils.utils import ( + check_config_resources, + check_prov_for_regtool +) + +logger = logging.getLogger('nipype.workflow') + + +@nodeblock( + name="mask_T1w_longitudinal_template", + config=["longitudinal_template_generation"], + switch=["run"], + inputs=["desc-brain_T1w"], + outputs=["space-T1w_desc-brain_mask"], +) +def mask_T1w_longitudinal_template(wf, cfg, strat_pool, pipe_num, opt=None): + + brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), + name=f'longitudinal_anatomical_brain_mask_' + f'{pipe_num}') + brain_mask.inputs.args = '-bin' + + node, out = strat_pool.get_data('desc-brain_T1w') + wf.connect(node, out, brain_mask, 'in_file') + + outputs = { + 'space-T1w_desc-brain_mask': (brain_mask, 'out_file') + } + + return (wf, outputs) + + +def create_datasink(datasink_name, config, subject_id, session_id='', + strat_name='', map_node_iterfield=None): + """ + + Parameters + ---------- + datasink_name + config + subject_id + session_id + strat_name + map_node_iterfield + + Returns + ------- + + """ + try: + encrypt_data = bool( + config.pipeline_setup['Amazon-AWS']['s3_encryption']) + except: + encrypt_data = False + + # TODO Enforce value with schema validation + # Extract credentials path for output if it exists + try: + # Get path to creds file + creds_path = '' + if config.pipeline_setup['Amazon-AWS'][ + 'aws_output_bucket_credentials']: + creds_path = str(config.pipeline_setup['Amazon-AWS'][ + 'aws_output_bucket_credentials']) + creds_path = os.path.abspath(creds_path) + + if config.pipeline_setup['output_directory'][ + 'path'].lower().startswith('s3://'): + # Test for s3 write access + s3_write_access = \ + aws_utils.test_bucket_access(creds_path, + config.pipeline_setup[ + 'output_directory']['path']) + + if not s3_write_access: + raise Exception('Not able to write to bucket!') + + except Exception as e: + if config.pipeline_setup['output_directory'][ + 'path'].lower().startswith('s3://'): + err_msg = 'There was an error processing credentials or ' \ + 'accessing the S3 bucket. Check and try again.\n' \ + 'Error: %s' % e + raise Exception(err_msg) + + if map_node_iterfield is not None: + ds = pe.MapNode( + DataSink(infields=map_node_iterfield), + name='sinker_{}'.format(datasink_name), + iterfield=map_node_iterfield + ) + else: + ds = pe.Node( + DataSink(), + name='sinker_{}'.format(datasink_name) + ) + + ds.inputs.base_directory = config.pipeline_setup['output_directory'][ + 'path'] + ds.inputs.creds_path = creds_path + ds.inputs.encrypt_bucket_keys = encrypt_data + ds.inputs.container = os.path.join( + 'pipeline_%s_%s' % ( + config.pipeline_setup['pipeline_name'], strat_name), + subject_id, session_id + ) + return ds + + +def connect_anat_preproc_inputs(strat, anat_preproc, strat_name, + strat_nodes_list_list, workflow): + """ + Parameters + ---------- + strat : Strategy + the strategy object you want to fork + anat_preproc : Workflow + the anat_preproc workflow node to be connected and added to the resource pool + strat_name : str + name of the strategy + strat_nodes_list_list : list + a list of strat_nodes_list + workflow : Workflow + main longitudinal workflow + + Returns + ------- + new_strat : Strategy + the fork of strat with the resource pool updated + strat_nodes_list_list : list + a list of strat_nodes_list + """ + + new_strat = strat.fork() + + tmp_node, out_key = new_strat['anatomical'] + workflow.connect(tmp_node, out_key, anat_preproc, 'inputspec.anat') + + tmp_node, out_key = new_strat['template_cmass'] + workflow.connect(tmp_node, out_key, anat_preproc, + 'inputspec.template_cmass') + + new_strat.append_name(anat_preproc.name) + + new_strat.update_resource_pool({ + 'anatomical_brain': ( + anat_preproc, 'outputspec.brain'), + 'anatomical_skull_leaf': ( + anat_preproc, 'outputspec.reorient'), + 'anatomical_brain_mask': ( + anat_preproc, 'outputspec.brain_mask'), + }) + + try: + strat_nodes_list_list[strat_name].append(new_strat) + except KeyError: + strat_nodes_list_list[strat_name] = [new_strat] + + return new_strat, strat_nodes_list_list + + +def pick_map(file_list, index, file_type): + if isinstance(file_list, list): + if len(file_list) == 1: + file_list = file_list[0] + for file_name in file_list: + if file_name.endswith(f"{file_type}_{index}.nii.gz"): + return file_name + return None + + +def select_session(session, output_brains, warps): + brain_path = None + warp_path = None + for brain_path in output_brains: + if f'{session}_' in brain_path: + break + for warp_path in warps: + if f'{session}_' in warp_path: + break + return (brain_path, warp_path) + + +@nodeblock( + name="mask_longitudinal_T1w_brain", + config=["longitudinal_template_generation"], + switch=["run"], + inputs=["space-longitudinal_desc-brain_T1w"], + outputs=["space-longitudinal_desc-brain_mask"], +) +def mask_longitudinal_T1w_brain(wf, cfg, strat_pool, pipe_num, opt=None): + + brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), + name=f'longitudinal_T1w_brain_mask_{pipe_num}') + brain_mask.inputs.args = '-bin' + + node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") + wf.connect(node, out, brain_mask, 'in_file') + + outputs = { + 'space-longitudinal_desc-brain_mask': (brain_mask, 'out_file') + } + + return (wf, outputs) + + +@nodeblock( + name="warp_longitudinal_T1w_to_template", + config=["longitudinal_template_generation"], + switch=["run"], + inputs=[ + ( + "space-longitudinal_desc-brain_T1w", + "from-longitudinal_to-template_mode-image_xfm", + ) + ], + outputs=["space-template_desc-brain_T1w"], +) +def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, + opt=None): + + xfm_prov = strat_pool.get_cpac_provenance( + 'from-longitudinal_to-template_mode-image_xfm') + reg_tool = check_prov_for_regtool(xfm_prov) + + num_cpus = cfg.pipeline_setup['system_config'][ + 'max_cores_per_participant'] + + num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + + apply_xfm = apply_transform(f'warp_longitudinal_to_T1template_{pipe_num}', + reg_tool, time_series=False, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores) + + if reg_tool == 'ants': + apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ + 'anatomical_registration']['registration']['ANTs'][ + 'interpolation'] + elif reg_tool == 'fsl': + apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ + 'anatomical_registration']['registration']['FSL-FNIRT'][ + 'interpolation'] + + node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") + wf.connect(node, out, apply_xfm, 'inputspec.input_image') + + node, out = strat_pool.get_data("T1w_brain_template") + wf.connect(node, out, apply_xfm, 'inputspec.reference') + + node, out = \ + strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") + wf.connect(node, out, apply_xfm, 'inputspec.transform') + + outputs = { + 'space-template_desc-brain_T1w': + (apply_xfm, 'outputspec.output_image') + } + + return (wf, outputs) + + +@nodeblock( + name="warp_longitudinal_seg_to_T1w", + config=["longitudinal_template_generation"], + switch=["run"], + inputs=[ + ( + "from-longitudinal_to-T1w_mode-image_desc-linear_xfm", + "space-longitudinal_label-CSF_mask", + "space-longitudinal_label-GM_mask", + "space-longitudinal_label-WM_mask", + "space-longitudinal_label-CSF_desc-preproc_mask", + "space-longitudinal_label-GM_desc-preproc_mask", + "space-longitudinal_label-WM_desc-preproc_mask", + "space-longitudinal_label-CSF_probseg", + "space-longitudinal_label-GM_probseg", + "space-longitudinal_label-WM_probseg", + ) + ], + outputs=[ + "label-CSF_mask", + "label-GM_mask", + "label-WM_mask", + "label-CSF_desc-preproc_mask", + "label-GM_desc-preproc_mask", + "label-WM_desc-preproc_mask", + "label-CSF_probseg", + "label-GM_probseg", + "label-WM_probseg", + ], +) +def warp_longitudinal_seg_to_T1w(wf, cfg, strat_pool, pipe_num, opt=None): + + xfm_prov = strat_pool.get_cpac_provenance( + 'from-longitudinal_to-T1w_mode-image_desc-linear_xfm') + reg_tool = check_prov_for_regtool(xfm_prov) + + num_cpus = cfg.pipeline_setup['system_config'][ + 'max_cores_per_participant'] + + num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + + outputs = {} + + labels = [ + 'CSF_mask', 'CSF_desc-preproc_mask', 'CSF_probseg', + 'GM_mask', 'GM_desc-preproc_mask', 'GM_probseg', + 'WM_mask', 'WM_desc-preproc_mask', 'WM_probseg', + ] + + for label in labels: + apply_xfm = apply_transform(f'warp_longitudinal_seg_to_T1w_{label}_' + f'{pipe_num}', reg_tool, + time_series=False, num_cpus=num_cpus, + num_ants_cores=num_ants_cores) + + if reg_tool == 'ants': + apply_xfm.inputs.inputspec.interpolation = \ + cfg.registration_workflows['anatomical_registration'][ + 'registration']['ANTs']['interpolation'] + elif reg_tool == 'fsl': + apply_xfm.inputs.inputspec.interpolation = \ + cfg.registration_workflows['anatomical_registration'][ + 'registration']['FSL-FNIRT']['interpolation'] + + node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") + wf.connect(node, out, apply_xfm, 'inputspec.input_image') + + node, out = strat_pool.get_data("T1w_brain_template") + wf.connect(node, out, apply_xfm, 'inputspec.reference') + + node, out = \ + strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") + wf.connect(node, out, apply_xfm, 'inputspec.transform') + + outputs[f'label-{label}'] = (apply_xfm, 'outputspec.output_image') + + return (wf, outputs) + + +def anat_longitudinal_wf(subject_id, sub_list, config): + """ + Parameters + ---------- + subject_id : str + the id of the subject + sub_list : list of dict + this is a list of sessions for one subject and each session if the same dictionary as the one given to + prep_workflow + config : configuration + a configuration object containing the information of the pipeline config. (Same as for prep_workflow) + + Returns + ------- + None + """ + config["subject_id"] = subject_id + # list of lists for every strategy + session_id_list = [] + session_wfs = {} + + cpac_dirs = [] + out_dir = config.pipeline_setup['output_directory']['path'] + + orig_pipe_name = config.pipeline_setup['pipeline_name'] + + # Loop over the sessions to create the input for the longitudinal + # algorithm + for session in sub_list: + + unique_id = session['unique_id'] + session_id_list.append(unique_id) + + try: + creds_path = session['creds_path'] + if creds_path and 'none' not in creds_path.lower(): + if os.path.exists(creds_path): + input_creds_path = os.path.abspath(creds_path) + else: + err_msg = 'Credentials path: "%s" for subject "%s" ' \ + 'session "%s" was not found. Check this path ' \ + 'and try again.' % (creds_path, subject_id, + unique_id) + raise Exception(err_msg) + else: + input_creds_path = None + except KeyError: + input_creds_path = None + + workflow = initialize_nipype_wf(config, sub_list[0], + # just grab the first one for the name + name="anat_longitudinal_pre-preproc") + + workflow, rpool = initiate_rpool(workflow, config, session) + pipeline_blocks = build_anat_preproc_stack(rpool, config) + workflow = connect_pipeline(workflow, config, rpool, pipeline_blocks) + + session_wfs[unique_id] = rpool + + rpool.gather_pipes(workflow, config) + + workflow.run() + + cpac_dir = os.path.join(out_dir, f'pipeline_{orig_pipe_name}', + f'{subject_id}_{unique_id}') + cpac_dirs.append(os.path.join(cpac_dir, 'anat')) + + # Now we have all the anat_preproc set up for every session + # loop over the different anat preproc strategies + strats_brain_dct = {} + strats_head_dct = {} + for cpac_dir in cpac_dirs: + if os.path.isdir(cpac_dir): + for filename in os.listdir(cpac_dir): + if 'T1w.nii' in filename: + for tag in filename.split('_'): + if 'desc-' in tag and 'brain' in tag: + if tag not in strats_brain_dct: + strats_brain_dct[tag] = [] + strats_brain_dct[tag].append(os.path.join(cpac_dir, + filename)) + if tag not in strats_head_dct: + strats_head_dct[tag] = [] + head_file = filename.replace(tag, 'desc-reorient') + strats_head_dct[tag].append(os.path.join(cpac_dir, + head_file)) + + longitudinal_rpool: Optional[ResourcePool] = None + + for strat in strats_brain_dct.keys(): + + wf = initialize_nipype_wf(config, sub_list[0], + # just grab the first one for the name + name=f"template_node_{strat}") + + config.pipeline_setup[ + 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' + + template_node_name = f'longitudinal_anat_template_{strat}' + + # This node will generate the longitudinal template (the functions are + # in longitudinal_preproc) + # Later other algorithms could be added to calculate it, like the + # multivariate template from ANTS + # It would just require to change it here. + template_node = subject_specific_template( + workflow_name=template_node_name + ) + + template_node.inputs.set( + avg_method=config.longitudinal_template_generation[ + 'average_method'], + dof=config.longitudinal_template_generation['dof'], + interp=config.longitudinal_template_generation['interp'], + cost=config.longitudinal_template_generation['cost'], + convergence_threshold=config.longitudinal_template_generation[ + 'convergence_threshold'], + thread_pool=config.longitudinal_template_generation[ + 'thread_pool'], + unique_id_list=list(session_wfs.keys()) + ) + + template_node.inputs.input_brain_list = strats_brain_dct[strat] + template_node.inputs.input_skull_list = strats_head_dct[strat] + + long_id = f'longitudinal_{subject_id}_strat-{strat}' + + wf, rpool = initiate_rpool(wf, config, part_id=long_id) + + rpool.set_data("space-longitudinal_desc-brain_T1w", + template_node, 'brain_template', {}, + "", template_node_name) + + rpool.set_data("space-longitudinal_desc-brain_T1w-template", + template_node, 'brain_template', {}, + "", template_node_name) + + rpool.set_data("space-longitudinal_desc-reorient_T1w", + template_node, 'skull_template', {}, + "", template_node_name) + + rpool.set_data("space-longitudinal_desc-reorient_T1w-template", + template_node, 'skull_template', {}, + "", template_node_name) + + pipeline_blocks = [mask_longitudinal_T1w_brain] + + pipeline_blocks = build_T1w_registration_stack(rpool, config, + pipeline_blocks) + + pipeline_blocks = build_segmentation_stack(rpool, config, + pipeline_blocks) + + wf = connect_pipeline(wf, config, rpool, pipeline_blocks) + + excl = ['space-longitudinal_desc-brain_T1w', + 'space-longitudinal_desc-reorient_T1w', + 'space-longitudinal_desc-brain_mask'] + rpool.gather_pipes(wf, config, add_excl=excl) + + # this is going to run multiple times! + # once for every strategy! + wf.run() + + longitudinal_rpool = rpool.copy_rpool() + + # now, just write out a copy of the above to each session + config.pipeline_setup['pipeline_name'] = orig_pipe_name + for session in sub_list: + + unique_id = session['unique_id'] + + try: + creds_path = session['creds_path'] + if creds_path and 'none' not in creds_path.lower(): + if os.path.exists(creds_path): + input_creds_path = os.path.abspath(creds_path) + else: + err_msg = 'Credentials path: "%s" for subject "%s" ' \ + 'session "%s" was not found. Check this path ' \ + 'and try again.' % (creds_path, subject_id, + unique_id) + raise Exception(err_msg) + else: + input_creds_path = None + except KeyError: + input_creds_path = None + + wf = initialize_nipype_wf(config, sub_list[0]) + + wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) + + config.pipeline_setup[ + 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' + rpool = ingress_output_dir(config, rpool, long_id, + creds_path=input_creds_path) + + select_node_name = f'select_{unique_id}' + select_sess = pe.Node(Function(input_names=['session', + 'output_brains', + 'warps'], + output_names=['brain_path', + 'warp_path'], + function=select_session), + name=select_node_name) + select_sess.inputs.session = unique_id + + wf.connect(template_node, 'output_brain_list', select_sess, + 'output_brains') + wf.connect(template_node, 'warp_list', select_sess, 'warps') + + rpool.set_data("space-longitudinal_desc-brain_T1w", + select_sess, 'brain_path', {}, "", + select_node_name) + + rpool.set_data("from-T1w_to-longitudinal_mode-image_" + "desc-linear_xfm", + select_sess, 'warp_path', {}, "", + select_node_name) + + config.pipeline_setup['pipeline_name'] = orig_pipe_name + excl = ['space-template_desc-brain_T1w', + 'space-T1w_desc-brain_mask'] + + rpool.gather_pipes(wf, config, add_excl=excl) + wf.run() + longitudinal_rpool = rpool.copy_rpool() + + # begin single-session stuff again + for session in sub_list: + + unique_id = session['unique_id'] + + try: + creds_path = session['creds_path'] + if creds_path and 'none' not in creds_path.lower(): + if os.path.exists(creds_path): + input_creds_path = os.path.abspath(creds_path) + else: + err_msg = 'Credentials path: "%s" for subject "%s" ' \ + 'session "%s" was not found. Check this path ' \ + 'and try again.' % (creds_path, subject_id, + unique_id) + raise Exception(err_msg) + else: + input_creds_path = None + except KeyError: + input_creds_path = None + + wf = initialize_nipype_wf(config, sub_list[0]) + + wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) + + pipeline_blocks = [warp_longitudinal_T1w_to_template, + warp_longitudinal_seg_to_T1w] + + wf = connect_pipeline(wf, config, rpool, pipeline_blocks) + + rpool.gather_pipes(wf, config) + + # this is going to run multiple times! + # once for every strategy! + wf.run() + + + + +# TODO check: +# 1 func alone works +# 2 anat + func works, pass anat strategy list? +def func_preproc_longitudinal_wf(subject_id, sub_list, config): + """ + Parameters + ---------- + subject_id : string + the id of the subject + sub_list : list of dict + this is a list of sessions for one subject and each session if the same dictionary as the one given to + prep_workflow + config : configuration + a configuration object containing the information of the pipeline config. (Same as for prep_workflow) + + Returns + ------- + strat_list_ses_list : list of list + a list of strategies; within each strategy, a list of sessions + """ + + datasink = pe.Node(nio.DataSink(), name='sinker') + datasink.inputs.base_directory = \ + config.pipeline_setup['working_directory']['path'] + + session_id_list = [] + ses_list_strat_list = {} + + workflow_name = 'func_preproc_longitudinal_' + str(subject_id) + workflow = pe.Workflow(name=workflow_name) + workflow.base_dir = config.pipeline_setup['working_directory']['path'] + workflow.config['execution'] = { + 'hash_method': 'timestamp', + 'crashdump_dir': os.path.abspath( + config.pipeline_setup['crash_directory']['path']) + } + + for sub_dict in sub_list: + if 'func' in sub_dict or 'rest' in sub_dict: + if 'func' in sub_dict: + func_paths_dict = sub_dict['func'] + else: + func_paths_dict = sub_dict['rest'] + + unique_id = sub_dict['unique_id'] + session_id_list.append(unique_id) + + try: + creds_path = sub_dict['creds_path'] + if creds_path and 'none' not in creds_path.lower(): + if os.path.exists(creds_path): + input_creds_path = os.path.abspath(creds_path) + else: + err_msg = 'Credentials path: "%s" for subject "%s" was not ' \ + 'found. Check this path and try again.' % ( + creds_path, subject_id) + raise Exception(err_msg) + else: + input_creds_path = None + except KeyError: + input_creds_path = None + + strat = Strategy() + strat_list = [strat] + node_suffix = '_'.join([subject_id, unique_id]) + + # Functional Ingress Workflow + # add optional flag + workflow, diff, blip, fmap_rp_list = connect_func_ingress( + workflow, + strat_list, + config, + sub_dict, + subject_id, + input_creds_path, + node_suffix) + + # Functional Initial Prep Workflow + workflow, strat_list = connect_func_init(workflow, strat_list, + config, node_suffix) + + # Functional Image Preprocessing Workflow + workflow, strat_list = connect_func_preproc(workflow, strat_list, + config, node_suffix) + + # Distortion Correction + workflow, strat_list = connect_distortion_correction(workflow, + strat_list, + config, + diff, + blip, + fmap_rp_list, + node_suffix) + + ses_list_strat_list[node_suffix] = strat_list + + # Here we have all the func_preproc set up for every session of the subject + + # TODO create a list of list ses_list_strat_list + # a list of skullstripping strategies, + # a list of sessions within each strategy list + # TODO rename and reorganize dict + # TODO update strat name + strat_list_ses_list = {} + strat_list_ses_list['func_default'] = [] + + for sub_ses_id, strat_nodes_list in ses_list_strat_list.items(): + strat_list_ses_list['func_default'].append(strat_nodes_list[0]) + + workflow.run() + + return strat_list_ses_list + + +def merge_func_preproc(working_directory): + """ + Parameters + ---------- + working_directory : string + a path to the working directory + + Returns + ------- + brain_list : list + a list of func preprocessed brain + skull_list : list + a list of func preprocessed skull + """ + + brain_list = [] + skull_list = [] + + for dirpath, dirnames, filenames in os.walk(working_directory): + for f in filenames: + if 'func_get_preprocessed_median' in dirpath and '.nii.gz' in f: + filepath = os.path.join(dirpath, f) + brain_list.append(filepath) + if 'func_get_motion_correct_median' in dirpath and '.nii.gz' in f: + filepath = os.path.join(dirpath, f) + skull_list.append(filepath) + + brain_list.sort() + skull_list.sort() + + return brain_list, skull_list + + +def register_func_longitudinal_template_to_standard( + longitudinal_template_node, c, workflow, strat_init, strat_name): + sub_mem_gb, num_cores_per_sub, num_ants_cores, num_omp_cores = \ + check_config_resources(c) + + strat_init_new = strat_init.fork() + + strat_init_new.update_resource_pool({ + 'functional_preprocessed_median': ( + longitudinal_template_node, 'brain_template'), + 'motion_correct_median': ( + longitudinal_template_node, 'skull_template') + }) + + strat_list = [strat_init_new] + + new_strat_list = [] + + regOption = c.anatomical_preproc[ + 'registration_workflow' + ]['registration']['using'] + + if 'FSL' in regOption: + + for num_strat, strat in enumerate(strat_list): + + flirt_reg_func_mni = create_fsl_flirt_linear_reg( + 'func_mni_flirt_register_%s_%d' % (strat_name, num_strat) + ) + + if c.functional_registration['2-func_registration_to_template'][ + 'FNIRT_pipelines']['interpolation'] not in ["trilinear", + "sinc", "spline"]: + err_msg = 'The selected FSL interpolation method may be in the list of values: "trilinear", "sinc", "spline"' + raise Exception(err_msg) + + # Input registration parameters + flirt_reg_func_mni.inputs.inputspec.interp = \ + c.functional_registration['2-func_registration_to_template'][ + 'FNIRT_pipelines']['interpolation'] + + node, out_file = strat['functional_preprocessed_median'] + workflow.connect(node, out_file, + flirt_reg_func_mni, 'inputspec.input_brain') + + # pass the reference files + node, out_file = strat['template_brain_for_func_preproc'] + workflow.connect(node, out_file, flirt_reg_func_mni, + 'inputspec.reference_brain') + + if 'ANTS' in regOption: + strat = strat.fork() + new_strat_list.append(strat) + + strat.append_name(flirt_reg_func_mni.name) + + strat.update_resource_pool({ + 'registration_method': 'FSL', + 'func_longitudinal_to_mni_linear_xfm': ( + flirt_reg_func_mni, 'outputspec.linear_xfm'), + 'mni_to_func_longitudinal_linear_xfm': ( + flirt_reg_func_mni, 'outputspec.invlinear_xfm'), + 'func_longitudinal_template_to_standard': ( + flirt_reg_func_mni, 'outputspec.output_brain') + }) + + strat_list += new_strat_list + + new_strat_list = [] + + try: + fsl_linear_reg_only = c.fsl_linear_reg_only + except AttributeError: + fsl_linear_reg_only = [0] + + if 'FSL' in regOption and 0 in fsl_linear_reg_only: + + for num_strat, strat in enumerate(strat_list): + + if strat.get('registration_method') == 'FSL': + + fnirt_reg_func_mni = create_fsl_fnirt_nonlinear_reg( + 'func_mni_fnirt_register_%s_%d' % (strat_name, num_strat) + ) + + # brain input + node, out_file = strat['functional_preprocessed_median'] + workflow.connect(node, out_file, + fnirt_reg_func_mni, 'inputspec.input_brain') + + # brain reference + node, out_file = strat['template_brain_for_func_preproc'] + workflow.connect(node, out_file, + fnirt_reg_func_mni, + 'inputspec.reference_brain') + + # skull input + node, out_file = strat['motion_correct_median'] + workflow.connect(node, out_file, + fnirt_reg_func_mni, 'inputspec.input_skull') + + # skull reference + node, out_file = strat['template_skull_for_func_preproc'] + workflow.connect(node, out_file, + fnirt_reg_func_mni, + 'inputspec.reference_skull') + + node, out_file = strat['func_longitudinal_to_mni_linear_xfm'] + workflow.connect(node, out_file, + fnirt_reg_func_mni, 'inputspec.linear_aff') + + node, out_file = strat['template_ref_mask'] + workflow.connect(node, out_file, + fnirt_reg_func_mni, 'inputspec.ref_mask') + + # assign the FSL FNIRT config file specified in pipeline + # config.yml + fnirt_reg_func_mni.inputs.inputspec.fnirt_config = \ + c.anatomical_preproc['registration_workflow']['registration'][ + 'FSL-FNIRT']['fnirt_config'] + + if 1 in fsl_linear_reg_only: + strat = strat.fork() + new_strat_list.append(strat) + + strat.append_name(fnirt_reg_func_mni.name) + + strat.update_resource_pool({ + 'func_longitudinal_to_mni_nonlinear_xfm': ( + fnirt_reg_func_mni, 'outputspec.nonlinear_xfm'), + 'func_longitudinal_template_to_standard': ( + fnirt_reg_func_mni, 'outputspec.output_brain') + }, override=True) + + strat_list += new_strat_list + + new_strat_list = [] + + for num_strat, strat in enumerate(strat_list): + + # or run ANTS anatomical-to-MNI registration instead + if 'ANTS' in regOption and \ + strat.get('registration_method') != 'FSL': + + ants_reg_func_mni = \ + create_wf_calculate_ants_warp( + 'func_mni_ants_register_%s_%d' % (strat_name, num_strat), + num_threads=num_ants_cores, + reg_ants_skull= + c.anatomical_preproc['registration_workflow'][ + 'reg_with_skull'] + ) + + if c.functional_registration['2-func_registration_to_template'][ + 'ANTs_pipelines']['interpolation'] not in ['Linear', + 'BSpline', + 'LanczosWindowedSinc']: + err_msg = 'The selected ANTS interpolation method may be in the list of values: "Linear", "BSpline", "LanczosWindowedSinc"' + raise Exception(err_msg) + + # Input registration parameters + ants_reg_func_mni.inputs.inputspec.interp = \ + c.functional_registration['2-func_registration_to_template'][ + 'ANTs_pipelines']['interpolation'] + + # calculating the transform with the skullstripped is + # reported to be better, but it requires very high + # quality skullstripping. If skullstripping is imprecise + # registration with skull is preferred + if c.anatomical_preproc['registration_workflow'][ + 'reg_with_skull']: + + # get the skull-stripped anatomical from resource pool + node, out_file = strat['functional_preprocessed_median'] + + # pass the anatomical to the workflow + workflow.connect(node, out_file, + ants_reg_func_mni, 'inputspec.moving_brain') + + # get the reorient skull-on anatomical from resource pool + node, out_file = strat['motion_correct_median'] + + # pass the anatomical to the workflow + workflow.connect(node, out_file, + ants_reg_func_mni, 'inputspec.moving_skull') + + # pass the reference file + node, out_file = strat['template_brain_for_func_preproc'] + workflow.connect(node, out_file, + ants_reg_func_mni, + 'inputspec.reference_brain') + + # pass the reference file + node, out_file = strat['template_skull_for_func_preproc'] + workflow.connect(node, out_file, + ants_reg_func_mni, + 'inputspec.reference_skull') + + else: + + node, out_file = strat['functional_preprocessed_median'] + + workflow.connect(node, out_file, + ants_reg_func_mni, 'inputspec.moving_brain') + + # pass the reference file + node, out_file = strat['template_brain_for_func_preproc'] + workflow.connect(node, out_file, + ants_reg_func_mni, + 'inputspec.reference_brain') + + # pass the reference mask file + node, out_file = strat['template_brain_mask_for_func_preproc'] + workflow.connect( + node, out_file, + ants_reg_func_mni, 'inputspec.reference_mask' + ) + + # pass the reference mask file + node, out_file = strat['functional_brain_mask'] + workflow.connect( + node, out_file, + ants_reg_func_mni, 'inputspec.moving_mask' + ) + + ants_reg_func_mni.inputs.inputspec.ants_para = \ + c.anatomical_preproc['registration_workflow']['registration'][ + 'ANTs']['T1_registration'] + ants_reg_func_mni.inputs.inputspec.fixed_image_mask = None + + strat.append_name(ants_reg_func_mni.name) + + strat.update_resource_pool({ + 'registration_method': 'ANTS', + 'ants_initial_xfm': ( + ants_reg_func_mni, 'outputspec.ants_initial_xfm'), + 'ants_rigid_xfm': ( + ants_reg_func_mni, 'outputspec.ants_rigid_xfm'), + 'ants_affine_xfm': ( + ants_reg_func_mni, 'outputspec.ants_affine_xfm'), + 'func_longitudinal_to_mni_nonlinear_xfm': ( + ants_reg_func_mni, 'outputspec.warp_field'), + 'mni_to_func_longitudinal_nonlinear_xfm': ( + ants_reg_func_mni, 'outputspec.inverse_warp_field'), + 'func_longitudinal_to_mni_ants_composite_xfm': ( + ants_reg_func_mni, 'outputspec.composite_transform'), + 'func_longitudinal_template_to_standard': ( + ants_reg_func_mni, 'outputspec.normalized_output_brain') + }) + + strat_list += new_strat_list + + ''' + # Func -> T1 Registration (Initial Linear Reg) + workflow, strat_list, diff_complete = connect_func_to_anat_init_reg(workflow, strat_list, c) + + # Func -> T1 Registration (BBREG) + workflow, strat_list = connect_func_to_anat_bbreg(workflow, strat_list, c, diff_complete) + + # Func -> T1/EPI Template + workflow, strat_list = connect_func_to_template_reg(workflow, strat_list, c) + ''' + + return workflow, strat_list + + +def func_longitudinal_template_wf(subject_id, strat_list, config): + ''' + Parameters + ---------- + subject_id : string + the id of the subject + strat_list : list of list + first level strategy, second level session + config : configuration + a configuration object containing the information of the pipeline config. + + Returns + ------- + None + ''' + + workflow_name = 'func_longitudinal_template_' + str(subject_id) + workflow = pe.Workflow(name=workflow_name) + workflow.base_dir = config.pipeline_setup['working_directory']['path'] + workflow.config['execution'] = { + 'hash_method': 'timestamp', + 'crashdump_dir': os.path.abspath( + config.pipeline_setup['crash_directory']['path']) + } + + # strat_nodes_list = strat_list['func_default'] + strat_init = Strategy() + + templates_for_resampling = [ + (config.resolution_for_func_preproc, + config.template_brain_only_for_func, + 'template_brain_for_func_preproc', 'resolution_for_func_preproc'), + (config.resolution_for_func_preproc, config.template_skull_for_func, + 'template_skull_for_func_preproc', 'resolution_for_func_preproc'), + (config.resolution_for_func_preproc, config.ref_mask_for_func, + 'template_ref_mask', 'resolution_for_func_preproc'), + # TODO check float resolution + (config.resolution_for_func_preproc, + config.functional_registration['2-func_registration_to_template'][ + 'target_template']['EPI_template']['template_epi'], + 'template_epi', 'resolution_for_func_preproc'), + (config.resolution_for_func_derivative, + config.functional_registration['2-func_registration_to_template'][ + 'target_template']['EPI_template']['template_epi'], + 'template_epi_derivative', 'resolution_for_func_derivative'), + (config.resolution_for_func_derivative, + config.template_brain_only_for_func, + 'template_brain_for_func_derivative', 'resolution_for_func_preproc'), + ( + config.resolution_for_func_derivative, config.template_skull_for_func, + 'template_skull_for_func_derivative', 'resolution_for_func_preproc'), + ] + + for resolution, template, template_name, tag in templates_for_resampling: + resampled_template = pe.Node(Function( + input_names=['resolution', 'template', 'template_name', 'tag'], + output_names=['resampled_template'], + function=resolve_resolution, + as_module=True), + name='resampled_' + template_name) + + resampled_template.inputs.resolution = resolution + resampled_template.inputs.template = template + resampled_template.inputs.template_name = template_name + resampled_template.inputs.tag = tag + + strat_init.update_resource_pool({ + template_name: (resampled_template, 'resampled_template') + }) + + merge_func_preproc_node = pe.Node( + Function(input_names=['working_directory'], + output_names=['brain_list', 'skull_list'], + function=merge_func_preproc, + as_module=True), + name='merge_func_preproc') + + merge_func_preproc_node.inputs.working_directory = \ + config.pipeline_setup['working_directory']['path'] + + template_node = subject_specific_template( + workflow_name='subject_specific_func_template_' + subject_id + ) + + template_node.inputs.set( + avg_method=config.longitudinal_template_average_method, + dof=config.longitudinal_template_dof, + interp=config.longitudinal_template_interp, + cost=config.longitudinal_template_cost, + convergence_threshold=config.longitudinal_template_convergence_threshold, + thread_pool=config.longitudinal_template_thread_pool, + ) + + workflow.connect(merge_func_preproc_node, 'brain_list', + template_node, 'input_brain_list') + + workflow.connect(merge_func_preproc_node, 'skull_list', + template_node, 'input_skull_list') + + workflow, strat_list = register_func_longitudinal_template_to_standard( + template_node, + config, + workflow, + strat_init, + 'default' + ) + + workflow.run() + + return diff --git a/CPAC/pipeline/engine.py b/CPAC/pipeline/engine.py index 6488713023..0715b623be 100644 --- a/CPAC/pipeline/engine.py +++ b/CPAC/pipeline/engine.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2023 C-PAC Developers +# Copyright (C) 2021-2024 C-PAC Developers # This file is part of C-PAC. @@ -2335,7 +2335,9 @@ def _set_nested(attr, keys): return rpool -def initiate_rpool(wf, cfg, data_paths=None, part_id=None): +def initiate_rpool( + wf, cfg, data_paths=None, part_id=None, *, rpool: Optional[ResourcePool] = None +): ''' data_paths format: @@ -2374,7 +2376,7 @@ def initiate_rpool(wf, cfg, data_paths=None, part_id=None): unique_id = part_id creds_path = None - rpool = ResourcePool(name=unique_id, cfg=cfg) + rpool = ResourcePool(rpool=rpool.rpool if rpool else None, name=unique_id, cfg=cfg) if data_paths: # ingress outdir diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml b/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml index 457e246934..d3c4f32731 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ABIDE.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml index 6ccf800c25..133a7ca488 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml index 0a756d99f4..9d64140664 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-ADHD200_only2.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml b/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml index 820eeeef3e..3a0aec44ba 100644 --- a/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml +++ b/CPAC/resources/configs/data_config_S3-BIDS-NKI-RocklandSample.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_config_cpac_benchmark.yml b/CPAC/resources/configs/data_config_cpac_benchmark.yml index a3491d702b..f487a259db 100644 --- a/CPAC/resources/configs/data_config_cpac_benchmark.yml +++ b/CPAC/resources/configs/data_config_cpac_benchmark.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/data_settings_template.yml b/CPAC/resources/configs/data_settings_template.yml index 70414856f3..3378ac6c7c 100644 --- a/CPAC/resources/configs/data_settings_template.yml +++ b/CPAC/resources/configs/data_settings_template.yml @@ -1,5 +1,5 @@ # CPAC Data Settings File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/group_config_template.yml b/CPAC/resources/configs/group_config_template.yml index 0a3ff76984..da828118df 100644 --- a/CPAC/resources/configs/group_config_template.yml +++ b/CPAC/resources/configs/group_config_template.yml @@ -1,5 +1,5 @@ # CPAC Group-Level Analysis Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_abcd-options.yml b/CPAC/resources/configs/pipeline_config_abcd-options.yml index 1ba775612f..187e7be282 100644 --- a/CPAC/resources/configs/pipeline_config_abcd-options.yml +++ b/CPAC/resources/configs/pipeline_config_abcd-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_abcd-prep.yml b/CPAC/resources/configs/pipeline_config_abcd-prep.yml index d664397fd6..18f9fab116 100644 --- a/CPAC/resources/configs/pipeline_config_abcd-prep.yml +++ b/CPAC/resources/configs/pipeline_config_abcd-prep.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_anat-only.yml b/CPAC/resources/configs/pipeline_config_anat-only.yml index a935cc330b..37a80923d9 100644 --- a/CPAC/resources/configs/pipeline_config_anat-only.yml +++ b/CPAC/resources/configs/pipeline_config_anat-only.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml b/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml index d6a2f084ae..aa9f757f3c 100644 --- a/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml +++ b/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml b/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml index 1af7eb337d..797f2909e1 100644 --- a/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml +++ b/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_blank.yml b/CPAC/resources/configs/pipeline_config_blank.yml index fc56a42fb4..c8721ab893 100644 --- a/CPAC/resources/configs/pipeline_config_blank.yml +++ b/CPAC/resources/configs/pipeline_config_blank.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_ccs-options.yml b/CPAC/resources/configs/pipeline_config_ccs-options.yml index 351ec8a18c..037a5cdd41 100644 --- a/CPAC/resources/configs/pipeline_config_ccs-options.yml +++ b/CPAC/resources/configs/pipeline_config_ccs-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_default-deprecated.yml b/CPAC/resources/configs/pipeline_config_default-deprecated.yml index f29ad21ca3..3f1d4cb814 100644 --- a/CPAC/resources/configs/pipeline_config_default-deprecated.yml +++ b/CPAC/resources/configs/pipeline_config_default-deprecated.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_default.yml b/CPAC/resources/configs/pipeline_config_default.yml index 8dc38545f3..202d452807 100644 --- a/CPAC/resources/configs/pipeline_config_default.yml +++ b/CPAC/resources/configs/pipeline_config_default.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml b/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml index b1219d67ff..bda6bfdf9e 100644 --- a/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml +++ b/CPAC/resources/configs/pipeline_config_fmriprep-ingress.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fmriprep-options.yml b/CPAC/resources/configs/pipeline_config_fmriprep-options.yml index 9f62c7dafa..c52402fe7d 100644 --- a/CPAC/resources/configs/pipeline_config_fmriprep-options.yml +++ b/CPAC/resources/configs/pipeline_config_fmriprep-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_fx-options.yml b/CPAC/resources/configs/pipeline_config_fx-options.yml index 32cb82325e..b88215e423 100644 --- a/CPAC/resources/configs/pipeline_config_fx-options.yml +++ b/CPAC/resources/configs/pipeline_config_fx-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml b/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml index 82d19e51f1..90a9d34209 100644 --- a/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml +++ b/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_monkey.yml b/CPAC/resources/configs/pipeline_config_monkey.yml index ee5961b720..1bb83e3a4a 100644 --- a/CPAC/resources/configs/pipeline_config_monkey.yml +++ b/CPAC/resources/configs/pipeline_config_monkey.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_ndmg.yml b/CPAC/resources/configs/pipeline_config_ndmg.yml index 0174c3e84d..1dedcc640c 100644 --- a/CPAC/resources/configs/pipeline_config_ndmg.yml +++ b/CPAC/resources/configs/pipeline_config_ndmg.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_nhp-macaque.yml b/CPAC/resources/configs/pipeline_config_nhp-macaque.yml index c6052fc859..24df5e033b 100644 --- a/CPAC/resources/configs/pipeline_config_nhp-macaque.yml +++ b/CPAC/resources/configs/pipeline_config_nhp-macaque.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_preproc.yml b/CPAC/resources/configs/pipeline_config_preproc.yml index 0b9eafe8c2..ee146d07c2 100644 --- a/CPAC/resources/configs/pipeline_config_preproc.yml +++ b/CPAC/resources/configs/pipeline_config_preproc.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_rbc-options.yml b/CPAC/resources/configs/pipeline_config_rbc-options.yml index c6432797d7..2e10e6d2e0 100644 --- a/CPAC/resources/configs/pipeline_config_rbc-options.yml +++ b/CPAC/resources/configs/pipeline_config_rbc-options.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-1.yml b/CPAC/resources/configs/pipeline_config_regtest-1.yml index 9ef2b7b825..bc3e4d3897 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-1.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-1.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-2.yml b/CPAC/resources/configs/pipeline_config_regtest-2.yml index 8561e8fa17..683ac46bc1 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-2.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-2.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-3.yml b/CPAC/resources/configs/pipeline_config_regtest-3.yml index d136e154a6..409ac8fbfb 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-3.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-3.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_regtest-4.yml b/CPAC/resources/configs/pipeline_config_regtest-4.yml index 6fdf687c5c..995f801adc 100644 --- a/CPAC/resources/configs/pipeline_config_regtest-4.yml +++ b/CPAC/resources/configs/pipeline_config_regtest-4.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/pipeline_config_rodent.yml b/CPAC/resources/configs/pipeline_config_rodent.yml index 39c3ff88f7..c11b6fded2 100644 --- a/CPAC/resources/configs/pipeline_config_rodent.yml +++ b/CPAC/resources/configs/pipeline_config_rodent.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/system_config.yml b/CPAC/resources/configs/system_config.yml index 7a00128ff5..af764667e3 100644 --- a/CPAC/resources/configs/system_config.yml +++ b/CPAC/resources/configs/system_config.yml @@ -1,5 +1,5 @@ # C-PAC System Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml index 5e8d6f9320..c5d08ffe5b 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_1.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml index 081cb29165..45f1a27a5c 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-ADHD200_no-params.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml b/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml index 345ca7455d..6a778b10f7 100644 --- a/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml +++ b/CPAC/resources/configs/test_configs/data-test_S3-NKI-RS_fmap.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml index 15c161de2c..c58457088f 100644 --- a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml +++ b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-scan.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml index 0149953399..25f78befaf 100644 --- a/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml +++ b/CPAC/resources/configs/test_configs/data_config_S3_CoRR_5only_mult-sess.yml @@ -1,5 +1,5 @@ # CPAC Data Configuration File -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml b/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml index 8a12182a40..9890639a23 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml index ed821b6bcc..ba467b15a5 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml index a21996f40c..c9f9425a25 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml index f6e5b610ad..616a978fc7 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml index 3f8481f29a..b68daccf1a 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml index 514efb3a71..416296fdde 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml index 92441194d0..2b6a15ebb5 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml index 92441194d0..2b6a15ebb5 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml index 92441194d0..2b6a15ebb5 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml index 92441194d0..2b6a15ebb5 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml index 92441194d0..2b6a15ebb5 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/CPAC/resources/configs/test_configs/pipe-test_all.yml b/CPAC/resources/configs/test_configs/pipe-test_all.yml index 43eb774958..5f03ecf00f 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_all.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_all.yml @@ -1,7 +1,7 @@ %YAML 1.1 --- # CPAC Pipeline Configuration YAML file -# Version 1.8.7.post1.dev2 +# Version 1.8.7.post1.dev3 # # http://fcp-indi.github.io for more info. # diff --git a/version b/version index b659be2ca3..3b40862c87 100644 --- a/version +++ b/version @@ -1 +1 @@ -v1.8.7.post1.dev2 +v1.8.7.post1.dev3 From 282f3f74169db6ca8ca2e6f66ee3c838871cfdb3 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 28 Oct 2024 15:31:01 -0400 Subject: [PATCH 07/33] :loud_sound: :alembic: Pickle rpool --- CPAC/pipeline/engine.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CPAC/pipeline/engine.py b/CPAC/pipeline/engine.py index 0715b623be..173c11d731 100644 --- a/CPAC/pipeline/engine.py +++ b/CPAC/pipeline/engine.py @@ -21,6 +21,7 @@ from itertools import chain import logging import os +import pickle import re from typing import Any, Optional, Union import warnings @@ -555,6 +556,9 @@ def get_strats(self, resources, debug=False): total_pool.append(sub_pool) if not total_pool: + print(self.rpool.keys()) + with open("rpool.pickle", "wb") as _f: + pickle.dump(self, _f) raise LookupError('\n\n[!] C-PAC says: None of the listed ' 'resources in the node block being connected ' 'exist in the resource pool.\n\nResources:\n' From 828d5895c9b4e5c2c4173d1e1754ea03d7bc302a Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 28 Oct 2024 22:39:55 -0400 Subject: [PATCH 08/33] :alembic: :loud_sound: Pickle lots of rpools --- .../longitudinal_workflow.py | 17 ++++++++++------- CPAC/pipeline/engine.py | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 7e11d6a7a7..7ce5d6fa58 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -467,9 +467,9 @@ def anat_longitudinal_wf(subject_id, sub_list, config): session_wfs[unique_id] = rpool rpool.gather_pipes(workflow, config) - + rpool.pickle("470") workflow.run() - + rpool.pickle("472") cpac_dir = os.path.join(out_dir, f'pipeline_{orig_pipe_name}', f'{subject_id}_{unique_id}') cpac_dirs.append(os.path.join(cpac_dir, 'anat')) @@ -494,8 +494,6 @@ def anat_longitudinal_wf(subject_id, sub_list, config): strats_head_dct[tag].append(os.path.join(cpac_dir, head_file)) - longitudinal_rpool: Optional[ResourcePool] = None - for strat in strats_brain_dct.keys(): wf = initialize_nipype_wf(config, sub_list[0], @@ -565,12 +563,13 @@ def anat_longitudinal_wf(subject_id, sub_list, config): excl = ['space-longitudinal_desc-brain_T1w', 'space-longitudinal_desc-reorient_T1w', 'space-longitudinal_desc-brain_mask'] + rpool.pickle("566") rpool.gather_pipes(wf, config, add_excl=excl) # this is going to run multiple times! # once for every strategy! wf.run() - + rpool.pickle("572") longitudinal_rpool = rpool.copy_rpool() # now, just write out a copy of the above to each session @@ -598,12 +597,13 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf = initialize_nipype_wf(config, sub_list[0]) wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) + rpool.pickle("600") config.pipeline_setup[ 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' rpool = ingress_output_dir(config, rpool, long_id, creds_path=input_creds_path) - + rpool.pickle("606") select_node_name = f'select_{unique_id}' select_sess = pe.Node(Function(input_names=['session', 'output_brains', @@ -632,8 +632,10 @@ def anat_longitudinal_wf(subject_id, sub_list, config): 'space-T1w_desc-brain_mask'] rpool.gather_pipes(wf, config, add_excl=excl) + rpool.pickle("635") wf.run() longitudinal_rpool = rpool.copy_rpool() + rpool.pickle("638") # begin single-session stuff again for session in sub_list: @@ -666,10 +668,11 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf = connect_pipeline(wf, config, rpool, pipeline_blocks) rpool.gather_pipes(wf, config) - + rpool.pickle("671") # this is going to run multiple times! # once for every strategy! wf.run() + rpool.pickle("675") diff --git a/CPAC/pipeline/engine.py b/CPAC/pipeline/engine.py index 173c11d731..42b5d46468 100644 --- a/CPAC/pipeline/engine.py +++ b/CPAC/pipeline/engine.py @@ -21,6 +21,7 @@ from itertools import chain import logging import os +from pathlib import Path import pickle import re from typing import Any, Optional, Union @@ -62,6 +63,22 @@ class ResourcePool: + def pickle(self, name: Optional[str] = None) -> None: + """Pickle rpool.""" + if name: + filename: str = f"{name}.pickle" + elif hasattr(self, "name") and isinstance(self.name, str): + filename: str = f"{self.name.replace(' ', '')}.pickle" + else: + existing_names: list[str] = [path.name[:-7] for path in Path().iterdir() if path.name.endswith(".pickle")] + i: int = 0 + while str(i) in existing_names: + i += 1 + filename = f"{i}.pickle" + with open(filename, "wb") as _f: + pickle.dump(self, _f) + + def __init__(self, rpool=None, name=None, cfg=None, pipe_list=None): if not rpool: @@ -557,7 +574,7 @@ def get_strats(self, resources, debug=False): if not total_pool: print(self.rpool.keys()) - with open("rpool.pickle", "wb") as _f: + with open("rpool_no_total_pool.pickle", "wb") as _f: pickle.dump(self, _f) raise LookupError('\n\n[!] C-PAC says: None of the listed ' 'resources in the node block being connected ' From 0cd163971b9aa4a901d2befc3d169d7ac8e2c341 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 12:33:38 -0400 Subject: [PATCH 09/33] :alien: Update path matching for longitudinal --- .../longitudinal_workflow.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 7ce5d6fa58..c77db3378c 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -471,7 +471,7 @@ def anat_longitudinal_wf(subject_id, sub_list, config): workflow.run() rpool.pickle("472") cpac_dir = os.path.join(out_dir, f'pipeline_{orig_pipe_name}', - f'{subject_id}_{unique_id}') + f'{subject_id}/{unique_id}') cpac_dirs.append(os.path.join(cpac_dir, 'anat')) # Now we have all the anat_preproc set up for every session @@ -483,16 +483,18 @@ def anat_longitudinal_wf(subject_id, sub_list, config): for filename in os.listdir(cpac_dir): if 'T1w.nii' in filename: for tag in filename.split('_'): - if 'desc-' in tag and 'brain' in tag: - if tag not in strats_brain_dct: - strats_brain_dct[tag] = [] - strats_brain_dct[tag].append(os.path.join(cpac_dir, - filename)) - if tag not in strats_head_dct: - strats_head_dct[tag] = [] - head_file = filename.replace(tag, 'desc-reorient') - strats_head_dct[tag].append(os.path.join(cpac_dir, - head_file)) + braintag: str = tag + if 'desc-' in tag and ('brain' in tag or 'preproc' in tag): + braintag = tag.replace('preproc', 'brain') + if braintag not in strats_brain_dct: + strats_brain_dct[braintag] = [] + strats_brain_dct[braintag].append(os.path.join(cpac_dir, + filename)) + if braintag not in strats_head_dct: + strats_head_dct[braintag] = [] + head_file = filename.replace(braintag, 'desc-head') + strats_head_dct[braintag].append(os.path.join(cpac_dir, + head_file)) for strat in strats_brain_dct.keys(): From 7fb44ccc78d5348b1211ece67548473e75a85a94 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 12:37:42 -0400 Subject: [PATCH 10/33] :bug: Iterate keys without mutating [skip ci] --- CPAC/registration/registration.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index 8658aae219..4c3387b576 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2023 C-PAC Developers +# Copyright (C) 2012-2024 C-PAC Developers # This file is part of C-PAC. @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . # pylint: disable=too-many-lines,ungrouped-imports,wrong-import-order +from copy import deepcopy from typing import Optional from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nodeblock import nodeblock @@ -2365,11 +2366,11 @@ def register_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): wf.connect(node, out, ants_rc, 'inputspec.lesion_mask') if 'space-longitudinal' in brain: - for key in outputs: + for key in list(outputs.keys()): for direction in ['from', 'to']: if f'{direction}-T1w' in key: new_key = key.replace(f'{direction}-T1w', - f'{direction}-longitudinal') + f'{direction}-longitudinal') outputs[new_key] = outputs[key] del outputs[key] From f2da817f5e5b4d76dc5fedc8079f891c6a77b0a2 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 13:11:19 -0400 Subject: [PATCH 11/33] :bug: Fallback to config for regtool [skip ci] --- CPAC/registration/registration.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index 4c3387b576..1148aa5203 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -3529,9 +3529,12 @@ def apply_blip_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, ) def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-template_mode-image_xfm') - reg_tool = check_prov_for_regtool(xfm_prov) + try: + xfm_prov = strat_pool.get_cpac_provenance( + 'from-T1w_to-template_mode-image_xfm') + reg_tool = check_prov_for_regtool(xfm_prov) + except KeyError: + reg_tool = cfg["registration_workflows", "anatomical_registration", "registration", "using"] num_cpus = cfg.pipeline_setup['system_config'][ 'max_cores_per_participant'] From 5d34172a8697606b51ce4770721edad0fc6c870b Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 13:21:44 -0400 Subject: [PATCH 12/33] :necktie: Use longitudinal transform [skip ci] --- CPAC/registration/registration.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index 1148aa5203..7f79057fe0 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -3519,8 +3519,9 @@ def apply_blip_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, switch=["run"], inputs=[ ( - "desc-head_T1w", - "from-T1w_to-template_mode-image_xfm", + ["desc-head_T1w", "space-longitudinal_desc-reorient_T1w"], + ["from-T1w_to-template_mode-image_xfm", + "from-longitudinal_to-template_mode-image_xfm"], "space-template_desc-head_T1w", ), "T1w-template", @@ -3528,10 +3529,10 @@ def apply_blip_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, outputs={"space-template_desc-head_T1w": {"Template": "T1w-template"}}, ) def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - + xfm: list[str] = ["from-T1w_to-template_mode-image_xfm", + "from-longitudinal_to-template_mode-image_xfm"] try: - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-template_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance(xfm) reg_tool = check_prov_for_regtool(xfm_prov) except KeyError: reg_tool = cfg["registration_workflows", "anatomical_registration", "registration", "using"] @@ -3554,7 +3555,8 @@ def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): 'functional_registration']['func_registration_to_template'][ 'FNIRT_pipelines']['interpolation'] - connect = strat_pool.get_data("desc-head_T1w") + connect = strat_pool.get_data(["desc-head_T1w", + "space-longitudinal_desc-reorient_T1w"]) node, out = connect wf.connect(node, out, apply_xfm, 'inputspec.input_image') From 87e723eb89a0eef0a9493633d794121414ed006b Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 13:23:28 -0400 Subject: [PATCH 13/33] :alembic: Don't delete T1w key [skip ci] --- CPAC/registration/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index 7f79057fe0..f159735336 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -2372,7 +2372,7 @@ def register_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): new_key = key.replace(f'{direction}-T1w', f'{direction}-longitudinal') outputs[new_key] = outputs[key] - del outputs[key] + # del outputs[key] return (wf, outputs) From 626984db9822e23bea867d03ae570df5f9acd2ee Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 13:37:29 -0400 Subject: [PATCH 14/33] :necktie: Differentiate T1w/longitudinal space [skip ci] --- .../longitudinal_workflow.py | 2 +- CPAC/pipeline/cpac_pipeline.py | 10 +- CPAC/registration/registration.py | 100 ++++++++++-------- 3 files changed, 63 insertions(+), 49 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index c77db3378c..7d249d97f7 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -555,7 +555,7 @@ def anat_longitudinal_wf(subject_id, sub_list, config): pipeline_blocks = [mask_longitudinal_T1w_brain] pipeline_blocks = build_T1w_registration_stack(rpool, config, - pipeline_blocks) + pipeline_blocks, space="longitudinal") pipeline_blocks = build_segmentation_stack(rpool, config, pipeline_blocks) diff --git a/CPAC/pipeline/cpac_pipeline.py b/CPAC/pipeline/cpac_pipeline.py index 471ead78d8..ea3adac14e 100644 --- a/CPAC/pipeline/cpac_pipeline.py +++ b/CPAC/pipeline/cpac_pipeline.py @@ -98,7 +98,7 @@ create_func_to_T1template_xfm, create_func_to_T1template_symmetric_xfm, warp_wholeheadT1_to_template, - warp_T1mask_to_template, + warp_mask_to_template, apply_phasediff_to_timeseries_separately, apply_blip_to_timeseries_separately, warp_timeseries_to_T1template, @@ -1033,25 +1033,25 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None): return pipeline_blocks -def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None): +def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None, space="T1w"): if not pipeline_blocks: pipeline_blocks = [] reg_blocks = [] - if not rpool.check_rpool('from-T1w_to-template_mode-image_xfm'): + if not rpool.check_rpool(f'from-{space}_to-template_mode-image_xfm'): reg_blocks = [ [register_ANTs_anat_to_template, register_FSL_anat_to_template], overwrite_transform_anat_to_template, warp_wholeheadT1_to_template, - warp_T1mask_to_template + warp_mask_to_template[space] ] if not rpool.check_rpool('desc-restore-brain_T1w'): reg_blocks.append(correct_restore_brain_intensity_abcd) if cfg.voxel_mirrored_homotopic_connectivity['run']: - if not rpool.check_rpool('from-T1w_to-symtemplate_mode-image_xfm'): + if not rpool.check_rpool(f'from-{space}_to-symtemplate_mode-image_xfm'): reg_blocks.append([register_symmetric_ANTs_anat_to_template, register_symmetric_FSL_anat_to_template]) pipeline_blocks += reg_blocks diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index f159735336..957440dc45 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -3572,8 +3572,51 @@ def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): return (wf, outputs) +def _warp_mask_to_template(space): + def warp_mask_to_template_fxn(wf, cfg, strat_pool, pipe_num, opt=None): -@nodeblock( + xfm_prov = strat_pool.get_cpac_provenance( + f'from-{space}_to-template_mode-image_xfm') + reg_tool = check_prov_for_regtool(xfm_prov) + + num_cpus = cfg.pipeline_setup['system_config'][ + 'max_cores_per_participant'] + + num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + + apply_xfm = apply_transform(f'warp_T1mask_to_T1template_{pipe_num}', + reg_tool, time_series=False, num_cpus=num_cpus, + num_ants_cores=num_ants_cores) + + apply_xfm.inputs.inputspec.interpolation = "NearestNeighbor" + ''' + if reg_tool == 'ants': + apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ + 'functional_registration']['func_registration_to_template'][ + 'ANTs_pipelines']['interpolation'] + elif reg_tool == 'fsl': + apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ + 'functional_registration']['func_registration_to_template'][ + 'FNIRT_pipelines']['interpolation'] + ''' + connect = strat_pool.get_data(f"space-{space}_desc-brain_mask") + node, out = connect + wf.connect(node, out, apply_xfm, 'inputspec.input_image') + + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, apply_xfm, 'inputspec.reference') + + node, out = strat_pool.get_data(f"from-{space}_to-template_mode-image_xfm") + wf.connect(node, out, apply_xfm, 'inputspec.transform') + + outputs = { + 'space-template_desc-brain_mask': (apply_xfm, 'outputspec.output_image') + } + + return wf, outputs + return warp_mask_to_template_fxn + +warp_mask_to_template = {"T1": nodeblock( name="transform_T1mask_to_T1template", switch=[ ["registration_workflows", "anatomical_registration", "run"], @@ -3585,48 +3628,19 @@ def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): "T1w-template", ], outputs={"space-template_desc-brain_mask": {"Template": "T1w-template"}}, -) -def warp_T1mask_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-template_mode-image_xfm') - reg_tool = check_prov_for_regtool(xfm_prov) - - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] - - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] - - apply_xfm = apply_transform(f'warp_T1mask_to_T1template_{pipe_num}', - reg_tool, time_series=False, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) - - apply_xfm.inputs.inputspec.interpolation = "NearestNeighbor" - ''' - if reg_tool == 'ants': - apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': - apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] - ''' - connect = strat_pool.get_data("space-T1w_desc-brain_mask") - node, out = connect - wf.connect(node, out, apply_xfm, 'inputspec.input_image') - - node, out = strat_pool.get_data("T1w-template") - wf.connect(node, out, apply_xfm, 'inputspec.reference') - - node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') - - outputs = { - 'space-template_desc-brain_mask': (apply_xfm, 'outputspec.output_image') - } - - return (wf, outputs) +)(_warp_mask_to_template("T1")), "longitudinal": nodeblock( + name="transform_longitudinal_mask_to_T1template", + switch=[ + ["registration_workflows", "anatomical_registration", "run"], + ["anatomical_preproc", "run"], + ["anatomical_preproc", "brain_extraction", "run"], + ], + inputs=[ + ("space-longitudinal_desc-brain_mask", "from-longitudinal_to-template_mode-image_xfm"), + "T1w-template", + ], + outputs={"space-template_desc-brain_mask": {"Template": "T1w-template"}}, +)(_warp_mask_to_template("longitudinal"))} @nodeblock( From f6c27f84571e96500baecf047661f82acb0bfed4 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 21:20:39 -0400 Subject: [PATCH 15/33] :bug: Include `space-longitudinal_pveseg` in `tissue_seg_fsl)fast` --- CPAC/seg_preproc/seg_preproc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CPAC/seg_preproc/seg_preproc.py b/CPAC/seg_preproc/seg_preproc.py index 78d40703d9..be5ccd1299 100644 --- a/CPAC/seg_preproc/seg_preproc.py +++ b/CPAC/seg_preproc/seg_preproc.py @@ -456,6 +456,9 @@ def create_seg_preproc_antsJointLabel_method( "space-longitudinal_label-CSF_probseg", "space-longitudinal_label-GM_probseg", "space-longitudinal_label-WM_probseg", + "space-longitudinal_label-CSF_pveseg", + "space-longitudinal_label-GM_pveseg", + "space-longitudinal_label-WM_pveseg", ], ) def tissue_seg_fsl_fast(wf, cfg, strat_pool, pipe_num, opt=None): From 8a17f72508df870991399f424869c75bb1f9cf0e Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 21:22:24 -0400 Subject: [PATCH 16/33] :recycle: Simplify outputs definition [skip ci] --- CPAC/seg_preproc/seg_preproc.py | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/CPAC/seg_preproc/seg_preproc.py b/CPAC/seg_preproc/seg_preproc.py index be5ccd1299..c060dfdbd9 100644 --- a/CPAC/seg_preproc/seg_preproc.py +++ b/CPAC/seg_preproc/seg_preproc.py @@ -434,32 +434,10 @@ def create_seg_preproc_antsJointLabel_method( "GM-path", "WM-path", ], - outputs=[ - "label-CSF_mask", - "label-GM_mask", - "label-WM_mask", - "label-CSF_desc-preproc_mask", - "label-GM_desc-preproc_mask", - "label-WM_desc-preproc_mask", - "label-CSF_probseg", - "label-GM_probseg", - "label-WM_probseg", - "label-CSF_pveseg", - "label-GM_pveseg", - "label-WM_pveseg", - "space-longitudinal_label-CSF_mask", - "space-longitudinal_label-GM_mask", - "space-longitudinal_label-WM_mask", - "space-longitudinal_label-CSF_desc-preproc_mask", - "space-longitudinal_label-GM_desc-preproc_mask", - "space-longitudinal_label-WM_desc-preproc_mask", - "space-longitudinal_label-CSF_probseg", - "space-longitudinal_label-GM_probseg", - "space-longitudinal_label-WM_probseg", - "space-longitudinal_label-CSF_pveseg", - "space-longitudinal_label-GM_pveseg", - "space-longitudinal_label-WM_pveseg", - ], + outputs=[f"{long}label-{tissue}_{entity}" for + long in ["", "space-longitudinal_"] for + tissue in ["CSF", "GM", "WM"] for + entity in ["mask", "desc-preproc_mask", "probseg", "pveseg"]], ) def tissue_seg_fsl_fast(wf, cfg, strat_pool, pipe_num, opt=None): # FSL-FAST From 2a54356efd11e7aad9cb2c8723d38aeeb3c117c7 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 29 Oct 2024 21:50:08 -0400 Subject: [PATCH 17/33] :alien: Only call `ingress_output_dir` if "derivatives_dir" [skip ci] ref 3a0c1236b034dc563eadea3db27dd7125403f9e1 --- CPAC/longitudinal_pipeline/longitudinal_workflow.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 7d249d97f7..a4b4e6c0d6 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -603,8 +603,10 @@ def anat_longitudinal_wf(subject_id, sub_list, config): config.pipeline_setup[ 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' - rpool = ingress_output_dir(config, rpool, long_id, - creds_path=input_creds_path) + if "derivatives_dir" in session: + rpool = ingress_output_dir( + wf, config, rpool, long_id, data_paths=session, part_id=subject_id, + ses_id=unique_id, creds_path=input_creds_path) rpool.pickle("606") select_node_name = f'select_{unique_id}' select_sess = pe.Node(Function(input_names=['session', From b09257a8b6c020c07251b51adcf68db1145e25af Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Thu, 31 Oct 2024 10:20:19 -0400 Subject: [PATCH 18/33] :recycle: Fix broken connections in `anat_longitudinal_wf` --- .../longitudinal_preproc.py | 3 +- .../longitudinal_workflow.py | 372 +++-- .../longitudinal_workflow.py.orig | 1215 ----------------- CPAC/pipeline/cpac_runner.py | 6 +- CPAC/pipeline/engine.py | 5 +- CPAC/registration/registration.py | 4 +- 6 files changed, 196 insertions(+), 1409 deletions(-) delete mode 100644 CPAC/longitudinal_pipeline/longitudinal_workflow.py.orig diff --git a/CPAC/longitudinal_pipeline/longitudinal_preproc.py b/CPAC/longitudinal_pipeline/longitudinal_preproc.py index 44f88675b4..93f2cb99f9 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_preproc.py +++ b/CPAC/longitudinal_pipeline/longitudinal_preproc.py @@ -270,7 +270,8 @@ def flirt_node(in_img, output_img, output_mat): return node_list -def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, avg_method='median', dof=12, +def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, + avg_method='median', dof=12, interp='trilinear', cost='corratio', mat_type='matrix', convergence_threshold=-1, thread_pool=2, unique_id_list=None): """ diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index a4b4e6c0d6..83bfcc9a63 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -23,8 +23,7 @@ from CPAC.pipeline.nodeblock import nodeblock -from nipype import config -from nipype import logging +from nipype import config as nipype_config, logging from CPAC.pipeline import nipype_pipeline_engine as pe import nipype.interfaces.afni as afni import nipype.interfaces.fsl as fsl @@ -64,6 +63,7 @@ ) from CPAC.utils import find_files, function +from CPAC.utils.configuration import Configuration from CPAC.utils.outputs import Outputs from CPAC.utils.strategy import Strategy from CPAC.utils.utils import ( @@ -280,7 +280,7 @@ def mask_longitudinal_T1w_brain(wf, cfg, strat_pool, pipe_num, opt=None): ( "space-longitudinal_desc-brain_T1w", "from-longitudinal_to-template_mode-image_xfm", - ) + ), "T1w-brain-template" ], outputs=["space-template_desc-brain_T1w"], ) @@ -313,7 +313,7 @@ def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") wf.connect(node, out, apply_xfm, 'inputspec.input_image') - node, out = strat_pool.get_data("T1w_brain_template") + node, out = strat_pool.get_data("T1w-brain-template") wf.connect(node, out, apply_xfm, 'inputspec.reference') node, out = \ @@ -334,7 +334,9 @@ def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, switch=["run"], inputs=[ ( - "from-longitudinal_to-T1w_mode-image_desc-linear_xfm", + "space-longitudinal_desc-brain_T1w", + ["from-longitudinal_to-T1w_mode-image_desc-linear_xfm", + "from-T1w_to-longitudinal_mode-image_desc-linear_xfm"], "space-longitudinal_label-CSF_mask", "space-longitudinal_label-GM_mask", "space-longitudinal_label-WM_mask", @@ -344,7 +346,8 @@ def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, "space-longitudinal_label-CSF_probseg", "space-longitudinal_label-GM_probseg", "space-longitudinal_label-WM_probseg", - ) + ), + "T1w-brain-template" ], outputs=[ "label-CSF_mask", @@ -358,11 +361,23 @@ def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, "label-WM_probseg", ], ) -def warp_longitudinal_seg_to_T1w(wf, cfg, strat_pool, pipe_num, opt=None): +def warp_longitudinal_seg_to_T1w(wf, cfg, strat_pool: ResourcePool, pipe_num, opt=None): - xfm_prov = strat_pool.get_cpac_provenance( - 'from-longitudinal_to-T1w_mode-image_desc-linear_xfm') - reg_tool = check_prov_for_regtool(xfm_prov) + if strat_pool.check_rpool("from-longitudinal_to-T1w_mode-image_desc-linear_xfm"): + xfm_prov = strat_pool.get_cpac_provenance( + "from-longitudinal_to-T1w_mode-image_desc-linear_xfm") + reg_tool = check_prov_for_regtool(xfm_prov) + xfm: tuple[pe.Node, str] = strat_pool.get_data("from-longitudinal_to-T1w_mode-image_desc-linear_xfm") + else: + xfm_prov = strat_pool.get_cpac_provenance( + "from-T1w_to-longitudinal_mode-image_desc-linear_xfm") + reg_tool = check_prov_for_regtool(xfm_prov) + # create inverse xfm if we don't have it + invt = pe.Node(interface=fsl.ConvertXFM(), name='convert_xfm') + invt.inputs.invert_xfm = True + wf.connect( + *strat_pool.get_data("from-T1w_to-longitudinal_mode-image_desc-linear_xfm"), invt, "in_file") + xfm = (invt, "out_file") num_cpus = cfg.pipeline_setup['system_config'][ 'max_cores_per_participant'] @@ -393,51 +408,57 @@ def warp_longitudinal_seg_to_T1w(wf, cfg, strat_pool, pipe_num, opt=None): 'registration']['FSL-FNIRT']['interpolation'] node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") + wf.connect(node, out, apply_xfm, 'inputspec.input_image') - node, out = strat_pool.get_data("T1w_brain_template") + node, out = strat_pool.get_data("T1w-brain-template") wf.connect(node, out, apply_xfm, 'inputspec.reference') - - node, out = \ - strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + + wf.connect(*xfm, apply_xfm, 'inputspec.transform') outputs[f'label-{label}'] = (apply_xfm, 'outputspec.output_image') return (wf, outputs) -def anat_longitudinal_wf(subject_id, sub_list, config): +def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configuration, + dry_run: bool = False) -> None: """ + Build and run an anatomical longitudinal workflow. + Parameters ---------- - subject_id : str + subject_id the id of the subject - sub_list : list of dict + sub_list this is a list of sessions for one subject and each session if the same dictionary as the one given to prep_workflow - config : configuration + config a configuration object containing the information of the pipeline config. (Same as for prep_workflow) - - Returns - ------- - None + dry_run + build graph without running? """ + nipype_config.update_config({ + 'execution': { + 'crashfile_format': 'txt', + 'stop_on_first_crash': config['pipeline_setup', 'system_config', + 'fail_fast']}}) config["subject_id"] = subject_id # list of lists for every strategy session_id_list = [] session_wfs = {} cpac_dirs = [] - out_dir = config.pipeline_setup['output_directory']['path'] + out_dir: str = config.pipeline_setup['output_directory']['path'] - orig_pipe_name = config.pipeline_setup['pipeline_name'] + orig_pipe_name: str = config.pipeline_setup['pipeline_name'] + pools: list[ResourcePool] = [] # Loop over the sessions to create the input for the longitudinal # algorithm for session in sub_list: - unique_id = session['unique_id'] + unique_id: str = session['unique_id'] session_id_list.append(unique_id) try: @@ -456,10 +477,10 @@ def anat_longitudinal_wf(subject_id, sub_list, config): except KeyError: input_creds_path = None - workflow = initialize_nipype_wf(config, sub_list[0], + workflow: pe.Workflow = initialize_nipype_wf(config, sub_list[0], # just grab the first one for the name name="anat_longitudinal_pre-preproc") - + rpool: ResourcePool workflow, rpool = initiate_rpool(workflow, config, session) pipeline_blocks = build_anat_preproc_stack(rpool, config) workflow = connect_pipeline(workflow, config, rpool, pipeline_blocks) @@ -467,179 +488,160 @@ def anat_longitudinal_wf(subject_id, sub_list, config): session_wfs[unique_id] = rpool rpool.gather_pipes(workflow, config) - rpool.pickle("470") - workflow.run() - rpool.pickle("472") - cpac_dir = os.path.join(out_dir, f'pipeline_{orig_pipe_name}', - f'{subject_id}/{unique_id}') - cpac_dirs.append(os.path.join(cpac_dir, 'anat')) + if not dry_run: + workflow.run() + pools.append(rpool) # Now we have all the anat_preproc set up for every session # loop over the different anat preproc strategies - strats_brain_dct = {} - strats_head_dct = {} - for cpac_dir in cpac_dirs: - if os.path.isdir(cpac_dir): - for filename in os.listdir(cpac_dir): - if 'T1w.nii' in filename: - for tag in filename.split('_'): - braintag: str = tag - if 'desc-' in tag and ('brain' in tag or 'preproc' in tag): - braintag = tag.replace('preproc', 'brain') - if braintag not in strats_brain_dct: - strats_brain_dct[braintag] = [] - strats_brain_dct[braintag].append(os.path.join(cpac_dir, - filename)) - if braintag not in strats_head_dct: - strats_head_dct[braintag] = [] - head_file = filename.replace(braintag, 'desc-head') - strats_head_dct[braintag].append(os.path.join(cpac_dir, - head_file)) - - for strat in strats_brain_dct.keys(): - - wf = initialize_nipype_wf(config, sub_list[0], - # just grab the first one for the name - name=f"template_node_{strat}") - - config.pipeline_setup[ - 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' - - template_node_name = f'longitudinal_anat_template_{strat}' - - # This node will generate the longitudinal template (the functions are - # in longitudinal_preproc) - # Later other algorithms could be added to calculate it, like the - # multivariate template from ANTS - # It would just require to change it here. - template_node = subject_specific_template( - workflow_name=template_node_name - ) + strats_dct: dict[str, list] = {key: [pool.get_data(key) for pool in pools] + for key in ["desc-brain_T1w", "desc-head_T1w"]} + # Rename nodes to include session name to avoid duplicates + for key in strats_dct: + for i, resource in enumerate(strats_dct[key]): + resource[0].name = f"{resource[0].name}_{session_id_list[i]}" + + wf = initialize_nipype_wf(config, sub_list[0], + # just grab the first one for the name + name=f"template_node_brain") + + config.pipeline_setup[ + 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' + + template_node_name = 'longitudinal_anat_template_brain' + + # This node will generate the longitudinal template (the functions are + # in longitudinal_preproc) + # Later other algorithms could be added to calculate it, like the + # multivariate template from ANTS + # It would just require to change it here. + template_node = subject_specific_template( + workflow_name=template_node_name + ) - template_node.inputs.set( - avg_method=config.longitudinal_template_generation[ - 'average_method'], - dof=config.longitudinal_template_generation['dof'], - interp=config.longitudinal_template_generation['interp'], - cost=config.longitudinal_template_generation['cost'], - convergence_threshold=config.longitudinal_template_generation[ - 'convergence_threshold'], - thread_pool=config.longitudinal_template_generation[ - 'thread_pool'], - unique_id_list=list(session_wfs.keys()) - ) + template_node.inputs.set( + avg_method=config.longitudinal_template_generation[ + 'average_method'], + dof=config.longitudinal_template_generation['dof'], + interp=config.longitudinal_template_generation['interp'], + cost=config.longitudinal_template_generation['cost'], + convergence_threshold=config.longitudinal_template_generation[ + 'convergence_threshold'], + thread_pool=config.longitudinal_template_generation[ + 'thread_pool'], + unique_id_list=list(session_wfs.keys()) + ) - template_node.inputs.input_brain_list = strats_brain_dct[strat] - template_node.inputs.input_skull_list = strats_head_dct[strat] + num_sessions = len(strats_dct["desc-brain_T1w"]) + merge_brains = pe.Node(Merge(num_sessions), name="merge_brains") + merge_skulls = pe.Node(Merge(num_sessions), name="merge_skulls") - long_id = f'longitudinal_{subject_id}_strat-{strat}' + for i in list(range(0, num_sessions)): + wf.connect(*strats_dct["desc-brain_T1w"][i], merge_brains, f"in{i + 1}") + wf.connect(*strats_dct["desc-head_T1w"][i], merge_skulls, f"in{i + 1}") + wf.connect(merge_brains, "out", template_node, "input_brain_list") + wf.connect(merge_skulls, "out", template_node, "input_skull_list") - wf, rpool = initiate_rpool(wf, config, part_id=long_id) + long_id = f'longitudinal_{subject_id}_strat-desc-brain_T1w' - rpool.set_data("space-longitudinal_desc-brain_T1w", - template_node, 'brain_template', {}, - "", template_node_name) + wf, rpool = initiate_rpool(wf, config, part_id=long_id) + + rpool.set_data("space-longitudinal_desc-brain_T1w", + template_node, 'brain_template', {}, + "", template_node_name) - rpool.set_data("space-longitudinal_desc-brain_T1w-template", - template_node, 'brain_template', {}, - "", template_node_name) + rpool.set_data("space-longitudinal_desc-brain_T1w-template", + template_node, 'brain_template', {}, + "", template_node_name) - rpool.set_data("space-longitudinal_desc-reorient_T1w", - template_node, 'skull_template', {}, - "", template_node_name) + rpool.set_data("space-longitudinal_desc-reorient_T1w", + template_node, 'skull_template', {}, + "", template_node_name) - rpool.set_data("space-longitudinal_desc-reorient_T1w-template", - template_node, 'skull_template', {}, - "", template_node_name) + rpool.set_data("space-longitudinal_desc-reorient_T1w-template", + template_node, 'skull_template', {}, + "", template_node_name) - pipeline_blocks = [mask_longitudinal_T1w_brain] + pipeline_blocks = [mask_longitudinal_T1w_brain] - pipeline_blocks = build_T1w_registration_stack(rpool, config, - pipeline_blocks, space="longitudinal") + pipeline_blocks = build_T1w_registration_stack(rpool, config, + pipeline_blocks, space="longitudinal") - pipeline_blocks = build_segmentation_stack(rpool, config, - pipeline_blocks) + pipeline_blocks = build_segmentation_stack(rpool, config, + pipeline_blocks) - wf = connect_pipeline(wf, config, rpool, pipeline_blocks) + wf = connect_pipeline(wf, config, rpool, pipeline_blocks) - excl = ['space-longitudinal_desc-brain_T1w', - 'space-longitudinal_desc-reorient_T1w', - 'space-longitudinal_desc-brain_mask'] - rpool.pickle("566") - rpool.gather_pipes(wf, config, add_excl=excl) + excl = ['space-longitudinal_desc-brain_T1w', + 'space-longitudinal_desc-reorient_T1w', + 'space-longitudinal_desc-brain_mask'] + rpool.gather_pipes(wf, config, add_excl=excl) - # this is going to run multiple times! - # once for every strategy! + if not dry_run: wf.run() - rpool.pickle("572") - longitudinal_rpool = rpool.copy_rpool() - # now, just write out a copy of the above to each session - config.pipeline_setup['pipeline_name'] = orig_pipe_name - for session in sub_list: + # now, just write out a copy of the above to each session + config.pipeline_setup['pipeline_name'] = orig_pipe_name + for session in sub_list: - unique_id = session['unique_id'] + unique_id = session['unique_id'] - try: - creds_path = session['creds_path'] - if creds_path and 'none' not in creds_path.lower(): - if os.path.exists(creds_path): - input_creds_path = os.path.abspath(creds_path) - else: - err_msg = 'Credentials path: "%s" for subject "%s" ' \ - 'session "%s" was not found. Check this path ' \ - 'and try again.' % (creds_path, subject_id, - unique_id) - raise Exception(err_msg) + try: + creds_path = session['creds_path'] + if creds_path and 'none' not in creds_path.lower(): + if os.path.exists(creds_path): + input_creds_path = os.path.abspath(creds_path) else: - input_creds_path = None - except KeyError: + err_msg = 'Credentials path: "%s" for subject "%s" ' \ + 'session "%s" was not found. Check this path ' \ + 'and try again.' % (creds_path, subject_id, + unique_id) + raise Exception(err_msg) + else: input_creds_path = None + except KeyError: + input_creds_path = None + + wf = initialize_nipype_wf(config, sub_list[0]) + + wf, rpool = initiate_rpool(wf, config, session, rpool=rpool) + + config.pipeline_setup[ + 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' + if "derivatives_dir" in session: + rpool = ingress_output_dir( + wf, config, rpool, long_id, data_paths=session, part_id=subject_id, + ses_id=unique_id, creds_path=input_creds_path) + + select_node_name = f'FSL_select_{unique_id}' + select_sess = pe.Node(Function(input_names=['session', + 'output_brains', + 'warps'], + output_names=['brain_path', 'warp_path'], + function=select_session), + name=select_node_name) + select_sess.inputs.session = unique_id + + wf.connect(template_node, 'output_brain_list', select_sess, + 'output_brains') + wf.connect(template_node, 'warp_list', select_sess, 'warps') + + rpool.set_data("space-longitudinal_desc-brain_T1w", + select_sess, 'brain_path', {}, "", + select_node_name) - wf = initialize_nipype_wf(config, sub_list[0]) - - wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) - rpool.pickle("600") - - config.pipeline_setup[ - 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' - if "derivatives_dir" in session: - rpool = ingress_output_dir( - wf, config, rpool, long_id, data_paths=session, part_id=subject_id, - ses_id=unique_id, creds_path=input_creds_path) - rpool.pickle("606") - select_node_name = f'select_{unique_id}' - select_sess = pe.Node(Function(input_names=['session', - 'output_brains', - 'warps'], - output_names=['brain_path', - 'warp_path'], - function=select_session), - name=select_node_name) - select_sess.inputs.session = unique_id - - wf.connect(template_node, 'output_brain_list', select_sess, - 'output_brains') - wf.connect(template_node, 'warp_list', select_sess, 'warps') - - rpool.set_data("space-longitudinal_desc-brain_T1w", - select_sess, 'brain_path', {}, "", - select_node_name) - - rpool.set_data("from-T1w_to-longitudinal_mode-image_" - "desc-linear_xfm", - select_sess, 'warp_path', {}, "", - select_node_name) - - config.pipeline_setup['pipeline_name'] = orig_pipe_name - excl = ['space-template_desc-brain_T1w', - 'space-T1w_desc-brain_mask'] - - rpool.gather_pipes(wf, config, add_excl=excl) - rpool.pickle("635") + rpool.set_data("from-T1w_to-longitudinal_mode-image_" + "desc-linear_xfm", + select_sess, 'warp_path', {}, "", + select_node_name) + + config.pipeline_setup['pipeline_name'] = orig_pipe_name + excl = ['space-template_desc-brain_T1w', + 'space-T1w_desc-brain_mask'] + + rpool.gather_pipes(wf, config, add_excl=excl) + if not dry_run: wf.run() - longitudinal_rpool = rpool.copy_rpool() - rpool.pickle("638") # begin single-session stuff again for session in sub_list: @@ -664,7 +666,7 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf = initialize_nipype_wf(config, sub_list[0]) - wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) + wf, rpool = initiate_rpool(wf, config, session, rpool=rpool) pipeline_blocks = [warp_longitudinal_T1w_to_template, warp_longitudinal_seg_to_T1w] @@ -672,13 +674,10 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf = connect_pipeline(wf, config, rpool, pipeline_blocks) rpool.gather_pipes(wf, config) - rpool.pickle("671") # this is going to run multiple times! # once for every strategy! - wf.run() - rpool.pickle("675") - - + if not dry_run: + wf.run() # TODO check: @@ -1187,11 +1186,10 @@ def func_longitudinal_template_wf(subject_id, strat_list, config): thread_pool=config.longitudinal_template_thread_pool, ) - workflow.connect(merge_func_preproc_node, 'brain_list', - template_node, 'input_brain_list') - - workflow.connect(merge_func_preproc_node, 'skull_list', - template_node, 'input_skull_list') + # workflow.connect(merge_func_preproc_node, 'brain_list', + # template_node, 'input_brain_list') + # workflow.connect(merge_func_preproc_node, 'skull_list', + # template_node, 'input_skull_list') workflow, strat_list = register_func_longitudinal_template_to_standard( template_node, diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py.orig b/CPAC/longitudinal_pipeline/longitudinal_workflow.py.orig deleted file mode 100644 index 73b2a1ff77..0000000000 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py.orig +++ /dev/null @@ -1,1215 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (C) 2020-2022 C-PAC Developers - -# This file is part of C-PAC. - -# C-PAC is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. - -# C-PAC is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public -# License for more details. - -# You should have received a copy of the GNU Lesser General Public -# License along with C-PAC. If not, see . -import os -<<<<<<< HEAD -import copy -import time -import shutil -from CPAC.pipeline.nodeblock import nodeblock -======= -from typing import Optional ->>>>>>> 4dc13cdc1 (:necktie: Persist longitudinal resource pool) - -from nipype import config -from nipype import logging -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.afni as afni -import nipype.interfaces.fsl as fsl -import nipype.interfaces.io as nio -from nipype.interfaces.utility import Merge, IdentityInterface -import nipype.interfaces.utility as util - -from indi_aws import aws_utils - -<<<<<<< HEAD -from CPAC.utils.utils import concat_list -from CPAC.utils.interfaces.datasink import DataSink -from CPAC.utils.interfaces.function import Function - -import CPAC - -from CPAC.pipeline.cpac_pipeline import initialize_nipype_wf, \ - connect_pipeline, build_anat_preproc_stack, build_T1w_registration_stack,\ - build_segmentation_stack -from CPAC.pipeline.engine import initiate_rpool, ingress_output_dir - -======= -from CPAC.longitudinal_pipeline.longitudinal_preproc import subject_specific_template -from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.pipeline.cpac_pipeline import ( - build_anat_preproc_stack, - build_segmentation_stack, - build_T1w_registration_stack, - connect_pipeline, - initialize_nipype_wf, -) -from CPAC.pipeline.engine import ingress_output_dir, initiate_rpool, ResourcePool -from CPAC.pipeline.nodeblock import nodeblock ->>>>>>> 4dc13cdc1 (:necktie: Persist longitudinal resource pool) -from CPAC.registration import ( - create_fsl_flirt_linear_reg, - create_fsl_fnirt_nonlinear_reg, - create_wf_calculate_ants_warp -) - -from CPAC.registration.registration import apply_transform - -from CPAC.utils.datasource import ( - resolve_resolution, - create_anat_datasource, - create_check_for_s3_node -) - -from CPAC.longitudinal_pipeline.longitudinal_preproc import ( - subject_specific_template -) - -from CPAC.utils import find_files, function -from CPAC.utils.outputs import Outputs -from CPAC.utils.strategy import Strategy -from CPAC.utils.utils import ( - check_config_resources, - check_prov_for_regtool -) - -logger = logging.getLogger('nipype.workflow') - - -@nodeblock( - name="mask_T1w_longitudinal_template", - config=["longitudinal_template_generation"], - switch=["run"], - inputs=["desc-brain_T1w"], - outputs=["space-T1w_desc-brain_mask"], -) -def mask_T1w_longitudinal_template(wf, cfg, strat_pool, pipe_num, opt=None): - - brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'longitudinal_anatomical_brain_mask_' - f'{pipe_num}') - brain_mask.inputs.args = '-bin' - - node, out = strat_pool.get_data('desc-brain_T1w') - wf.connect(node, out, brain_mask, 'in_file') - - outputs = { - 'space-T1w_desc-brain_mask': (brain_mask, 'out_file') - } - - return (wf, outputs) - - -def create_datasink(datasink_name, config, subject_id, session_id='', - strat_name='', map_node_iterfield=None): - """ - - Parameters - ---------- - datasink_name - config - subject_id - session_id - strat_name - map_node_iterfield - - Returns - ------- - - """ - try: - encrypt_data = bool( - config.pipeline_setup['Amazon-AWS']['s3_encryption']) - except: - encrypt_data = False - - # TODO Enforce value with schema validation - # Extract credentials path for output if it exists - try: - # Get path to creds file - creds_path = '' - if config.pipeline_setup['Amazon-AWS'][ - 'aws_output_bucket_credentials']: - creds_path = str(config.pipeline_setup['Amazon-AWS'][ - 'aws_output_bucket_credentials']) - creds_path = os.path.abspath(creds_path) - - if config.pipeline_setup['output_directory'][ - 'path'].lower().startswith('s3://'): - # Test for s3 write access - s3_write_access = \ - aws_utils.test_bucket_access(creds_path, - config.pipeline_setup[ - 'output_directory']['path']) - - if not s3_write_access: - raise Exception('Not able to write to bucket!') - - except Exception as e: - if config.pipeline_setup['output_directory'][ - 'path'].lower().startswith('s3://'): - err_msg = 'There was an error processing credentials or ' \ - 'accessing the S3 bucket. Check and try again.\n' \ - 'Error: %s' % e - raise Exception(err_msg) - - if map_node_iterfield is not None: - ds = pe.MapNode( - DataSink(infields=map_node_iterfield), - name='sinker_{}'.format(datasink_name), - iterfield=map_node_iterfield - ) - else: - ds = pe.Node( - DataSink(), - name='sinker_{}'.format(datasink_name) - ) - - ds.inputs.base_directory = config.pipeline_setup['output_directory'][ - 'path'] - ds.inputs.creds_path = creds_path - ds.inputs.encrypt_bucket_keys = encrypt_data - ds.inputs.container = os.path.join( - 'pipeline_%s_%s' % ( - config.pipeline_setup['pipeline_name'], strat_name), - subject_id, session_id - ) - return ds - - -def connect_anat_preproc_inputs(strat, anat_preproc, strat_name, - strat_nodes_list_list, workflow): - """ - Parameters - ---------- - strat : Strategy - the strategy object you want to fork - anat_preproc : Workflow - the anat_preproc workflow node to be connected and added to the resource pool - strat_name : str - name of the strategy - strat_nodes_list_list : list - a list of strat_nodes_list - workflow : Workflow - main longitudinal workflow - - Returns - ------- - new_strat : Strategy - the fork of strat with the resource pool updated - strat_nodes_list_list : list - a list of strat_nodes_list - """ - - new_strat = strat.fork() - - tmp_node, out_key = new_strat['anatomical'] - workflow.connect(tmp_node, out_key, anat_preproc, 'inputspec.anat') - - tmp_node, out_key = new_strat['template_cmass'] - workflow.connect(tmp_node, out_key, anat_preproc, - 'inputspec.template_cmass') - - new_strat.append_name(anat_preproc.name) - - new_strat.update_resource_pool({ - 'anatomical_brain': ( - anat_preproc, 'outputspec.brain'), - 'anatomical_skull_leaf': ( - anat_preproc, 'outputspec.reorient'), - 'anatomical_brain_mask': ( - anat_preproc, 'outputspec.brain_mask'), - }) - - try: - strat_nodes_list_list[strat_name].append(new_strat) - except KeyError: - strat_nodes_list_list[strat_name] = [new_strat] - - return new_strat, strat_nodes_list_list - - -def pick_map(file_list, index, file_type): - if isinstance(file_list, list): - if len(file_list) == 1: - file_list = file_list[0] - for file_name in file_list: - if file_name.endswith(f"{file_type}_{index}.nii.gz"): - return file_name - return None - - -def select_session(session, output_brains, warps): - brain_path = None - warp_path = None - for brain_path in output_brains: - if f'{session}_' in brain_path: - break - for warp_path in warps: - if f'{session}_' in warp_path: - break - return (brain_path, warp_path) - - -@nodeblock( - name="mask_longitudinal_T1w_brain", - config=["longitudinal_template_generation"], - switch=["run"], - inputs=["space-longitudinal_desc-brain_T1w"], - outputs=["space-longitudinal_desc-brain_mask"], -) -def mask_longitudinal_T1w_brain(wf, cfg, strat_pool, pipe_num, opt=None): - - brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'longitudinal_T1w_brain_mask_{pipe_num}') - brain_mask.inputs.args = '-bin' - - node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") - wf.connect(node, out, brain_mask, 'in_file') - - outputs = { - 'space-longitudinal_desc-brain_mask': (brain_mask, 'out_file') - } - - return (wf, outputs) - - -@nodeblock( - name="warp_longitudinal_T1w_to_template", - config=["longitudinal_template_generation"], - switch=["run"], - inputs=[ - ( - "space-longitudinal_desc-brain_T1w", - "from-longitudinal_to-template_mode-image_xfm", - ) - ], - outputs=["space-template_desc-brain_T1w"], -) -def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, - opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-longitudinal_to-template_mode-image_xfm') - reg_tool = check_prov_for_regtool(xfm_prov) - - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] - - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] - - apply_xfm = apply_transform(f'warp_longitudinal_to_T1template_{pipe_num}', - reg_tool, time_series=False, - num_cpus=num_cpus, - num_ants_cores=num_ants_cores) - - if reg_tool == 'ants': - apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'anatomical_registration']['registration']['ANTs'][ - 'interpolation'] - elif reg_tool == 'fsl': - apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT'][ - 'interpolation'] - - node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") - wf.connect(node, out, apply_xfm, 'inputspec.input_image') - - node, out = strat_pool.get_data("T1w_brain_template") - wf.connect(node, out, apply_xfm, 'inputspec.reference') - - node, out = \ - strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') - - outputs = { - 'space-template_desc-brain_T1w': - (apply_xfm, 'outputspec.output_image') - } - - return (wf, outputs) - - -@nodeblock( - name="warp_longitudinal_seg_to_T1w", - config=["longitudinal_template_generation"], - switch=["run"], - inputs=[ - ( - "from-longitudinal_to-T1w_mode-image_desc-linear_xfm", - "space-longitudinal_label-CSF_mask", - "space-longitudinal_label-GM_mask", - "space-longitudinal_label-WM_mask", - "space-longitudinal_label-CSF_desc-preproc_mask", - "space-longitudinal_label-GM_desc-preproc_mask", - "space-longitudinal_label-WM_desc-preproc_mask", - "space-longitudinal_label-CSF_probseg", - "space-longitudinal_label-GM_probseg", - "space-longitudinal_label-WM_probseg", - ) - ], - outputs=[ - "label-CSF_mask", - "label-GM_mask", - "label-WM_mask", - "label-CSF_desc-preproc_mask", - "label-GM_desc-preproc_mask", - "label-WM_desc-preproc_mask", - "label-CSF_probseg", - "label-GM_probseg", - "label-WM_probseg", - ], -) -def warp_longitudinal_seg_to_T1w(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-longitudinal_to-T1w_mode-image_desc-linear_xfm') - reg_tool = check_prov_for_regtool(xfm_prov) - - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] - - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] - - outputs = {} - - labels = [ - 'CSF_mask', 'CSF_desc-preproc_mask', 'CSF_probseg', - 'GM_mask', 'GM_desc-preproc_mask', 'GM_probseg', - 'WM_mask', 'WM_desc-preproc_mask', 'WM_probseg', - ] - - for label in labels: - apply_xfm = apply_transform(f'warp_longitudinal_seg_to_T1w_{label}_' - f'{pipe_num}', reg_tool, - time_series=False, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) - - if reg_tool == 'ants': - apply_xfm.inputs.inputspec.interpolation = \ - cfg.registration_workflows['anatomical_registration'][ - 'registration']['ANTs']['interpolation'] - elif reg_tool == 'fsl': - apply_xfm.inputs.inputspec.interpolation = \ - cfg.registration_workflows['anatomical_registration'][ - 'registration']['FSL-FNIRT']['interpolation'] - - node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") - wf.connect(node, out, apply_xfm, 'inputspec.input_image') - - node, out = strat_pool.get_data("T1w_brain_template") - wf.connect(node, out, apply_xfm, 'inputspec.reference') - - node, out = \ - strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') - - outputs[f'label-{label}'] = (apply_xfm, 'outputspec.output_image') - - return (wf, outputs) - - -def anat_longitudinal_wf(subject_id, sub_list, config): - """ - Parameters - ---------- - subject_id : str - the id of the subject - sub_list : list of dict - this is a list of sessions for one subject and each session if the same dictionary as the one given to - prep_workflow - config : configuration - a configuration object containing the information of the pipeline config. (Same as for prep_workflow) - - Returns - ------- - None - """ - config["subject_id"] = subject_id - # list of lists for every strategy - session_id_list = [] - session_wfs = {} - - cpac_dirs = [] - out_dir = config.pipeline_setup['output_directory']['path'] - - orig_pipe_name = config.pipeline_setup['pipeline_name'] - - # Loop over the sessions to create the input for the longitudinal - # algorithm - for session in sub_list: - - unique_id = session['unique_id'] - session_id_list.append(unique_id) - - try: - creds_path = session['creds_path'] - if creds_path and 'none' not in creds_path.lower(): - if os.path.exists(creds_path): - input_creds_path = os.path.abspath(creds_path) - else: - err_msg = 'Credentials path: "%s" for subject "%s" ' \ - 'session "%s" was not found. Check this path ' \ - 'and try again.' % (creds_path, subject_id, - unique_id) - raise Exception(err_msg) - else: - input_creds_path = None - except KeyError: - input_creds_path = None - - workflow = initialize_nipype_wf(config, sub_list[0], - # just grab the first one for the name - name="anat_longitudinal_pre-preproc") - - workflow, rpool = initiate_rpool(workflow, config, session) - pipeline_blocks = build_anat_preproc_stack(rpool, config) - workflow = connect_pipeline(workflow, config, rpool, pipeline_blocks) - - session_wfs[unique_id] = rpool - - rpool.gather_pipes(workflow, config) - - workflow.run() - - cpac_dir = os.path.join(out_dir, f'pipeline_{orig_pipe_name}', - f'{subject_id}_{unique_id}') - cpac_dirs.append(os.path.join(cpac_dir, 'anat')) - - # Now we have all the anat_preproc set up for every session - # loop over the different anat preproc strategies - strats_brain_dct = {} - strats_head_dct = {} - for cpac_dir in cpac_dirs: - if os.path.isdir(cpac_dir): - for filename in os.listdir(cpac_dir): - if 'T1w.nii' in filename: - for tag in filename.split('_'): - if 'desc-' in tag and 'brain' in tag: - if tag not in strats_brain_dct: - strats_brain_dct[tag] = [] - strats_brain_dct[tag].append(os.path.join(cpac_dir, - filename)) - if tag not in strats_head_dct: - strats_head_dct[tag] = [] - head_file = filename.replace(tag, 'desc-reorient') - strats_head_dct[tag].append(os.path.join(cpac_dir, - head_file)) - - longitudinal_rpool: Optional[ResourcePool] = None - - for strat in strats_brain_dct.keys(): - - wf = initialize_nipype_wf(config, sub_list[0], - # just grab the first one for the name - name=f"template_node_{strat}") - - config.pipeline_setup[ - 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' - - template_node_name = f'longitudinal_anat_template_{strat}' - - # This node will generate the longitudinal template (the functions are - # in longitudinal_preproc) - # Later other algorithms could be added to calculate it, like the - # multivariate template from ANTS - # It would just require to change it here. - template_node = subject_specific_template( - workflow_name=template_node_name - ) - - template_node.inputs.set( - avg_method=config.longitudinal_template_generation[ - 'average_method'], - dof=config.longitudinal_template_generation['dof'], - interp=config.longitudinal_template_generation['interp'], - cost=config.longitudinal_template_generation['cost'], - convergence_threshold=config.longitudinal_template_generation[ - 'convergence_threshold'], - thread_pool=config.longitudinal_template_generation[ - 'thread_pool'], - unique_id_list=list(session_wfs.keys()) - ) - - template_node.inputs.input_brain_list = strats_brain_dct[strat] - template_node.inputs.input_skull_list = strats_head_dct[strat] - - long_id = f'longitudinal_{subject_id}_strat-{strat}' - - wf, rpool = initiate_rpool(wf, config, part_id=long_id) - - rpool.set_data("space-longitudinal_desc-brain_T1w", - template_node, 'brain_template', {}, - "", template_node_name) - - rpool.set_data("space-longitudinal_desc-brain_T1w-template", - template_node, 'brain_template', {}, - "", template_node_name) - - rpool.set_data("space-longitudinal_desc-reorient_T1w", - template_node, 'skull_template', {}, - "", template_node_name) - - rpool.set_data("space-longitudinal_desc-reorient_T1w-template", - template_node, 'skull_template', {}, - "", template_node_name) - - pipeline_blocks = [mask_longitudinal_T1w_brain] - - pipeline_blocks = build_T1w_registration_stack(rpool, config, - pipeline_blocks) - - pipeline_blocks = build_segmentation_stack(rpool, config, - pipeline_blocks) - - wf = connect_pipeline(wf, config, rpool, pipeline_blocks) - - excl = ['space-longitudinal_desc-brain_T1w', - 'space-longitudinal_desc-reorient_T1w', - 'space-longitudinal_desc-brain_mask'] - rpool.gather_pipes(wf, config, add_excl=excl) - - # this is going to run multiple times! - # once for every strategy! - wf.run() - - longitudinal_rpool = rpool.copy_rpool() - - # now, just write out a copy of the above to each session - config.pipeline_setup['pipeline_name'] = orig_pipe_name - for session in sub_list: - - unique_id = session['unique_id'] - - try: - creds_path = session['creds_path'] - if creds_path and 'none' not in creds_path.lower(): - if os.path.exists(creds_path): - input_creds_path = os.path.abspath(creds_path) - else: - err_msg = 'Credentials path: "%s" for subject "%s" ' \ - 'session "%s" was not found. Check this path ' \ - 'and try again.' % (creds_path, subject_id, - unique_id) - raise Exception(err_msg) - else: - input_creds_path = None - except KeyError: - input_creds_path = None - - wf = initialize_nipype_wf(config, sub_list[0]) - - wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) - - config.pipeline_setup[ - 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' - rpool = ingress_output_dir(config, rpool, long_id, - creds_path=input_creds_path) - - select_node_name = f'select_{unique_id}' - select_sess = pe.Node(Function(input_names=['session', - 'output_brains', - 'warps'], - output_names=['brain_path', - 'warp_path'], - function=select_session), - name=select_node_name) - select_sess.inputs.session = unique_id - - wf.connect(template_node, 'output_brain_list', select_sess, - 'output_brains') - wf.connect(template_node, 'warp_list', select_sess, 'warps') - - rpool.set_data("space-longitudinal_desc-brain_T1w", - select_sess, 'brain_path', {}, "", - select_node_name) - - rpool.set_data("from-T1w_to-longitudinal_mode-image_" - "desc-linear_xfm", - select_sess, 'warp_path', {}, "", - select_node_name) - - config.pipeline_setup['pipeline_name'] = orig_pipe_name - excl = ['space-template_desc-brain_T1w', - 'space-T1w_desc-brain_mask'] - - rpool.gather_pipes(wf, config, add_excl=excl) - wf.run() - longitudinal_rpool = rpool.copy_rpool() - - # begin single-session stuff again - for session in sub_list: - - unique_id = session['unique_id'] - - try: - creds_path = session['creds_path'] - if creds_path and 'none' not in creds_path.lower(): - if os.path.exists(creds_path): - input_creds_path = os.path.abspath(creds_path) - else: - err_msg = 'Credentials path: "%s" for subject "%s" ' \ - 'session "%s" was not found. Check this path ' \ - 'and try again.' % (creds_path, subject_id, - unique_id) - raise Exception(err_msg) - else: - input_creds_path = None - except KeyError: - input_creds_path = None - - wf = initialize_nipype_wf(config, sub_list[0]) - - wf, rpool = initiate_rpool(wf, config, session, rpool=longitudinal_rpool) - - pipeline_blocks = [warp_longitudinal_T1w_to_template, - warp_longitudinal_seg_to_T1w] - - wf = connect_pipeline(wf, config, rpool, pipeline_blocks) - - rpool.gather_pipes(wf, config) - - # this is going to run multiple times! - # once for every strategy! - wf.run() - - - - -# TODO check: -# 1 func alone works -# 2 anat + func works, pass anat strategy list? -def func_preproc_longitudinal_wf(subject_id, sub_list, config): - """ - Parameters - ---------- - subject_id : string - the id of the subject - sub_list : list of dict - this is a list of sessions for one subject and each session if the same dictionary as the one given to - prep_workflow - config : configuration - a configuration object containing the information of the pipeline config. (Same as for prep_workflow) - - Returns - ------- - strat_list_ses_list : list of list - a list of strategies; within each strategy, a list of sessions - """ - - datasink = pe.Node(nio.DataSink(), name='sinker') - datasink.inputs.base_directory = \ - config.pipeline_setup['working_directory']['path'] - - session_id_list = [] - ses_list_strat_list = {} - - workflow_name = 'func_preproc_longitudinal_' + str(subject_id) - workflow = pe.Workflow(name=workflow_name) - workflow.base_dir = config.pipeline_setup['working_directory']['path'] - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath( - config.pipeline_setup['crash_directory']['path']) - } - - for sub_dict in sub_list: - if 'func' in sub_dict or 'rest' in sub_dict: - if 'func' in sub_dict: - func_paths_dict = sub_dict['func'] - else: - func_paths_dict = sub_dict['rest'] - - unique_id = sub_dict['unique_id'] - session_id_list.append(unique_id) - - try: - creds_path = sub_dict['creds_path'] - if creds_path and 'none' not in creds_path.lower(): - if os.path.exists(creds_path): - input_creds_path = os.path.abspath(creds_path) - else: - err_msg = 'Credentials path: "%s" for subject "%s" was not ' \ - 'found. Check this path and try again.' % ( - creds_path, subject_id) - raise Exception(err_msg) - else: - input_creds_path = None - except KeyError: - input_creds_path = None - - strat = Strategy() - strat_list = [strat] - node_suffix = '_'.join([subject_id, unique_id]) - - # Functional Ingress Workflow - # add optional flag - workflow, diff, blip, fmap_rp_list = connect_func_ingress( - workflow, - strat_list, - config, - sub_dict, - subject_id, - input_creds_path, - node_suffix) - - # Functional Initial Prep Workflow - workflow, strat_list = connect_func_init(workflow, strat_list, - config, node_suffix) - - # Functional Image Preprocessing Workflow - workflow, strat_list = connect_func_preproc(workflow, strat_list, - config, node_suffix) - - # Distortion Correction - workflow, strat_list = connect_distortion_correction(workflow, - strat_list, - config, - diff, - blip, - fmap_rp_list, - node_suffix) - - ses_list_strat_list[node_suffix] = strat_list - - # Here we have all the func_preproc set up for every session of the subject - - # TODO create a list of list ses_list_strat_list - # a list of skullstripping strategies, - # a list of sessions within each strategy list - # TODO rename and reorganize dict - # TODO update strat name - strat_list_ses_list = {} - strat_list_ses_list['func_default'] = [] - - for sub_ses_id, strat_nodes_list in ses_list_strat_list.items(): - strat_list_ses_list['func_default'].append(strat_nodes_list[0]) - - workflow.run() - - return strat_list_ses_list - - -def merge_func_preproc(working_directory): - """ - Parameters - ---------- - working_directory : string - a path to the working directory - - Returns - ------- - brain_list : list - a list of func preprocessed brain - skull_list : list - a list of func preprocessed skull - """ - - brain_list = [] - skull_list = [] - - for dirpath, dirnames, filenames in os.walk(working_directory): - for f in filenames: - if 'func_get_preprocessed_median' in dirpath and '.nii.gz' in f: - filepath = os.path.join(dirpath, f) - brain_list.append(filepath) - if 'func_get_motion_correct_median' in dirpath and '.nii.gz' in f: - filepath = os.path.join(dirpath, f) - skull_list.append(filepath) - - brain_list.sort() - skull_list.sort() - - return brain_list, skull_list - - -def register_func_longitudinal_template_to_standard( - longitudinal_template_node, c, workflow, strat_init, strat_name): - sub_mem_gb, num_cores_per_sub, num_ants_cores, num_omp_cores = \ - check_config_resources(c) - - strat_init_new = strat_init.fork() - - strat_init_new.update_resource_pool({ - 'functional_preprocessed_median': ( - longitudinal_template_node, 'brain_template'), - 'motion_correct_median': ( - longitudinal_template_node, 'skull_template') - }) - - strat_list = [strat_init_new] - - new_strat_list = [] - - regOption = c.anatomical_preproc[ - 'registration_workflow' - ]['registration']['using'] - - if 'FSL' in regOption: - - for num_strat, strat in enumerate(strat_list): - - flirt_reg_func_mni = create_fsl_flirt_linear_reg( - 'func_mni_flirt_register_%s_%d' % (strat_name, num_strat) - ) - - if c.functional_registration['2-func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] not in ["trilinear", - "sinc", "spline"]: - err_msg = 'The selected FSL interpolation method may be in the list of values: "trilinear", "sinc", "spline"' - raise Exception(err_msg) - - # Input registration parameters - flirt_reg_func_mni.inputs.inputspec.interp = \ - c.functional_registration['2-func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] - - node, out_file = strat['functional_preprocessed_median'] - workflow.connect(node, out_file, - flirt_reg_func_mni, 'inputspec.input_brain') - - # pass the reference files - node, out_file = strat['template_brain_for_func_preproc'] - workflow.connect(node, out_file, flirt_reg_func_mni, - 'inputspec.reference_brain') - - if 'ANTS' in regOption: - strat = strat.fork() - new_strat_list.append(strat) - - strat.append_name(flirt_reg_func_mni.name) - - strat.update_resource_pool({ - 'registration_method': 'FSL', - 'func_longitudinal_to_mni_linear_xfm': ( - flirt_reg_func_mni, 'outputspec.linear_xfm'), - 'mni_to_func_longitudinal_linear_xfm': ( - flirt_reg_func_mni, 'outputspec.invlinear_xfm'), - 'func_longitudinal_template_to_standard': ( - flirt_reg_func_mni, 'outputspec.output_brain') - }) - - strat_list += new_strat_list - - new_strat_list = [] - - try: - fsl_linear_reg_only = c.fsl_linear_reg_only - except AttributeError: - fsl_linear_reg_only = [0] - - if 'FSL' in regOption and 0 in fsl_linear_reg_only: - - for num_strat, strat in enumerate(strat_list): - - if strat.get('registration_method') == 'FSL': - - fnirt_reg_func_mni = create_fsl_fnirt_nonlinear_reg( - 'func_mni_fnirt_register_%s_%d' % (strat_name, num_strat) - ) - - # brain input - node, out_file = strat['functional_preprocessed_median'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, 'inputspec.input_brain') - - # brain reference - node, out_file = strat['template_brain_for_func_preproc'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, - 'inputspec.reference_brain') - - # skull input - node, out_file = strat['motion_correct_median'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, 'inputspec.input_skull') - - # skull reference - node, out_file = strat['template_skull_for_func_preproc'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, - 'inputspec.reference_skull') - - node, out_file = strat['func_longitudinal_to_mni_linear_xfm'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, 'inputspec.linear_aff') - - node, out_file = strat['template_ref_mask'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, 'inputspec.ref_mask') - - # assign the FSL FNIRT config file specified in pipeline - # config.yml - fnirt_reg_func_mni.inputs.inputspec.fnirt_config = \ - c.anatomical_preproc['registration_workflow']['registration'][ - 'FSL-FNIRT']['fnirt_config'] - - if 1 in fsl_linear_reg_only: - strat = strat.fork() - new_strat_list.append(strat) - - strat.append_name(fnirt_reg_func_mni.name) - - strat.update_resource_pool({ - 'func_longitudinal_to_mni_nonlinear_xfm': ( - fnirt_reg_func_mni, 'outputspec.nonlinear_xfm'), - 'func_longitudinal_template_to_standard': ( - fnirt_reg_func_mni, 'outputspec.output_brain') - }, override=True) - - strat_list += new_strat_list - - new_strat_list = [] - - for num_strat, strat in enumerate(strat_list): - - # or run ANTS anatomical-to-MNI registration instead - if 'ANTS' in regOption and \ - strat.get('registration_method') != 'FSL': - - ants_reg_func_mni = \ - create_wf_calculate_ants_warp( - 'func_mni_ants_register_%s_%d' % (strat_name, num_strat), - num_threads=num_ants_cores, - reg_ants_skull= - c.anatomical_preproc['registration_workflow'][ - 'reg_with_skull'] - ) - - if c.functional_registration['2-func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] not in ['Linear', - 'BSpline', - 'LanczosWindowedSinc']: - err_msg = 'The selected ANTS interpolation method may be in the list of values: "Linear", "BSpline", "LanczosWindowedSinc"' - raise Exception(err_msg) - - # Input registration parameters - ants_reg_func_mni.inputs.inputspec.interp = \ - c.functional_registration['2-func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - - # calculating the transform with the skullstripped is - # reported to be better, but it requires very high - # quality skullstripping. If skullstripping is imprecise - # registration with skull is preferred - if c.anatomical_preproc['registration_workflow'][ - 'reg_with_skull']: - - # get the skull-stripped anatomical from resource pool - node, out_file = strat['functional_preprocessed_median'] - - # pass the anatomical to the workflow - workflow.connect(node, out_file, - ants_reg_func_mni, 'inputspec.moving_brain') - - # get the reorient skull-on anatomical from resource pool - node, out_file = strat['motion_correct_median'] - - # pass the anatomical to the workflow - workflow.connect(node, out_file, - ants_reg_func_mni, 'inputspec.moving_skull') - - # pass the reference file - node, out_file = strat['template_brain_for_func_preproc'] - workflow.connect(node, out_file, - ants_reg_func_mni, - 'inputspec.reference_brain') - - # pass the reference file - node, out_file = strat['template_skull_for_func_preproc'] - workflow.connect(node, out_file, - ants_reg_func_mni, - 'inputspec.reference_skull') - - else: - - node, out_file = strat['functional_preprocessed_median'] - - workflow.connect(node, out_file, - ants_reg_func_mni, 'inputspec.moving_brain') - - # pass the reference file - node, out_file = strat['template_brain_for_func_preproc'] - workflow.connect(node, out_file, - ants_reg_func_mni, - 'inputspec.reference_brain') - - # pass the reference mask file - node, out_file = strat['template_brain_mask_for_func_preproc'] - workflow.connect( - node, out_file, - ants_reg_func_mni, 'inputspec.reference_mask' - ) - - # pass the reference mask file - node, out_file = strat['functional_brain_mask'] - workflow.connect( - node, out_file, - ants_reg_func_mni, 'inputspec.moving_mask' - ) - - ants_reg_func_mni.inputs.inputspec.ants_para = \ - c.anatomical_preproc['registration_workflow']['registration'][ - 'ANTs']['T1_registration'] - ants_reg_func_mni.inputs.inputspec.fixed_image_mask = None - - strat.append_name(ants_reg_func_mni.name) - - strat.update_resource_pool({ - 'registration_method': 'ANTS', - 'ants_initial_xfm': ( - ants_reg_func_mni, 'outputspec.ants_initial_xfm'), - 'ants_rigid_xfm': ( - ants_reg_func_mni, 'outputspec.ants_rigid_xfm'), - 'ants_affine_xfm': ( - ants_reg_func_mni, 'outputspec.ants_affine_xfm'), - 'func_longitudinal_to_mni_nonlinear_xfm': ( - ants_reg_func_mni, 'outputspec.warp_field'), - 'mni_to_func_longitudinal_nonlinear_xfm': ( - ants_reg_func_mni, 'outputspec.inverse_warp_field'), - 'func_longitudinal_to_mni_ants_composite_xfm': ( - ants_reg_func_mni, 'outputspec.composite_transform'), - 'func_longitudinal_template_to_standard': ( - ants_reg_func_mni, 'outputspec.normalized_output_brain') - }) - - strat_list += new_strat_list - - ''' - # Func -> T1 Registration (Initial Linear Reg) - workflow, strat_list, diff_complete = connect_func_to_anat_init_reg(workflow, strat_list, c) - - # Func -> T1 Registration (BBREG) - workflow, strat_list = connect_func_to_anat_bbreg(workflow, strat_list, c, diff_complete) - - # Func -> T1/EPI Template - workflow, strat_list = connect_func_to_template_reg(workflow, strat_list, c) - ''' - - return workflow, strat_list - - -def func_longitudinal_template_wf(subject_id, strat_list, config): - ''' - Parameters - ---------- - subject_id : string - the id of the subject - strat_list : list of list - first level strategy, second level session - config : configuration - a configuration object containing the information of the pipeline config. - - Returns - ------- - None - ''' - - workflow_name = 'func_longitudinal_template_' + str(subject_id) - workflow = pe.Workflow(name=workflow_name) - workflow.base_dir = config.pipeline_setup['working_directory']['path'] - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath( - config.pipeline_setup['crash_directory']['path']) - } - - # strat_nodes_list = strat_list['func_default'] - strat_init = Strategy() - - templates_for_resampling = [ - (config.resolution_for_func_preproc, - config.template_brain_only_for_func, - 'template_brain_for_func_preproc', 'resolution_for_func_preproc'), - (config.resolution_for_func_preproc, config.template_skull_for_func, - 'template_skull_for_func_preproc', 'resolution_for_func_preproc'), - (config.resolution_for_func_preproc, config.ref_mask_for_func, - 'template_ref_mask', 'resolution_for_func_preproc'), - # TODO check float resolution - (config.resolution_for_func_preproc, - config.functional_registration['2-func_registration_to_template'][ - 'target_template']['EPI_template']['template_epi'], - 'template_epi', 'resolution_for_func_preproc'), - (config.resolution_for_func_derivative, - config.functional_registration['2-func_registration_to_template'][ - 'target_template']['EPI_template']['template_epi'], - 'template_epi_derivative', 'resolution_for_func_derivative'), - (config.resolution_for_func_derivative, - config.template_brain_only_for_func, - 'template_brain_for_func_derivative', 'resolution_for_func_preproc'), - ( - config.resolution_for_func_derivative, config.template_skull_for_func, - 'template_skull_for_func_derivative', 'resolution_for_func_preproc'), - ] - - for resolution, template, template_name, tag in templates_for_resampling: - resampled_template = pe.Node(Function( - input_names=['resolution', 'template', 'template_name', 'tag'], - output_names=['resampled_template'], - function=resolve_resolution, - as_module=True), - name='resampled_' + template_name) - - resampled_template.inputs.resolution = resolution - resampled_template.inputs.template = template - resampled_template.inputs.template_name = template_name - resampled_template.inputs.tag = tag - - strat_init.update_resource_pool({ - template_name: (resampled_template, 'resampled_template') - }) - - merge_func_preproc_node = pe.Node( - Function(input_names=['working_directory'], - output_names=['brain_list', 'skull_list'], - function=merge_func_preproc, - as_module=True), - name='merge_func_preproc') - - merge_func_preproc_node.inputs.working_directory = \ - config.pipeline_setup['working_directory']['path'] - - template_node = subject_specific_template( - workflow_name='subject_specific_func_template_' + subject_id - ) - - template_node.inputs.set( - avg_method=config.longitudinal_template_average_method, - dof=config.longitudinal_template_dof, - interp=config.longitudinal_template_interp, - cost=config.longitudinal_template_cost, - convergence_threshold=config.longitudinal_template_convergence_threshold, - thread_pool=config.longitudinal_template_thread_pool, - ) - - workflow.connect(merge_func_preproc_node, 'brain_list', - template_node, 'input_brain_list') - - workflow.connect(merge_func_preproc_node, 'skull_list', - template_node, 'input_skull_list') - - workflow, strat_list = register_func_longitudinal_template_to_standard( - template_node, - config, - workflow, - strat_init, - 'default' - ) - - workflow.run() - - return diff --git a/CPAC/pipeline/cpac_runner.py b/CPAC/pipeline/cpac_runner.py index 30d6b0fc18..58bbd5d578 100644 --- a/CPAC/pipeline/cpac_runner.py +++ b/CPAC/pipeline/cpac_runner.py @@ -191,7 +191,7 @@ def run_cpac_on_cluster(config_file, subject_list_file, f.write(pid) -def run_T1w_longitudinal(sublist, cfg): +def run_T1w_longitudinal(sublist, cfg, dry_run: bool = False): subject_id_dict = {} for sub in sublist: @@ -206,7 +206,7 @@ def run_T1w_longitudinal(sublist, cfg): for subject_id, sub_list in subject_id_dict.items(): if len(sub_list) > 1: valid_longitudinal_data = True - anat_longitudinal_wf(subject_id, sub_list, cfg) + anat_longitudinal_wf(subject_id, sub_list, cfg, dry_run=dry_run) elif len(sub_list) == 1: warnings.warn("\n\nThere is only one anatomical session " "for sub-%s. Longitudinal preprocessing " @@ -422,7 +422,7 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, if hasattr(c, 'longitudinal_template_generation') and \ c.longitudinal_template_generation['run']: - run_T1w_longitudinal(sublist, c) + run_T1w_longitudinal(sublist, c, dry_run=test_config) # TODO functional longitudinal pipeline ''' diff --git a/CPAC/pipeline/engine.py b/CPAC/pipeline/engine.py index 42b5d46468..edc9fd2879 100644 --- a/CPAC/pipeline/engine.py +++ b/CPAC/pipeline/engine.py @@ -1115,7 +1115,10 @@ def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None): unlabelled.remove(key) # del all_forks for pipe_idx in self.rpool[resource]: - pipe_x = self.get_pipe_number(pipe_idx) + try: + pipe_x = self.get_pipe_number(pipe_idx) + except ValueError: + continue json_info = self.rpool[resource][pipe_idx]['json'] out_dct = self.rpool[resource][pipe_idx]['out'] diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index 957440dc45..0cd748a0a7 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -3616,7 +3616,7 @@ def warp_mask_to_template_fxn(wf, cfg, strat_pool, pipe_num, opt=None): return wf, outputs return warp_mask_to_template_fxn -warp_mask_to_template = {"T1": nodeblock( +warp_mask_to_template = {"T1w": nodeblock( name="transform_T1mask_to_T1template", switch=[ ["registration_workflows", "anatomical_registration", "run"], @@ -3628,7 +3628,7 @@ def warp_mask_to_template_fxn(wf, cfg, strat_pool, pipe_num, opt=None): "T1w-template", ], outputs={"space-template_desc-brain_mask": {"Template": "T1w-template"}}, -)(_warp_mask_to_template("T1")), "longitudinal": nodeblock( +)(_warp_mask_to_template("T1w")), "longitudinal": nodeblock( name="transform_longitudinal_mask_to_T1template", switch=[ ["registration_workflows", "anatomical_registration", "run"], From 8666475829ed6b5f8dc2fae690f149c34647bd07 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Thu, 31 Oct 2024 13:09:27 -0400 Subject: [PATCH 19/33] :bug: Clone instead of rename --- CPAC/longitudinal_pipeline/longitudinal_workflow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 83bfcc9a63..f86be0310c 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -499,7 +499,9 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur # Rename nodes to include session name to avoid duplicates for key in strats_dct: for i, resource in enumerate(strats_dct[key]): - resource[0].name = f"{resource[0].name}_{session_id_list[i]}" + resource = ( + resource[0].clone(f"{resource[0].name}_{session_id_list[i]}"), + resource[1]) wf = initialize_nipype_wf(config, sub_list[0], # just grab the first one for the name From 024f28fc28a252e9afa8ecb82f8cb2f677768e71 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 17 Sep 2024 14:19:01 -0400 Subject: [PATCH 20/33] :bug: Fix header reading for bandpass filters --- CPAC/nuisance/bandpass.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CPAC/nuisance/bandpass.py b/CPAC/nuisance/bandpass.py index 91c72124c8..b5d2148fa8 100644 --- a/CPAC/nuisance/bandpass.py +++ b/CPAC/nuisance/bandpass.py @@ -110,16 +110,16 @@ def bandpass_voxels(realigned_file, regressor_file, bandpass_freqs, img.to_filename(regressor_bandpassed_file) else: - with open(regressor_file, 'r') as f: - header = [] - - # header wouldn't be longer than 5, right? I don't want to - # loop over the whole file - for i in range(5): - line = f.readline() - if line.startswith('#') or isinstance(line[0], str): + header = [] + with open(regressor_file, "r") as _f: + # Each leading line that doesn't start with a number goes into the header + for line in _f.readlines(): + try: + float(line.split()[0]) + break + except ValueError: header.append(line) - + # usecols=[list] regressor = np.loadtxt(regressor_file, skiprows=len(header)) Yc = regressor - np.tile(regressor.mean(0), (regressor.shape[0], 1)) From 87dd957a66c32b23a913d814a8c238f64397b9dd Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Thu, 31 Oct 2024 13:45:49 -0400 Subject: [PATCH 21/33] :memo: Add bandpass filter fix to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d2d06f59f..43dfc8e019 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A bug in which bandpass filters always assumed 1D regressor files have exactly 5 header rows. - Graph-building bugs that prevented longitudinal workflows from running. ## [1.8.7] - 2024-05-03 From 99adf4b3f4ed100c62bd454940101def0150b422 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 1 Nov 2024 11:06:30 -0400 Subject: [PATCH 22/33] :construction_worker: Skip `update-yaml-comments` job in pre-commit CI --- .github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile | 2 +- .../Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile | 2 +- .github/Dockerfiles/C-PAC.develop-jammy.Dockerfile | 2 +- .github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile | 2 +- .github/Dockerfiles/base-standard.Dockerfile | 2 +- .pre-commit-config.yaml | 4 +++- Dockerfile | 2 +- variant-ABCD-HCP.Dockerfile | 2 +- variant-fMRIPrep-LTS.Dockerfile | 2 +- variant-lite.Dockerfile | 2 +- 10 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile b/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile index 2a165f301d..27631dacd5 100644 --- a/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile +++ b/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [ABCD-HCP BIDS fMRI Pipeline](https://github.com/DCAN-Labs/abcd-hcp-pipeline/blob/e480a8f99534f1b05f37bf44c64827384b69b383/Dockerfile)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile b/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile index 096d3e2b0b..3071f2de78 100644 --- a/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile +++ b/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [fMRIPrep LTS](https://reproducibility.stanford.edu/fmriprep-lts#long-term-support-lts)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile b/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile index 03f04e7cea..6de7379c1d 100644 --- a/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile +++ b/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -FROM ghcr.io/fcp-indi/c-pac/stage-base:standard-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:standard-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "Full C-PAC image" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile b/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile index 4a186031f0..98d4ed9f8f 100644 --- a/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile +++ b/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "Full C-PAC image without FreeSurfer" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/.github/Dockerfiles/base-standard.Dockerfile b/.github/Dockerfiles/base-standard.Dockerfile index aa8f638dbc..2a448e1186 100644 --- a/.github/Dockerfiles/base-standard.Dockerfile +++ b/.github/Dockerfiles/base-standard.Dockerfile @@ -16,7 +16,7 @@ # License along with C-PAC. If not, see . FROM ghcr.io/fcp-indi/c-pac/freesurfer:6.0.0-min.neurodocker-jammy as FreeSurfer -FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \ Standard software dependencies for C-PAC standard images" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7357d71417..442a90339f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,9 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . - +ci: + skip: + - update-yaml-comments repos: - repo: local hooks: diff --git a/Dockerfile b/Dockerfile index 03f04e7cea..6de7379c1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -FROM ghcr.io/fcp-indi/c-pac/stage-base:standard-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:standard-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "Full C-PAC image" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/variant-ABCD-HCP.Dockerfile b/variant-ABCD-HCP.Dockerfile index 2a165f301d..27631dacd5 100644 --- a/variant-ABCD-HCP.Dockerfile +++ b/variant-ABCD-HCP.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [ABCD-HCP BIDS fMRI Pipeline](https://github.com/DCAN-Labs/abcd-hcp-pipeline/blob/e480a8f99534f1b05f37bf44c64827384b69b383/Dockerfile)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/variant-fMRIPrep-LTS.Dockerfile b/variant-fMRIPrep-LTS.Dockerfile index 096d3e2b0b..3071f2de78 100644 --- a/variant-fMRIPrep-LTS.Dockerfile +++ b/variant-fMRIPrep-LTS.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [fMRIPrep LTS](https://reproducibility.stanford.edu/fmriprep-lts#long-term-support-lts)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root diff --git a/variant-lite.Dockerfile b/variant-lite.Dockerfile index 4a186031f0..98d4ed9f8f 100644 --- a/variant-lite.Dockerfile +++ b/variant-lite.Dockerfile @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev2 +FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.7.post1.dev3 LABEL org.opencontainers.image.description "Full C-PAC image without FreeSurfer" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC USER root From 161b40d5656468db40b8c93c30fbf84fe3bada91 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 1 Nov 2024 11:07:59 -0400 Subject: [PATCH 23/33] :construction_worker: Merge updated `.github/scripts/autoversioning.sh` from `develop` --- .github/scripts/autoversioning.sh | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/scripts/autoversioning.sh b/.github/scripts/autoversioning.sh index 0543f626a1..f93dc3f57e 100755 --- a/.github/scripts/autoversioning.sh +++ b/.github/scripts/autoversioning.sh @@ -18,7 +18,14 @@ # License along with C-PAC. If not, see . # Update version comment strings -cd CPAC +function wait_for_git_lock() { + while [ -f "./.git/index.lock" ]; do + echo "Waiting for the git lock file to be removed..." + sleep 1 + done +} + +cd CPAC || exit 1 VERSION=$(python -c "from info import __version__; print(('.'.join(('.'.join(__version__[::-1].split('-')[1].split('.')[1:])[::-1], __version__.split('-')[1])) if '-' in __version__ else __version__).split('+', 1)[0])") cd .. echo "v${VERSION}" > version @@ -30,8 +37,8 @@ else # Linux and others find ./CPAC/resources/configs -name "*.yml" -exec sed -i'' -r "${_SED_COMMAND}" {} \; fi -git add version -VERSIONS=( `git show $(git log --pretty=format:'%h' -n 2 version | tail -n 1):version` `cat version` ) +wait_for_git_lock && git add version +VERSIONS=( `git show $(git log --pretty=format:'%h' -n 1 version | tail -n 1):version` `cat version` ) export PATTERN="(declare|typeset) -a" if [[ "$(declare -p VERSIONS)" =~ $PATTERN ]] then @@ -52,11 +59,12 @@ then done unset IFS fi -git add CPAC/resources/configs .github/Dockerfiles +wait_for_git_lock && git add CPAC/resources/configs .github/Dockerfiles # Overwrite top-level Dockerfiles with the CI Dockerfiles -cp .github/Dockerfiles/C-PAC.develop-jammy.Dockerfile Dockerfile -cp .github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile variant-ABCD-HCP.Dockerfile -cp .github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile variant-fMRIPrep-LTS.Dockerfile -cp .github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile variant-lite.Dockerfile -git add *Dockerfile \ No newline at end of file +wait_for_git_lock && cp .github/Dockerfiles/C-PAC.develop-jammy.Dockerfile Dockerfile +wait_for_git_lock && cp .github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile variant-lite.Dockerfile +for DOCKERFILE in $(ls *Dockerfile) +do + wait_for_git_lock && git add $DOCKERFILE +done From e916d3361fdcaf5cd46262690e00856fbd649f81 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 1 Nov 2024 12:02:00 -0400 Subject: [PATCH 24/33] :recycle: SSOT `warp_mask_to_template` decorator --- CPAC/pipeline/cpac_pipeline.py | 6 ++-- CPAC/registration/registration.py | 48 ++++++++++++------------------- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/CPAC/pipeline/cpac_pipeline.py b/CPAC/pipeline/cpac_pipeline.py index ea3adac14e..89d49b8d6f 100644 --- a/CPAC/pipeline/cpac_pipeline.py +++ b/CPAC/pipeline/cpac_pipeline.py @@ -17,6 +17,7 @@ import os import sys import time +from typing import Literal import csv import shutil import pickle @@ -1033,7 +1034,8 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None): return pipeline_blocks -def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None, space="T1w"): +def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None, + space: Literal["longitudinal", "T1w"] = "T1w"): if not pipeline_blocks: pipeline_blocks = [] @@ -1044,7 +1046,7 @@ def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None, space="T1w"): [register_ANTs_anat_to_template, register_FSL_anat_to_template], overwrite_transform_anat_to_template, warp_wholeheadT1_to_template, - warp_mask_to_template[space] + warp_mask_to_template(space) ] if not rpool.check_rpool('desc-restore-brain_T1w'): diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index 0cd748a0a7..65e8a1ac1b 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -16,9 +16,9 @@ # License along with C-PAC. If not, see . # pylint: disable=too-many-lines,ungrouped-imports,wrong-import-order from copy import deepcopy -from typing import Optional +from typing import Literal, Optional from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.pipeline.nodeblock import nodeblock +from CPAC.pipeline.nodeblock import nodeblock, NodeBlockFunction from nipype.interfaces import afni, ants, c3, fsl, utility as util from nipype.interfaces.afni import utils as afni_utils @@ -3572,8 +3572,24 @@ def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): return (wf, outputs) -def _warp_mask_to_template(space): +def warp_mask_to_template(space: Literal["longitudinal", "T1w"]) -> NodeBlockFunction: + """Get a NodeBlockFunction to transform a mask from ``space`` to template.""" + @nodeblock( + name=f"transform_{space}-mask_to_T1-template", + switch=[ + ["registration_workflows", "anatomical_registration", "run"], + ["anatomical_preproc", "run"], + ["anatomical_preproc", "brain_extraction", "run"], + ], + inputs=[ + (f"space-{space}_desc-brain_mask", + f"from-{space}_to-template_mode-image_xfm"), + "T1w-template", + ], + outputs={"space-template_desc-brain_mask": {"Template": "T1w-template"}}, + ) def warp_mask_to_template_fxn(wf, cfg, strat_pool, pipe_num, opt=None): + """Transform a mask to template space.""" xfm_prov = strat_pool.get_cpac_provenance( f'from-{space}_to-template_mode-image_xfm') @@ -3616,32 +3632,6 @@ def warp_mask_to_template_fxn(wf, cfg, strat_pool, pipe_num, opt=None): return wf, outputs return warp_mask_to_template_fxn -warp_mask_to_template = {"T1w": nodeblock( - name="transform_T1mask_to_T1template", - switch=[ - ["registration_workflows", "anatomical_registration", "run"], - ["anatomical_preproc", "run"], - ["anatomical_preproc", "brain_extraction", "run"], - ], - inputs=[ - ("space-T1w_desc-brain_mask", "from-T1w_to-template_mode-image_xfm"), - "T1w-template", - ], - outputs={"space-template_desc-brain_mask": {"Template": "T1w-template"}}, -)(_warp_mask_to_template("T1w")), "longitudinal": nodeblock( - name="transform_longitudinal_mask_to_T1template", - switch=[ - ["registration_workflows", "anatomical_registration", "run"], - ["anatomical_preproc", "run"], - ["anatomical_preproc", "brain_extraction", "run"], - ], - inputs=[ - ("space-longitudinal_desc-brain_mask", "from-longitudinal_to-template_mode-image_xfm"), - "T1w-template", - ], - outputs={"space-template_desc-brain_mask": {"Template": "T1w-template"}}, -)(_warp_mask_to_template("longitudinal"))} - @nodeblock( name="transform_timeseries_to_T1template", From 16cb70c7d89badcb5a2d93a493c896627a84ee09 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 1 Nov 2024 12:09:45 -0400 Subject: [PATCH 25/33] :construction: Restore `template_node` connections --- CPAC/longitudinal_pipeline/longitudinal_workflow.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index f86be0310c..081374f168 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -1188,10 +1188,10 @@ def func_longitudinal_template_wf(subject_id, strat_list, config): thread_pool=config.longitudinal_template_thread_pool, ) - # workflow.connect(merge_func_preproc_node, 'brain_list', - # template_node, 'input_brain_list') - # workflow.connect(merge_func_preproc_node, 'skull_list', - # template_node, 'input_skull_list') + workflow.connect(merge_func_preproc_node, 'brain_list', + template_node, 'input_brain_list') + workflow.connect(merge_func_preproc_node, 'skull_list', + template_node, 'input_skull_list') workflow, strat_list = register_func_longitudinal_template_to_standard( template_node, From 4679ef4aa9df6a4c49b9bba917b9345b3d6e7b6a Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 1 Nov 2024 12:14:57 -0400 Subject: [PATCH 26/33] :coffin: Remove temporary debugging code --- CPAC/pipeline/engine.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/CPAC/pipeline/engine.py b/CPAC/pipeline/engine.py index edc9fd2879..1f697e0667 100644 --- a/CPAC/pipeline/engine.py +++ b/CPAC/pipeline/engine.py @@ -63,22 +63,6 @@ class ResourcePool: - def pickle(self, name: Optional[str] = None) -> None: - """Pickle rpool.""" - if name: - filename: str = f"{name}.pickle" - elif hasattr(self, "name") and isinstance(self.name, str): - filename: str = f"{self.name.replace(' ', '')}.pickle" - else: - existing_names: list[str] = [path.name[:-7] for path in Path().iterdir() if path.name.endswith(".pickle")] - i: int = 0 - while str(i) in existing_names: - i += 1 - filename = f"{i}.pickle" - with open(filename, "wb") as _f: - pickle.dump(self, _f) - - def __init__(self, rpool=None, name=None, cfg=None, pipe_list=None): if not rpool: @@ -573,9 +557,6 @@ def get_strats(self, resources, debug=False): total_pool.append(sub_pool) if not total_pool: - print(self.rpool.keys()) - with open("rpool_no_total_pool.pickle", "wb") as _f: - pickle.dump(self, _f) raise LookupError('\n\n[!] C-PAC says: None of the listed ' 'resources in the node block being connected ' 'exist in the resource pool.\n\nResources:\n' From 6eef5af2c2568852272ca3ea81f3280650bca1ee Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 1 Nov 2024 12:20:44 -0400 Subject: [PATCH 27/33] :coffin: Remove commented-out code --- CPAC/registration/registration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index 65e8a1ac1b..fd32a340c8 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -2372,7 +2372,6 @@ def register_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): new_key = key.replace(f'{direction}-T1w', f'{direction}-longitudinal') outputs[new_key] = outputs[key] - # del outputs[key] return (wf, outputs) From 9169203b77da9d958f4357633ff488e2903586b1 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 1 Nov 2024 12:23:52 -0400 Subject: [PATCH 28/33] :rewind: Revert f2da817f5 --- CPAC/registration/registration.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index fd32a340c8..6bed6b0b14 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -3530,11 +3530,8 @@ def apply_blip_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): xfm: list[str] = ["from-T1w_to-template_mode-image_xfm", "from-longitudinal_to-template_mode-image_xfm"] - try: - xfm_prov = strat_pool.get_cpac_provenance(xfm) - reg_tool = check_prov_for_regtool(xfm_prov) - except KeyError: - reg_tool = cfg["registration_workflows", "anatomical_registration", "registration", "using"] + xfm_prov = strat_pool.get_cpac_provenance(xfm) + reg_tool = check_prov_for_regtool(xfm_prov) num_cpus = cfg.pipeline_setup['system_config'][ 'max_cores_per_participant'] From e7e82b75bb8bec20d59f2c9d07bd19ee85dd97ed Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 1 Nov 2024 13:04:59 -0400 Subject: [PATCH 29/33] :recycle: SSOT `warp_to_template` --- .../longitudinal_workflow.py | 2 +- CPAC/pipeline/cpac_pipeline.py | 7 +- CPAC/registration/registration.py | 146 +++++++----------- CPAC/registration/utils.py | 33 ++++ 4 files changed, 97 insertions(+), 91 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 081374f168..67b52c975c 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -499,7 +499,7 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur # Rename nodes to include session name to avoid duplicates for key in strats_dct: for i, resource in enumerate(strats_dct[key]): - resource = ( + strats_dct[key][i] = ( resource[0].clone(f"{resource[0].name}_{session_id_list[i]}"), resource[1]) diff --git a/CPAC/pipeline/cpac_pipeline.py b/CPAC/pipeline/cpac_pipeline.py index 89d49b8d6f..a9df746179 100644 --- a/CPAC/pipeline/cpac_pipeline.py +++ b/CPAC/pipeline/cpac_pipeline.py @@ -98,8 +98,7 @@ coregistration, create_func_to_T1template_xfm, create_func_to_T1template_symmetric_xfm, - warp_wholeheadT1_to_template, - warp_mask_to_template, + warp_to_template, apply_phasediff_to_timeseries_separately, apply_blip_to_timeseries_separately, warp_timeseries_to_T1template, @@ -1045,8 +1044,8 @@ def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None, reg_blocks = [ [register_ANTs_anat_to_template, register_FSL_anat_to_template], overwrite_transform_anat_to_template, - warp_wholeheadT1_to_template, - warp_mask_to_template(space) + warp_to_template("wholehead", space), + warp_to_template("mask", space) ] if not rpool.check_rpool('desc-restore-brain_T1w'): diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index 6bed6b0b14..a73ff840b2 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -33,7 +33,8 @@ hardcoded_reg, \ one_d_to_mat, \ run_c3d, \ - run_c4d + run_c4d, \ + prepend_space from CPAC.utils.interfaces.fsl import Merge as fslMerge from CPAC.utils.typing import LIST_OR_STR, TUPLE from CPAC.utils.utils import check_prov_for_motion_tool, check_prov_for_regtool @@ -3512,83 +3513,53 @@ def apply_blip_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, return (wf, outputs) -@nodeblock( - name="transform_whole_head_T1w_to_T1template", - config=["registration_workflows", "anatomical_registration"], - switch=["run"], - inputs=[ - ( - ["desc-head_T1w", "space-longitudinal_desc-reorient_T1w"], - ["from-T1w_to-template_mode-image_xfm", - "from-longitudinal_to-template_mode-image_xfm"], - "space-template_desc-head_T1w", - ), - "T1w-template", - ], - outputs={"space-template_desc-head_T1w": {"Template": "T1w-template"}}, -) -def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - xfm: list[str] = ["from-T1w_to-template_mode-image_xfm", - "from-longitudinal_to-template_mode-image_xfm"] - xfm_prov = strat_pool.get_cpac_provenance(xfm) - reg_tool = check_prov_for_regtool(xfm_prov) - - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] - - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] - - apply_xfm = apply_transform(f'warp_wholehead_T1w_to_T1template_{pipe_num}', - reg_tool, time_series=False, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) - - if reg_tool == 'ants': - apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': - apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] - - connect = strat_pool.get_data(["desc-head_T1w", - "space-longitudinal_desc-reorient_T1w"]) - node, out = connect - wf.connect(node, out, apply_xfm, 'inputspec.input_image') - - node, out = strat_pool.get_data("T1w-template") - wf.connect(node, out, apply_xfm, 'inputspec.reference') - - node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') - - outputs = { - 'space-template_desc-head_T1w': (apply_xfm, 'outputspec.output_image') - } +def warp_to_template(warp_what: Literal["mask", "wholehead"], + space_from: Literal["longitudinal", "T1w"]) -> NodeBlockFunction: + """Get a NodeBlockFunction to transform a resource from ``space`` to template. - return (wf, outputs) - -def warp_mask_to_template(space: Literal["longitudinal", "T1w"]) -> NodeBlockFunction: - """Get a NodeBlockFunction to transform a mask from ``space`` to template.""" - @nodeblock( - name=f"transform_{space}-mask_to_T1-template", - switch=[ + The resource being warped needs to be the first list or string in the tuple + in the first position of the decorator's "inputs". + """ + _decorators = {"mask": { + "name": f"transform_{space_from}-mask_to_T1-template", + "switch": [ ["registration_workflows", "anatomical_registration", "run"], ["anatomical_preproc", "run"], ["anatomical_preproc", "brain_extraction", "run"], ], - inputs=[ - (f"space-{space}_desc-brain_mask", - f"from-{space}_to-template_mode-image_xfm"), + "inputs": [ + (f"space-{space_from}_desc-brain_mask", + f"from-{space_from}_to-template_mode-image_xfm"), "T1w-template", ], - outputs={"space-template_desc-brain_mask": {"Template": "T1w-template"}}, - ) - def warp_mask_to_template_fxn(wf, cfg, strat_pool, pipe_num, opt=None): - """Transform a mask to template space.""" + "outputs": {"space-template_desc-brain_mask": {"Template": "T1w-template"}}, + }, "wholehead": { + "name": f"transform_wholehead_{space_from}_to_T1template", + "config": ["registration_workflows", "anatomical_registration"], + "switch": ["run"], + "inputs": [ + ( + ["desc-head_T1w", "desc-reorient_T1w"], + [f"from-{space_from}_to-template_mode-image_xfm", + f"from-{space_from}_to-template_mode-image_xfm"], + "space-template_desc-head_T1w", + ), + "T1w-template", + ], + "outputs": {"space-template_desc-head_T1w": {"Template": "T1w-template"}}, + }} + if space_from != "T1w": + _decorators[warp_what]["inputs"][0] = tuple((prepend_space( + _decorators[warp_what]["inputs"][0][0], space_from), + *_decorators[warp_what]["inputs"][0][1:] + )) + + @nodeblock(**_decorators[warp_what]) + def warp_to_template_fxn(wf, cfg, strat_pool, pipe_num, opt=None): + """Transform a resource to template space.""" xfm_prov = strat_pool.get_cpac_provenance( - f'from-{space}_to-template_mode-image_xfm') + f'from-{space_from}_to-template_mode-image_xfm') reg_tool = check_prov_for_regtool(xfm_prov) num_cpus = cfg.pipeline_setup['system_config'][ @@ -3596,37 +3567,40 @@ def warp_mask_to_template_fxn(wf, cfg, strat_pool, pipe_num, opt=None): num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] - apply_xfm = apply_transform(f'warp_T1mask_to_T1template_{pipe_num}', - reg_tool, time_series=False, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f'warp_{space_from}{warp_what}_to_T1template_{pipe_num}', + reg_tool, time_series=False, num_cpus=num_cpus, + num_ants_cores=num_ants_cores) - apply_xfm.inputs.inputspec.interpolation = "NearestNeighbor" - ''' - if reg_tool == 'ants': - apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': + if warp_what == "mask": + apply_xfm.inputs.inputspec.interpolation = "NearestNeighbor" + else: + tool = "ANTs" if reg_tool == 'ants' else 'FNIRT' if reg_tool == 'fsl' else None + if not tool: + msg = f"Warp {warp_what} to template not implemented for {reg_tool}." + raise NotImplementedError(msg) apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] - ''' - connect = strat_pool.get_data(f"space-{space}_desc-brain_mask") - node, out = connect + f'{tool}_pipelines']['interpolation'] + + # the resource being warped needs to be inputs[0][0] for this + node, out = strat_pool.get_data(_decorators[warp_what]["inputs"][0][0]) wf.connect(node, out, apply_xfm, 'inputspec.input_image') node, out = strat_pool.get_data("T1w-template") wf.connect(node, out, apply_xfm, 'inputspec.reference') - node, out = strat_pool.get_data(f"from-{space}_to-template_mode-image_xfm") + node, out = strat_pool.get_data(f"from-{space_from}_to-template_mode-image_xfm") wf.connect(node, out, apply_xfm, 'inputspec.transform') outputs = { - 'space-template_desc-brain_mask': (apply_xfm, 'outputspec.output_image') + # there's only one output, so that's what we give here + list(_decorators[warp_what]["outputs"].keys())[0]: ( + apply_xfm, 'outputspec.output_image') } return wf, outputs - return warp_mask_to_template_fxn + return warp_to_template_fxn @nodeblock( diff --git a/CPAC/registration/utils.py b/CPAC/registration/utils.py index 1185f0190b..2bf3d62850 100644 --- a/CPAC/registration/utils.py +++ b/CPAC/registration/utils.py @@ -1,4 +1,22 @@ +# Copyright (C) 2014-2024 C-PAC Developers + +# This file is part of C-PAC. + +# C-PAC is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. + +# C-PAC is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. + +# You should have received a copy of the GNU Lesser General Public +# License along with C-PAC. If not, see . +# pylint: disable=too-many-lines,ungrouped-imports,wrong-import-order import os +from typing import overload import numpy as np @@ -638,3 +656,18 @@ def run_c4d(input, output_name): os.system(cmd) return output1, output2, output3 + + +@overload +def prepend_space(resource: list[str], space: str) -> list[str]: ... +@overload +def prepend_space(resource: str, space: str) -> str: ... +def prepend_space(resource: str | list[str], space: str) -> str | list[str]: + """Given a resource or list of resources, return same but with updated space.""" + if isinstance(resource, list): + return [prepend_space(_, space) for _ in resource] + if "space" not in resource: + return f"space-{space}_{resource}" + pre, post = resource.split("space-") + _old_space, post = post.split("_", 1) + return f"space-{space}_".join([pre, post]) From 4ddb58f1047c95d3c1792cb3bf25835318ef88b7 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 4 Nov 2024 12:05:25 -0500 Subject: [PATCH 30/33] fixup! :bug: Clone instead of rename --- .../longitudinal_workflow.py | 22 +++++++------------ .../pipeline/nipype_pipeline_engine/engine.py | 20 +++++++++++++++++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 67b52c975c..f8dd8f1831 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -453,10 +453,11 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur orig_pipe_name: str = config.pipeline_setup['pipeline_name'] - pools: list[ResourcePool] = [] # Loop over the sessions to create the input for the longitudinal # algorithm - for session in sub_list: + strats_dct: dict[str, list[tuple[pe.Node, str]]] = {"desc-brain_T1w": [], + "desc-head_T1w": []} + for i, session in enumerate(sub_list): unique_id: str = session['unique_id'] session_id_list.append(unique_id) @@ -488,20 +489,13 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur session_wfs[unique_id] = rpool rpool.gather_pipes(workflow, config) + for key in strats_dct.keys(): + _resource: tuple[pe.Node, str] = rpool.get_data(key) + clone = _resource[0].clone(f"{_resource[0].name}_{session_id_list[i]}") + workflow.copy_input_connections(_resource[0], clone) + strats_dct[key].append((clone, _resource[1])) if not dry_run: workflow.run() - pools.append(rpool) - - # Now we have all the anat_preproc set up for every session - # loop over the different anat preproc strategies - strats_dct: dict[str, list] = {key: [pool.get_data(key) for pool in pools] - for key in ["desc-brain_T1w", "desc-head_T1w"]} - # Rename nodes to include session name to avoid duplicates - for key in strats_dct: - for i, resource in enumerate(strats_dct[key]): - strats_dct[key][i] = ( - resource[0].clone(f"{resource[0].name}_{session_id_list[i]}"), - resource[1]) wf = initialize_nipype_wf(config, sub_list[0], # just grab the first one for the name diff --git a/CPAC/pipeline/nipype_pipeline_engine/engine.py b/CPAC/pipeline/nipype_pipeline_engine/engine.py index 31092e5269..c899fbc128 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/engine.py +++ b/CPAC/pipeline/nipype_pipeline_engine/engine.py @@ -496,6 +496,26 @@ def __init__(self, name, base_dir=None, debug=False): self._nodes_cache = set() self._nested_workflows_cache = set() + def copy_input_connections(self, node1: pe.Node, node2: pe.Node) -> None: + """Copy input connections from ``node1`` to ``node2``.""" + new_connections: list[tuple[pe.Node, str, pe.Node, str]] = [] + for connection in self._graph.edges: + _out: pe.Node + _in: pe.Node + _out, _in = connection + if _in == node1: + details = self._graph.get_edge_data(*connection) + if "connect" in details: + for connect in details["connect"]: + new_connections.append((_out, connect[0], node2, connect[1])) + for connection in new_connections: + try: + self.connect(*connection) + except Exception: + # connection already exists + continue + + def _configure_exec_nodes(self, graph): """Ensure that each node knows where to get inputs from""" for node in graph.nodes(): From 0cf793fa2cf983e7f8386a3b5ed15c5aa39bb666 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 8 Nov 2024 17:15:48 -0500 Subject: [PATCH 31/33] :necktie: Handle connections from unconnected graphs that run separately --- .../longitudinal_workflow.py | 31 +++++++++++++------ .../pipeline/nipype_pipeline_engine/engine.py | 18 ++++++++++- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index f8dd8f1831..4cd9b61da0 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -19,7 +19,7 @@ import os import shutil import time -from typing import Optional +from typing import cast, Optional from CPAC.pipeline.nodeblock import nodeblock @@ -455,8 +455,8 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur # Loop over the sessions to create the input for the longitudinal # algorithm - strats_dct: dict[str, list[tuple[pe.Node, str]]] = {"desc-brain_T1w": [], - "desc-head_T1w": []} + strats_dct: dict[str, list[tuple[pe.Node, str] | str]] = {"desc-brain_T1w": [], + "desc-head_T1w": []} for i, session in enumerate(sub_list): unique_id: str = session['unique_id'] @@ -489,13 +489,16 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur session_wfs[unique_id] = rpool rpool.gather_pipes(workflow, config) - for key in strats_dct.keys(): - _resource: tuple[pe.Node, str] = rpool.get_data(key) - clone = _resource[0].clone(f"{_resource[0].name}_{session_id_list[i]}") - workflow.copy_input_connections(_resource[0], clone) - strats_dct[key].append((clone, _resource[1])) + if dry_run: # build tbe graphs with connections that may be in other graphs + for key in strats_dct.keys(): + _resource = cast(tuple[pe.Node, str], rpool.get_data(key)) + clone = _resource[0].clone(f"{_resource[0].name}_{session_id_list[i]}") + workflow.copy_input_connections(_resource[0], clone) + strats_dct[key].append((clone, _resource[1])) if not dry_run: workflow.run() + for key in strats_dct.keys(): # get the outputs from run-nodes + strats_dct[key].append(workflow.get_output_path(key, rpool)) wf = initialize_nipype_wf(config, sub_list[0], # just grab the first one for the name @@ -533,8 +536,8 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur merge_skulls = pe.Node(Merge(num_sessions), name="merge_skulls") for i in list(range(0, num_sessions)): - wf.connect(*strats_dct["desc-brain_T1w"][i], merge_brains, f"in{i + 1}") - wf.connect(*strats_dct["desc-head_T1w"][i], merge_skulls, f"in{i + 1}") + _connect_node_or_path(wf, merge_brains, strats_dct, "desc-brain_T1w", i) + _connect_node_or_path(wf, merge_skulls, strats_dct, "desc-head_T1w", i) wf.connect(merge_brains, "out", template_node, "input_brain_list") wf.connect(merge_skulls, "out", template_node, "input_skull_list") @@ -1198,3 +1201,11 @@ def func_longitudinal_template_wf(subject_id, strat_list, config): workflow.run() return + +def _connect_node_or_path(wf: pe.Workflow, node: pe.Node, strats_dct: dict[str, list[tuple[pe.Node, str] | str]], key: str, index: int) -> None: + """Set input appropriately for either a Node or a path string.""" + input: str = f"in{index + 1}" + if isinstance(strats_dct[key][index], str): + setattr(node.inputs, input, strats_dct[key][index]) + else: + wf.connect(*strats_dct[key][index], node, input) diff --git a/CPAC/pipeline/nipype_pipeline_engine/engine.py b/CPAC/pipeline/nipype_pipeline_engine/engine.py index c899fbc128..ca2fefa736 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/engine.py +++ b/CPAC/pipeline/nipype_pipeline_engine/engine.py @@ -51,8 +51,9 @@ import re from copy import deepcopy from inspect import Parameter, Signature, signature -from typing import ClassVar, Optional, Union +from typing import ClassVar, Optional, TYPE_CHECKING, Union from nibabel import load +from nipype.interfaces.base.support import InterfaceResult from nipype.interfaces.utility import Function from nipype.pipeline import engine as pe from nipype.pipeline.engine.utils import ( @@ -71,6 +72,8 @@ from traits.trait_handlers import TraitListObject from CPAC.utils.monitoring.custom_logging import getLogger from CPAC.utils.typing import DICT +if TYPE_CHECKING: + from CPAC.pipeline.engine import ResourcePool # set global default mem_gb DEFAULT_MEM_GB = 2.0 @@ -664,6 +667,19 @@ def _get_dot( logger.debug("cross connection: %s", dotlist[-1]) return ("\n" + prefix).join(dotlist) + def get_output_path(self, key: str, rpool: "ResourcePool") -> str: + """Get an output path from an already-run Node.""" + _node, _out = rpool.get_data(key) + assert isinstance(_node, pe.Node) + assert isinstance(_out, str) + try: + _run_node: pe.Node = [_ for _ in self.run(updatehash=True).nodes if _.fullname == _node.fullname][0] + except IndexError as index_error: + msg = f"Could not find {key} in {self}'s run Nodes." + raise LookupError(msg) from index_error + _res: InterfaceResult = _run_node.run() + return getattr(_res.outputs, _out) + def _handle_just_in_time_exception(self, node): # pylint: disable=protected-access if hasattr(self, '_local_func_scans'): From 606638c40058937446796a01b86e1ec0122fcbcf Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 11 Nov 2024 10:28:49 -0500 Subject: [PATCH 32/33] :necktie: Clone *and* handle unconnected graphs --- .../longitudinal_workflow.py | 15 ++++++++------- CPAC/pipeline/nipype_pipeline_engine/engine.py | 11 ++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 4cd9b61da0..39ccc33d1f 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -489,16 +489,17 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur session_wfs[unique_id] = rpool rpool.gather_pipes(workflow, config) - if dry_run: # build tbe graphs with connections that may be in other graphs - for key in strats_dct.keys(): - _resource = cast(tuple[pe.Node, str], rpool.get_data(key)) - clone = _resource[0].clone(f"{_resource[0].name}_{session_id_list[i]}") - workflow.copy_input_connections(_resource[0], clone) - strats_dct[key].append((clone, _resource[1])) + for key in strats_dct.keys(): + _resource = cast(tuple[pe.Node, str], rpool.get_data(key)) + clone = _resource[0].clone(f"{_resource[0].name}_{session_id_list[i]}") + workflow.copy_input_connections(_resource[0], clone) + strats_dct[key].append((clone, _resource[1])) if not dry_run: workflow.run() for key in strats_dct.keys(): # get the outputs from run-nodes - strats_dct[key].append(workflow.get_output_path(key, rpool)) + for index, data in enumerate(list(strats_dct[key])): + if isinstance(data, tuple): + strats_dct[key][index] = workflow.get_output_path(*data) wf = initialize_nipype_wf(config, sub_list[0], # just grab the first one for the name diff --git a/CPAC/pipeline/nipype_pipeline_engine/engine.py b/CPAC/pipeline/nipype_pipeline_engine/engine.py index ca2fefa736..b36bc6db67 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/engine.py +++ b/CPAC/pipeline/nipype_pipeline_engine/engine.py @@ -667,18 +667,15 @@ def _get_dot( logger.debug("cross connection: %s", dotlist[-1]) return ("\n" + prefix).join(dotlist) - def get_output_path(self, key: str, rpool: "ResourcePool") -> str: + def get_output_path(self, node: pe.Node, out: str) -> str: """Get an output path from an already-run Node.""" - _node, _out = rpool.get_data(key) - assert isinstance(_node, pe.Node) - assert isinstance(_out, str) try: - _run_node: pe.Node = [_ for _ in self.run(updatehash=True).nodes if _.fullname == _node.fullname][0] + _run_node: pe.Node = [_ for _ in self.run(updatehash=True).nodes if _.fullname == node.fullname][0] except IndexError as index_error: - msg = f"Could not find {key} in {self}'s run Nodes." + msg = f"Could not find {node.fullname} in {self}'s run Nodes." raise LookupError(msg) from index_error _res: InterfaceResult = _run_node.run() - return getattr(_res.outputs, _out) + return getattr(_res.outputs, out) def _handle_just_in_time_exception(self, node): # pylint: disable=protected-access From 4bb363ce7ec6839b17ba16a31bbc3151bf250815 Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 11 Nov 2024 15:03:55 -0500 Subject: [PATCH 33/33] :recycle: Name each preparatory graph uniquely --- CPAC/longitudinal_pipeline/longitudinal_workflow.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index 39ccc33d1f..855a8b201f 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -478,9 +478,10 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur except KeyError: input_creds_path = None - workflow: pe.Workflow = initialize_nipype_wf(config, sub_list[0], - # just grab the first one for the name - name="anat_longitudinal_pre-preproc") + workflow: pe.Workflow = initialize_nipype_wf( + config, + sub_list[i], + name="anat_longitudinal_pre-preproc") rpool: ResourcePool workflow, rpool = initiate_rpool(workflow, config, session) pipeline_blocks = build_anat_preproc_stack(rpool, config) @@ -490,10 +491,7 @@ def anat_longitudinal_wf(subject_id: str, sub_list: list[dict], config: Configur rpool.gather_pipes(workflow, config) for key in strats_dct.keys(): - _resource = cast(tuple[pe.Node, str], rpool.get_data(key)) - clone = _resource[0].clone(f"{_resource[0].name}_{session_id_list[i]}") - workflow.copy_input_connections(_resource[0], clone) - strats_dct[key].append((clone, _resource[1])) + strats_dct[key].append(cast(tuple[pe.Node, str], rpool.get_data(key))) if not dry_run: workflow.run() for key in strats_dct.keys(): # get the outputs from run-nodes