Merge pull request #245 from vinferrer/extra_folder

Create bids_ignore directory and redirect channels plot, trigger plot, logger output and subject log file to this directory

phys2bids/cli/run.py

@@ -120,11 +120,6 @@ def _get_parser():
                           type=str,
                           help='Column header (for json file output).',
                           default=[])
-    optional.add_argument('-chplot', '--channels-plot',
-                          dest='chplot',
-                          type=str,
-                          help='full path to store channels plot ',
-                          default='')
     optional.add_argument('-yml', '--participant-yml',
                           dest='yml',
                           type=str,

phys2bids/phys2bids.py

@@ -112,8 +112,7 @@ def print_json(outfile, samp_freq, time_offset, ch_name):
 
 def phys2bids(filename, info=False, indir='.', outdir='.', heur_file=None,
               sub=None, ses=None, chtrig=0, chsel=None, num_timepoints_expected=0,
-              tr=1, thr=None, ch_name=[], chplot='', debug=False, quiet=False,
-              yml=''):
+              tr=1, thr=None, ch_name=[], yml='', debug=False, quiet=False):
     """
     Main workflow of phys2bids.
 
@@ -133,12 +132,14 @@ def phys2bids(filename, info=False, indir='.', outdir='.', heur_file=None,
     # #!# This can probably be done while parsing?
     outdir = utils.check_input_dir(outdir)
     utils.path_exists_or_make_it(outdir)
-
+    # generate extra path
+    extra_dir = os.path.join(outdir, 'bids_ignore')
+    utils.path_exists_or_make_it(extra_dir)
     # Create logfile name
     basename = 'phys2bids_'
     extension = 'tsv'
     isotime = datetime.datetime.now().strftime('%Y-%m-%dT%H%M%S')
-    logname = os.path.join(outdir, (basename + isotime + '.' + extension))
+    logname = os.path.join(extra_dir, (basename + isotime + '.' + extension))
 
     # Set logging format
     log_formatter = logging.Formatter(
@@ -193,9 +194,8 @@ def phys2bids(filename, info=False, indir='.', outdir='.', heur_file=None,
     LGR.info('Reading infos')
     phys_in.print_info(filename)
     # #!# Here the function viz.plot_channel should be called
-    if chplot != '' or info:
-        viz.plot_all(phys_in.ch_name, phys_in.timeseries, phys_in.units,
-                     phys_in.freq, infile, chplot)
+    viz.plot_all(phys_in.ch_name, phys_in.timeseries, phys_in.units,
+                 phys_in.freq, infile, extra_dir)
     # If only info were asked, end here.
     if info:
         return
@@ -207,7 +207,7 @@ def phys2bids(filename, info=False, indir='.', outdir='.', heur_file=None,
         # #!# Get option of no trigger! (which is wrong practice or Respiract)
         phys_in.check_trigger_amount(thr, num_timepoints_expected, tr)
         LGR.info('Plot trigger')
-        plot_path = os.path.join(outdir,
+        plot_path = os.path.join(extra_dir,
                                  os.path.splitext(os.path.basename(filename))[0])
         if sub:
             plot_path += f'_sub-{sub}'
@@ -299,7 +299,8 @@ def phys2bids(filename, info=False, indir='.', outdir='.', heur_file=None,
                    phys_out[uniq_freq].ch_name)
         print_summary(filename, num_timepoints_expected,
                       phys_in.num_timepoints_found, uniq_freq,
-                      phys_out[uniq_freq].start_time, outfile)
+                      phys_out[uniq_freq].start_time,
+                      os.path.join(extra_dir, os.path.splitext(os.path.basename(outfile))[0]))
 
 
 def _main(argv=None):

phys2bids/tests/test_integration.py

@@ -3,6 +3,7 @@
 import math
 import os
 import re
+import shutil
 import subprocess
 from csv import reader
 from pkg_resources import resource_filename
@@ -33,14 +34,14 @@ def test_logger(multifreq_lab_file):
     test_chtrig = 3
     test_ntp = 1
     test_outdir = test_path
-
+    extra_dir = os.path.join(test_path, 'bids_ignore')
     # Phys2bids call through terminal
     subprocess.run(f'phys2bids -in {test_filename} -indir {test_path} '
                    f'-chtrig {test_chtrig} -ntp {test_ntp} -outdir {test_outdir}',
                    shell=True, check=True)
 
     # Read logger file
-    logger_file = glob.glob(os.path.join(test_path, '*phys2bids*'))[0]
+    logger_file = glob.glob(os.path.join(extra_dir, '*phys2bids*'))[0]
     with open(logger_file) as logger_info:
         logger_info = logger_info.readlines()
 
@@ -59,16 +60,21 @@ def test_integration_txt(samefreq_short_txt_file):
 
     test_path, test_filename = os.path.split(samefreq_short_txt_file)
     test_chtrig = 2
+    extra_dir = os.path.join(test_path, 'bids_ignore')
 
     phys2bids(filename=test_filename, indir=test_path, outdir=test_path,
               chtrig=test_chtrig, num_timepoints_expected=1)
 
     # Check that files are generated
-    for suffix in ['.log', '.json', '.tsv.gz', '_trigger_time.png']:
+    for suffix in ['.json', '.tsv.gz']:
         assert os.path.isfile(os.path.join(test_path, 'Test_belt_pulse_samefreq_short' + suffix))
 
+    # Check files in extra are generated
+    for suffix in ['.log', '_trigger_time.png']:
+        assert os.path.isfile(os.path.join(extra_dir, 'Test_belt_pulse_samefreq_short' + suffix))
+
     # Read log file (note that this file is not the logger file)
-    with open(os.path.join(test_path, 'Test_belt_pulse_samefreq_short.log')) as log_info:
+    with open(os.path.join(extra_dir, 'Test_belt_pulse_samefreq_short.log')) as log_info:
         log_info = log_info.readlines()
 
     # Check timepoints expected
@@ -96,6 +102,7 @@ def test_integration_txt(samefreq_short_txt_file):
         os.remove(filename)
     for filename in glob.glob(os.path.join(test_path, 'Test_belt_pulse_samefreq_short*')):
         os.remove(filename)
+    shutil.rmtree(extra_dir)
 
 
 def test_integration_acq(samefreq_full_acq_file):
@@ -105,16 +112,21 @@ def test_integration_acq(samefreq_full_acq_file):
 
     test_path, test_filename = os.path.split(samefreq_full_acq_file)
     test_chtrig = 3
+    extra_dir = os.path.join(test_path, 'bids_ignore')
 
     phys2bids(filename=test_filename, indir=test_path, outdir=test_path,
               chtrig=test_chtrig, num_timepoints_expected=1)
 
     # Check that files are generated
-    for suffix in ['.log', '.json', '.tsv.gz', '_trigger_time.png']:
+    for suffix in ['.json', '.tsv.gz']:
         assert os.path.isfile(os.path.join(test_path, 'Test_belt_pulse_samefreq' + suffix))
 
+    # Check files in extra are generated
+    for suffix in ['.log', '_trigger_time.png']:
+        assert os.path.isfile(os.path.join(extra_dir, 'Test_belt_pulse_samefreq' + suffix))
+
     # Read log file (note that this file is not the logger file)
-    with open(os.path.join(test_path, 'Test_belt_pulse_samefreq.log')) as log_info:
+    with open(os.path.join(extra_dir, 'Test_belt_pulse_samefreq.log')) as log_info:
         log_info = log_info.readlines()
 
     # Check timepoints expected
@@ -139,10 +151,11 @@ def test_integration_acq(samefreq_full_acq_file):
                                     'MR TRIGGER - Custom, HLT100C - A 5', 'PPG100C', 'CO2', 'O2']
 
     # Remove generated files
-    for filename in glob.glob(os.path.join(test_path, 'phys2bids*')):
+    for filename in glob.glob(os.path.join(extra_dir, 'phys2bids*')):
         os.remove(filename)
     for filename in glob.glob(os.path.join(test_path, 'Test_belt_pulse_samefreq*')):
         os.remove(filename)
+    shutil.rmtree(extra_dir)
 
 
 def test_integration_multifreq(multifreq_lab_file):
@@ -152,30 +165,34 @@ def test_integration_multifreq(multifreq_lab_file):
 
     test_path, test_filename = os.path.split(multifreq_lab_file)
     test_chtrig = 3
+    extra_dir = os.path.join(test_path, 'bids_ignore')
 
     phys2bids(filename=test_filename, indir=test_path, outdir=test_path,
               chtrig=test_chtrig, num_timepoints_expected=1)
 
     # Check that files are generated
-    for suffix in ['.log', '.json', '.tsv.gz']:
+    for suffix in ['.json', '.tsv.gz']:
         assert os.path.isfile(os.path.join(test_path,
                                            'Test1_multifreq_onescan_40.0' + suffix))
-    for suffix in ['.log', '.json', '.tsv.gz']:
+    for suffix in ['.json', '.tsv.gz']:
         assert os.path.isfile(os.path.join(test_path,
                                            'Test1_multifreq_onescan_100.0' + suffix))
-    for suffix in ['.log', '.json', '.tsv.gz']:
+    for suffix in ['.json', '.tsv.gz']:
         assert os.path.isfile(os.path.join(test_path,
                                            'Test1_multifreq_onescan_500.0' + suffix))
-    for suffix in ['.log', '.json', '.tsv.gz']:
+    for suffix in ['.json', '.tsv.gz']:
         assert os.path.isfile(os.path.join(test_path,
                                            'Test1_multifreq_onescan_1000.0' + suffix))
-    assert os.path.isfile(os.path.join(test_path, 'Test1_multifreq_onescan_trigger_time.png'))
+    for freq in ['40', '100', '500', '1000']:
+        assert os.path.isfile(os.path.join(extra_dir,
+                                           'Test1_multifreq_onescan_' + freq + '.log'))
+    assert os.path.isfile(os.path.join(extra_dir, 'Test1_multifreq_onescan_trigger_time.png'))
 
     """
     Checks 40 Hz output
     """
     # Read log file of frequency 625 (note that this file is not the logger file)
-    with open(os.path.join(test_path, 'Test1_multifreq_onescan_40.0.log')) as log_info:
+    with open(os.path.join(extra_dir, 'Test1_multifreq_onescan_40.log')) as log_info:
         log_info = log_info.readlines()
 
     # Check timepoints expected
@@ -202,7 +219,7 @@ def test_integration_multifreq(multifreq_lab_file):
     Checks 100 Hz output
     """
     # Read log file of frequency 625 (note that this file is not the logger file)
-    with open(os.path.join(test_path, 'Test1_multifreq_onescan_100.0.log')) as log_info:
+    with open(os.path.join(extra_dir, 'Test1_multifreq_onescan_100.log')) as log_info:
         log_info = log_info.readlines()
 
     # Check timepoints expected
@@ -229,7 +246,7 @@ def test_integration_multifreq(multifreq_lab_file):
     Checks 500 Hz output
     """
     # Read log file of frequency 625 (note that this file is not the logger file)
-    with open(os.path.join(test_path, 'Test1_multifreq_onescan_500.0.log')) as log_info:
+    with open(os.path.join(extra_dir, 'Test1_multifreq_onescan_500.log')) as log_info:
         log_info = log_info.readlines()
 
     # Check timepoints expected
@@ -253,10 +270,10 @@ def test_integration_multifreq(multifreq_lab_file):
     assert json_data['Columns'] == ['Belt']
 
     """
-    Checks 100 Hz output
+    Checks 1000 Hz output
     """
     # Read log file of frequency 625 (note that this file is not the logger file)
-    with open(os.path.join(test_path, 'Test1_multifreq_onescan_1000.0.log')) as log_info:
+    with open(os.path.join(extra_dir, 'Test1_multifreq_onescan_1000.log')) as log_info:
         log_info = log_info.readlines()
 
     # Check timepoints expected
@@ -280,10 +297,11 @@ def test_integration_multifreq(multifreq_lab_file):
     assert json_data['Columns'] == ['time', 'Trigger']
 
     # Remove generated files
-    for filename in glob.glob(os.path.join(test_path, 'phys2bids*')):
+    for filename in glob.glob(os.path.join(extra_dir, 'phys2bids*')):
         os.remove(filename)
     for filename in glob.glob(os.path.join(test_path, 'Test_belt_pulse_multifreq*')):
         os.remove(filename)
+    shutil.rmtree(extra_dir)
 
 
 def test_integration_heuristic(samefreq_short_txt_file):
@@ -295,6 +313,7 @@ def test_integration_heuristic(samefreq_short_txt_file):
     test_full_path = os.path.join(test_path, test_filename)
     test_chtrig = 1
     test_outdir = test_path
+    extra_dir = os.path.join(test_path, 'bids_ignore')
     test_ntp = 158
     test_tr = 1.2
     test_thr = 0.735
@@ -308,12 +327,12 @@ def test_integration_heuristic(samefreq_short_txt_file):
 
     # Check that files are generated
     base_filename = 'sub-006_ses-01_task-test_rec-biopac_run-01_physio'
-    for suffix in ['.log', '.json', '.tsv.gz']:
+    for suffix in ['.json', '.tsv.gz']:
         assert os.path.isfile(os.path.join(test_path_output, base_filename + suffix))
-
+    assert os.path.isfile(os.path.join(extra_dir, base_filename + '.log'))
     # Read log file (note that this file is not the logger file)
     log_filename = 'sub-006_ses-01_task-test_rec-biopac_run-01_physio.log'
-    with open(os.path.join(test_path_output, log_filename)) as log_info:
+    with open(os.path.join(extra_dir, log_filename)) as log_info:
         log_info = log_info.readlines()
 
     # Check timepoints expected
@@ -339,8 +358,8 @@ def test_integration_heuristic(samefreq_short_txt_file):
 
     # Check that participant.tsv gets updated
     phys2bids(filename=test_full_path, chtrig=test_chtrig, outdir=test_outdir,
-            num_timepoints_expected=test_ntp, tr=test_tr, thr=test_thr, sub='002',
-            ses='01', heur_file=test_heur)
+              num_timepoints_expected=test_ntp, tr=test_tr, thr=test_thr, sub='002',
+              ses='01', heur_file=test_heur)
 
     counter = 0
     subject_list = ['participant_id', '006', '002']
@@ -351,12 +370,13 @@ def test_integration_heuristic(samefreq_short_txt_file):
             counter += 1
 
     # Remove generated files
-    for filename in glob.glob(os.path.join(test_path, 'phys2bids*')):
+    for filename in glob.glob(os.path.join(extra_dir, 'phys2bids*')):
         os.remove(filename)
     for filename in glob.glob(os.path.join(test_path, 'Test_belt_pulse_samefreq*')):
         os.remove(filename)
     for filename in glob.glob(os.path.join(test_path_output, '*')):
         os.remove(filename)
+    shutil.rmtree(extra_dir)
 
 
 def test_integration_info(samefreq_short_txt_file):
@@ -370,7 +390,7 @@ def test_integration_info(samefreq_short_txt_file):
     test_ntp = 158
     test_tr = 1.2
     test_thr = 0.735
-
+    extra_dir = os.path.join(test_path, 'bids_ignore')
     # Move into folder
     subprocess.run(f'cd {test_path}', shell=True, check=True)
     # Phys2bids call through terminal
@@ -382,10 +402,11 @@ def test_integration_info(samefreq_short_txt_file):
     subprocess.run(command_str, shell=True, check=True)
 
     # Check that plot all file is generated
-    assert os.path.isfile('Test_belt_pulse_samefreq_short.png')
+    assert os.path.isfile(os.path.join(test_outdir,
+                                       'bids_ignore/Test_belt_pulse_samefreq_short.png'))
 
     # Read logger file
-    logger_file = glob.glob(os.path.join(test_path, '*phys2bids*'))[0]
+    logger_file = glob.glob(os.path.join(extra_dir, '*phys2bids*'))[0]
     with open(logger_file) as logger_info:
         logger_info = logger_info.readlines()
 
@@ -394,5 +415,6 @@ def test_integration_info(samefreq_short_txt_file):
                         '02. MR TRIGGER - Custom, HLT100C - A 5; sampled at', '10000.0')
 
     # Remove generated files
-    for filename in glob.glob(os.path.join(test_path, 'phys2bids*')):
+    for filename in glob.glob(os.path.join(extra_dir, 'phys2bids*')):
         os.remove(filename)
+    shutil.rmtree(extra_dir)

phys2bids/tests/test_viz.py

@@ -8,10 +8,10 @@ def test_plot_all(samefreq_full_acq_file):
     chtrig = 3
     test_path, test_filename = os.path.split(samefreq_full_acq_file)
     phys_obj = acq.populate_phys_input(samefreq_full_acq_file, chtrig)
-    out = os.path.join(test_path, 'Test_belt_pulse_samefreq.png')
     viz.plot_all(phys_obj.ch_name, phys_obj.timeseries, phys_obj.units,
-                 phys_obj.freq, test_filename, outfile=out)
-    assert os.path.isfile(out)
+                 phys_obj.freq, test_filename, outfile=test_path)
+    assert os.path.isfile(os.path.join(test_path,
+                          os.path.splitext(os.path.basename(test_filename))[0] + '.png'))
 
 
 def test_plot_trigger(samefreq_full_acq_file):

phys2bids/viz.py

@@ -128,7 +128,7 @@ def ntr2time(x):
     plt.close()
 
 
-def plot_all(ch_name, timeseries, units, freq, infile, outfile='', dpi=SET_DPI, size=FIGSIZE):
+def plot_all(ch_name, timeseries, units, freq, infile, outfile, dpi=SET_DPI, size=FIGSIZE):
     """
     Plot all the channels for visualizations and saves them in outfile.
 
@@ -179,7 +179,6 @@ def plot_all(ch_name, timeseries, units, freq, infile, outfile='', dpi=SET_DPI,
         ax[row].xlim = 30 * 60 * freq[0]  # maximum display of half an hour
         ax[row].grid()
     ax[row].set_xlabel("seconds")
-    if outfile == '':
-        outfile = os.path.splitext(os.path.basename(infile))[0] + '.png'
+    outfile = os.path.join(outfile, os.path.splitext(os.path.basename(infile))[0] + '.png')
     LGR.info(f'saving channel plot to {outfile}')
     fig.savefig(outfile, dpi=dpi, bbox_inches='tight')