Added re-referencing per headbox; additional settings in config file; BIDS validation off by default; Late re-referencing in CLI

Author: MaxvandenBoom

erdetect/_erdetect.py

@@ -102,13 +102,18 @@ def process(bids_subset_data_path, output_dir, preproc_prioritize_speed=False):
 
     channels_measured_incl = []                             # the channels that are used as measured electrodes
     channels_stim_incl = []                                 # the channels which stim-pairs should be included (actual filtering of stim-pairs happens at the reading of the events)
-    channels_early_reref_incl = []                          #
+    channels_early_reref_incl_names = []                    # the names of the channels that are included for early re-referencing
+    channels_early_reref_incl_headbox = []                  # the headbox that each of the included early re-referencing channels belong to
+    channels_late_reref_incl_names = []                     # the names of the channels that are included for late re-referencing
+    channels_late_reref_incl_headbox = []                   # the headbox that each of the included late re-referencing channels belong to
 
     channels_measured_excl_by_type = []                     # channels that were excluded as measured electrodes (by type)
     channels_stim_excl_by_type = []                         # channels that were excluded (and as a result exclude stim-pairs)
     channels_early_reref_excl_by_type = []                  #
+    channels_late_reref_excl_by_type = []                  #
 
     channels_have_status = 'status' in channel_tsv.columns
+    channels_have_headbox = 'headbox' in channel_tsv.columns
     for index, row in channel_tsv.iterrows():
 
         # check if bad channel
@@ -135,10 +140,12 @@ def process(bids_subset_data_path, output_dir, preproc_prioritize_speed=False):
 
         # determine if included or excluded from early re-referencing electrodes (by type)
         if cfg('preprocess', 'early_re_referencing', 'enabled'):
-            if row['type'].upper() in cfg('preprocess', 'early_re_referencing', 'types'):
+            if row['type'].upper() in cfg('preprocess', 'early_re_referencing', 'channel_types'):
 
                 # save for log output and the early-referencing (structure)
-                channels_early_reref_incl.append(row['name'])
+                channels_early_reref_incl_names.append(row['name'])
+                if channels_have_status:
+                    channels_early_reref_incl_headbox.append(row['headbox'])
 
                 # save for data reading (no duplicates)
                 if not row['name'] in channels_incl:
@@ -147,6 +154,23 @@ def process(bids_subset_data_path, output_dir, preproc_prioritize_speed=False):
             else:
                 channels_early_reref_excl_by_type.append(row['name'])   # save for log output
 
+        # determine if included or excluded from late re-referencing electrodes (by type)
+        if cfg('preprocess', 'late_re_referencing', 'enabled'):
+            if row['type'].upper() in cfg('preprocess', 'late_re_referencing', 'channel_types'):
+
+                # save for log output and the late-referencing (structure)
+                channels_late_reref_incl_names.append(row['name'])
+                if channels_have_status:
+                    channels_late_reref_incl_headbox.append(row['headbox'])
+                    # TODO: what if nan or not a number
+
+                # save for data reading (no duplicates)
+                if not row['name'] in channels_incl:
+                    channels_incl.append(row['name'])
+
+            else:
+                channels_late_reref_excl_by_type.append(row['name'])   # save for log output
+
     # print channel information
     logging.info(multi_line_list(channels_excl_bad, LOGGING_CAPTION_INDENT_LENGTH, 'Bad channels (excluded):', 25, ' '))
     if channels_measured_excl_by_type == channels_stim_excl_by_type:
@@ -161,19 +185,95 @@ def process(bids_subset_data_path, output_dir, preproc_prioritize_speed=False):
         logging.info(multi_line_list(channels_measured_incl, LOGGING_CAPTION_INDENT_LENGTH, 'Channels incl. as measured electrodes:', 25, ' ', str(len(channels_measured_incl))))
         logging.info(multi_line_list(channels_stim_incl, LOGGING_CAPTION_INDENT_LENGTH, 'Channels incl. as stim electrodes:', 25, ' ', str(len(channels_stim_incl))))
 
+
     # check if there are any channels (as measured electrodes, or to re-reference on)
     if len(channels_measured_incl) == 0:
         logging.error('No channels were found (after filtering by type), exiting...')
         raise RuntimeError('No channels were found')
+
+    # check early re-referencing settings and prepare reref struct
+    early_reref = None
     if cfg('preprocess', 'early_re_referencing', 'enabled'):
-        if len(channels_early_reref_incl) == 0:
-            logging.info(multi_line_list(channels_early_reref_incl, LOGGING_CAPTION_INDENT_LENGTH, 'Channels included (by type) for early re-ref:', 25, ' '))
+
+        if cfg('preprocess', 'early_re_referencing', 'method') == 'CAR_headbox' and not channels_have_headbox:
+            logging.error('Early re-referencing is set to CAR per headbox, but the _channels.tsv file does not have a \'headbox\' column, exiting...')
+            raise RuntimeError('No \'headbox\' column in _channels.tsv file, needed to perform early re-referencing per headbox')
+
+        if len(channels_early_reref_incl_names) == 0:
+            logging.info(multi_line_list(channels_early_reref_incl_names, LOGGING_CAPTION_INDENT_LENGTH, 'Channels included (by type) for early re-ref:', 25, ' '))
             logging.info(multi_line_list(channels_early_reref_excl_by_type, LOGGING_CAPTION_INDENT_LENGTH, 'Channels excluded by type for early re-ref:', 25, ' '))
             logging.error('Early re-referencing is enabled but (after filtering by type) no channels were found, exiting...')
             raise RuntimeError('No channels were found for early re-referencing')
+
+        # generate an early re-referencing object
+        if cfg('preprocess', 'early_re_referencing', 'method') == 'CAR':
+            early_reref = RerefStruct.generate_car(channels_early_reref_incl_names)
+        elif cfg('preprocess', 'early_re_referencing', 'method') == 'CAR_headbox':
+            early_reref = RerefStruct.generate_car_per_headbox(channels_early_reref_incl_names, channels_early_reref_incl_headbox)
+
+            # print CAR headbox info
+            logging.info('')
+            log_indented_line('Early re-referencing groups:', '')
+            for ind, group in enumerate(early_reref.groups):
+                logging.info(multi_line_list(group, LOGGING_CAPTION_INDENT_LENGTH, '      CAR group ' + str(ind) + ':', 25, ' '))
+
+            # check to make sure all included channels are also included in early re-referencing
+            missing_channels = []
+            for channel in channels_measured_incl:
+                if channel not in early_reref.channel_group.keys():
+                    missing_channels.append(channel)
+            if len(missing_channels) == 1:
+                logging.error('Channel \'' + missing_channels[0] + '\' is included but cannot be found in any early re-referencing group, make sure the channel has a valid headbox value in the _channels.tsv')
+                raise RuntimeError('Included channel not in re-referencing group')
+            elif len(missing_channels) > 1:
+                logging.error('Channels \'' + ', '.join(missing_channels) + '\' are included but cannot be found in any early re-referencing group, make sure the channels have valid headbox values in the _channels.tsv')
+                raise RuntimeError('Included channel not in re-referencing group')
+
+
+    # check late re-referencing settings and prepare reref struct
+    late_reref = None
+    if cfg('preprocess', 'late_re_referencing', 'enabled'):
+
+        if cfg('preprocess', 'late_re_referencing', 'method') == 'CAR_headbox' and not channels_have_headbox:
+            logging.error('Late re-referencing is set to CAR per headbox, but the _channels.tsv file does not have a \'headbox\' column, exiting...')
+            raise RuntimeError('No \'headbox\' column in _channels.tsv file, needed to perform late re-referencing per headbox')
+
+        if len(channels_late_reref_incl_names) == 0:
+            logging.info(multi_line_list(channels_late_reref_incl_names, LOGGING_CAPTION_INDENT_LENGTH, 'Channels included (by type) for late re-ref:', 25, ' '))
+            logging.info(multi_line_list(channels_late_reref_excl_by_type, LOGGING_CAPTION_INDENT_LENGTH, 'Channels excluded by type for late re-ref:', 25, ' '))
+            logging.error('Late re-referencing is enabled but (after filtering by type) no channels were found, exiting...')
+            raise RuntimeError('No channels were found for late re-referencing')
+
+        # generate a late re-referencing object
+        if cfg('preprocess', 'late_re_referencing', 'method') == 'CAR':
+            late_reref = RerefStruct.generate_car(channels_late_reref_incl_names)
+        elif cfg('preprocess', 'late_re_referencing', 'method') == 'CAR_headbox':
+            late_reref = RerefStruct.generate_car_per_headbox(channels_late_reref_incl_names, channels_late_reref_incl_headbox)
+
+            # print CAR headbox info
+            logging.info('')
+            log_indented_line('Late re-referencing groups:', '')
+            for ind, group in enumerate(late_reref.groups):
+                logging.info(multi_line_list(group, LOGGING_CAPTION_INDENT_LENGTH, '      CAR group ' + str(ind) + ':', 25, ' '))
+
+            # check to make sure all included channels are also included in late re-referencing
+            missing_channels = []
+            for channel in channels_measured_incl:
+                if channel not in late_reref.channel_group.keys():
+                    missing_channels.append(channel)
+            if len(missing_channels) == 1:
+                logging.error('Channel \'' + missing_channels[0] + '\' is included but cannot be found in any late re-referencing group, make sure the channel has a valid headbox value in the _channels.tsv')
+                raise RuntimeError('Included channel not in re-referencing group')
+            elif len(missing_channels) > 1:
+                logging.error('Channels \'' + ', '.join(missing_channels) + '\' are included but cannot be found in any late re-referencing group, make sure the channels have valid headbox values in the _channels.tsv')
+                raise RuntimeError('Included channel not in re-referencing group')
+
+
+
     logging.info('')
 
 
+
     #
     # retrieve trials
     #
@@ -283,18 +383,16 @@ def process(bids_subset_data_path, output_dir, preproc_prioritize_speed=False):
         logging.error('No stimulus-pairs were found, exiting...')
         raise RuntimeError('No stimulus-pairs found')
 
-    # prepare some preprocessing variables
-    early_reref = None
-    late_reref = None
-    if cfg('preprocess', 'early_re_referencing', 'enabled'):
-
-        # set referencing
-        early_reref = RerefStruct.generate_car(channels_early_reref_incl)
-
-        # set the parts of stimulation (of specific channels) to exclude
+    # set the parts of stimulation (of specific channels) to exclude from early or late re-referencing
+    if early_reref is not None:
         early_reref.set_exclude_reref_epochs(stim_pairs_onsets,
                                              (cfg('preprocess', 'early_re_referencing', 'stim_excl_epoch')[0], cfg('preprocess', 'early_re_referencing', 'stim_excl_epoch')[1]),
                                              '-')
+    if late_reref is not None:
+        late_reref.set_exclude_reref_epochs(stim_pairs_onsets,
+                                             (cfg('preprocess', 'late_re_referencing', 'stim_excl_epoch')[0], cfg('preprocess', 'late_re_referencing', 'stim_excl_epoch')[1]),
+                                             '-')
+    logging.info('')
 
 
     #

erdetect/core/config.py

@@ -29,7 +29,7 @@
 CONFIG_DETECTION_WAVEFORM_PROJ_THRESHOLD = 1000
 
 
-def __create_default_config():
+def create_default_config():
     """
     Create and return a config dictionary with default values
 
@@ -41,12 +41,17 @@ def __create_default_config():
 
     config['preprocess'] = dict()
     config['preprocess']['high_pass']                               = False                     #
-    config['preprocess']['line_noise_removal']                      = 'off'                     #
     config['preprocess']['early_re_referencing'] = dict()
     config['preprocess']['early_re_referencing']['enabled']         = False                     #
     config['preprocess']['early_re_referencing']['method']          = 'CAR'                     #
     config['preprocess']['early_re_referencing']['stim_excl_epoch'] = (-1.0, 2.0)
-    config['preprocess']['early_re_referencing']['types']           = ('ECOG', 'SEEG', 'DBS')   # the type of channels that will be included for early re-referencing
+    config['preprocess']['early_re_referencing']['channel_types']   = ('ECOG', 'SEEG', 'DBS')   # the type of channels that will be included for early re-referencing
+    config['preprocess']['line_noise_removal']                      = 'off'                     # TODO: off, json, 60, 50, 60Hz 50Hz
+    config['preprocess']['late_re_referencing'] = dict()
+    config['preprocess']['late_re_referencing']['enabled']          = False                     #
+    config['preprocess']['late_re_referencing']['method']           = 'CAR'                     #
+    config['preprocess']['late_re_referencing']['stim_excl_epoch']  = (-1.0, 2.0)
+    config['preprocess']['late_re_referencing']['channel_types']    = ('ECOG', 'SEEG', 'DBS')   # the type of channels that will be included for late re-referencing
 
     config['trials'] = dict()
     config['trials']['trial_epoch']                                 = (-1.0, 2.0)               # the time-span (in seconds) relative to the stimulus onset that will be used to extract the signal for each trial
@@ -175,7 +180,7 @@ def load_config(filepath):
     """
 
     # first retrieve a default config
-    config = __create_default_config()
+    config = create_default_config()
 
     # try to read the JSON configuration file
     try:
@@ -268,7 +273,7 @@ def retrieve_config_string(json_dict, ref_config, level1, level2, level3=None, o
                         else:
                             value_cased = json_dict[level1][level2]
                             if not case_sensitive:
-                                options = (option.lower() for option in options)
+                                options = [option.lower() for option in options]
                                 value_cased = value_cased.lower()
                             if value_cased in options:
                                 ref_config[level1][level2] = json_dict[level1][level2]
@@ -287,7 +292,7 @@ def retrieve_config_string(json_dict, ref_config, level1, level2, level3=None, o
                             else:
                                 value_cased = json_dict[level1][level2][level3]
                                 if not case_sensitive:
-                                    options = (option.lower() for option in options)
+                                    options = [option.lower() for option in options]
                                     value_cased = value_cased.lower()
                                 if value_cased in options:
                                     ref_config[level1][level2][level3] = json_dict[level1][level2][level3]
@@ -358,6 +363,26 @@ def retrieve_config_tuple(json_dict, ref_config, level1, level2, level3=None, op
     # preprocessing settings
     if not retrieve_config_bool(json_config, config, 'preprocess', 'high_pass'):
         return False
+    if not retrieve_config_bool(json_config, config, 'preprocess', 'early_re_referencing', 'enabled'):
+        return False
+    if not retrieve_config_string(json_config, config, 'preprocess', 'early_re_referencing', 'method', options=('CAR', 'CAR_headbox')):
+        return False
+
+    if not retrieve_config_string(json_config, config, 'preprocess', 'line_noise_removal', options=('off', 'json', '50', '60', '50hz', '60hz')):
+        return False
+    if config['preprocess']['line_noise_removal'].lower() == '50hz':
+        config['preprocess']['line_noise_removal'] = '50'
+    if config['preprocess']['line_noise_removal'].lower() == '60hz':
+        config['preprocess']['line_noise_removal'] = '60'
+    # TODO: load line noise removal, try also to accept a number instead of a string
+
+    if not retrieve_config_bool(json_config, config, 'preprocess', 'late_re_referencing', 'enabled'):
+        return False
+    if not retrieve_config_string(json_config, config, 'preprocess', 'late_re_referencing', 'method', options=('CAR', 'CAR_headbox')):
+        return False
+    # TODO: load early re-referencing channels
+    # TODO: load late re-referencing channels
+
 
     # trials settings
     if not retrieve_config_range(json_config, config, 'trials', 'trial_epoch'):
@@ -486,11 +511,18 @@ def write_config(filepath):
     config_str = '{\n' \
                  '    "preprocess": {\n' \
                  '        "high_pass":                        ' + ('true' if _config['preprocess']['high_pass'] else 'false') + ',\n' \
-                 '        "line_noise_removal":               "' + _config['preprocess']['line_noise_removal'] + '",\n' \
                  '        "early_re_referencing": {\n' \
                  '            "enabled":                      ' + ('true' if _config['preprocess']['early_re_referencing']['enabled'] else 'false') + ',\n' \
                  '            "method":                       "' + _config['preprocess']['early_re_referencing']['method'] + '",\n' \
-                 '            "stim_excl_epoch":              [' + numbers_to_padded_string(_config['preprocess']['early_re_referencing']['stim_excl_epoch'], 16) + ']\n' \
+                 '            "stim_excl_epoch":              [' + numbers_to_padded_string(_config['preprocess']['early_re_referencing']['stim_excl_epoch'], 16) + '],\n' \
+                 '            "channel_types":                ' + json.dumps(_config['preprocess']['early_re_referencing']['channel_types']) + '\n' \
+                 '        },\n' \
+                 '        "line_noise_removal":               "' + _config['preprocess']['line_noise_removal'] + '",\n' \
+                 '        "late_re_referencing": {\n' \
+                 '            "enabled":                      ' + ('true' if _config['preprocess']['late_re_referencing']['enabled'] else 'false') + ',\n' \
+                 '            "method":                       "' + _config['preprocess']['late_re_referencing']['method'] + '",\n' \
+                 '            "stim_excl_epoch":              [' + numbers_to_padded_string(_config['preprocess']['late_re_referencing']['stim_excl_epoch'], 16) + '],\n' \
+                 '            "channel_types":                ' + json.dumps(_config['preprocess']['late_re_referencing']['channel_types']) + '\n' \
                  '        }\n' \
                  '    },\n\n' \
                  '    "trials": {\n' \
@@ -719,4 +751,4 @@ def check_range_order(ref_config, level1, level2, level3=None):
 
 
 # initialize a variable with a default configuration dictionary for this module
-_config = __create_default_config()
+_config = create_default_config()