Merge pull request hed-standard#741 from VisLab/develop

Handled nan conversion in df_to_hed

Author: VisLab

hed/tools/analysis/annotation_util.py

@@ -32,17 +32,18 @@ def df_to_hed(dataframe, description_tag=True):
         description_tag (bool):  If True description tag is included.
 
     Returns:
-        dict:  A dictionary compatible compatible with BIDS JSON tabular file that includes HED.
+        dict:  A dictionary compatible with BIDS JSON tabular file that includes HED.
 
     Notes:
         - The DataFrame must have the columns with names: column_name, column_value, description, and HED.
 
     """
-    missing_cols = check_df_columns(dataframe)
+    df = dataframe.fillna('n/a')
+    missing_cols = check_df_columns(df)
     if missing_cols:
         raise HedFileError("RequiredColumnsMissing", f"Columns {str(missing_cols)} are missing from dataframe", "")
     hed_dict = {}
-    for index, row in dataframe.iterrows():
+    for index, row in df.iterrows():
         if row['HED'] == 'n/a' and row['description'] == 'n/a':
             continue
         if row['column_value'] == 'n/a':

hed/tools/analysis/event_manager.py

@@ -1,17 +1,19 @@
 """ Manages events of temporal extent. """
 
-from hed.tools.analysis.temporal_event import TemporalEvent
 from hed.models.model_constants import DefTagNames
+from hed.models.df_util import get_assembled
+from hed.tools.analysis.temporal_event import TemporalEvent
 
 
 class EventManager:
 
-    def __init__(self, hed_strings, onsets, def_dict):
-        """ Create an event manager for an events file. Manages events of temporal extent. This
+    def __init__(self, input_data, hed_schema, extra_defs=None):
+        """ Create an event manager for an events file. Manages events of temporal extent.
 
         Parameters:
-            data (TabularInput): A tabular input file.
-            schema (HedSchema): A HED schema
+            input_data (TabularInput): Represents an events file with its sidecar.
+            hed_schema (HedSchema): HED schema used in this
+            extra_defs (DefinitionDict):  Extra definitions not included in the input_data information.
 
         :raises HedFileError:
             - if there are any unmatched offsets.
@@ -21,40 +23,61 @@ def __init__(self, hed_strings, onsets, def_dict):
 
         """
 
-        self.event_list = [[] for _ in range(len(onsets))]
-        self.onsets = onsets
-        self.hed_strings = hed_strings
-        self.def_dict = def_dict
-        self.contexts = []
-        self._create_event_list()
+        self.event_list = [[] for _ in range(len(input_data.dataframe))]
+        self.hed_schema = hed_schema
+        self.def_dict = input_data.get_def_dict(hed_schema, extra_def_dicts=extra_defs)
+        self.onsets = input_data.dataframe['onset'].tolist()
+        self.hed_strings = None  # Remaining HED strings copy.deepcopy(hed_strings)
+        self.anchor_dict = {}
+        self._create_event_list(input_data)
+        self._create_anchor_list()
+
+    # def iter_context(self):
+    #     """ Iterate rows of context.
+    #
+    #     Yields:
+    #         int:  position in the dataFrame
+    #         HedString: Context
+    #
+    #     """
+    #
+    #     for index in range(len(self.contexts)):
+    #         yield index, self.contexts[index]
+
+    def _create_anchor_list(self):
+        """ Populate the dictionary of def names to list of temporal events.
 
-    def iter_context(self):
-        """ Iterate rows of context.
+        :raises HedFileError:
+            - If the hed_strings contain unmatched offsets.
 
-        Yields:
-            int:  position in the dataFrame
-            HedString: Context
+        Notes:
 
         """
+        for index, events in enumerate(self.event_list):
+            for event in events:
+                index_list = self.anchor_dict.get(event.anchor, [])
+                index_list.append(event)
+                self.anchor_dict[event.anchor] = index_list
 
-        for index in range(len(self.contexts)):
-            yield index, self.contexts[index]
-
-    def _create_event_list(self):
-        """ Create a list of events of extended duration.
+    def _create_event_list(self, input_data):
+        """ Populate the event_list with the events with temporal extent indexed by event number.
 
         :raises HedFileError:
             - If the hed_strings contain unmatched offsets.
 
         Notes:
 
         """
+        hed_strings, def_dict = get_assembled(input_data, input_data._sidecar, self.hed_schema,
+                                              extra_def_dicts=None, join_columns=True,
+                                              shrink_defs=True, expand_defs=False)
         onset_dict = {}  # Temporary dictionary keeping track of temporal events that haven't ended yet.
-        for event_index, hed in enumerate(self.hed_strings):
+        for event_index, hed in enumerate(hed_strings):
             self._extract_temporal_events(hed, event_index, onset_dict)
         # Now handle the events that extend to end of list
         for item in onset_dict.values():
             item.set_end(len(self.onsets), None)
+        self.hed_strings = hed_strings
 
     def _extract_temporal_events(self, hed, event_index, onset_dict):
         """ Extract the temporal events and remove them from the other HED strings.

hed/tools/analysis/event_manager_copy.py

@@ -0,0 +1,147 @@
+""" Manages events of temporal extent. """
+
+from hed.tools.analysis.temporal_event import TemporalEvent
+from hed.models.model_constants import DefTagNames
+
+
+class EventManagerCopy:
+
+    def __init__(self, input_data, hed_schema, extra_def_dict=None):
+        """ Create an event manager for an events file. Manages events of temporal extent. This
+
+        Parameters:
+            hed_strings (list): A list of HED strings
+            onsets (list): A list of onset times that is the same length as hed_strings
+            def_dict (DefinitionDict):  Contains the definitions for this dataset.
+
+        :raises HedFileError:
+            - if there are any unmatched offsets.
+
+        Notes:  Keeps the events of temporal extend by their starting index in events file. These events
+        are separated from the rest of the annotations.
+
+        """
+
+        self.event_list = [[] for _ in range(len(onsets))]
+        self.onsets = onsets
+        self.hed_strings = hed_strings ## copy.deepcopy(hed_strings)
+        self.def_dict = def_dict
+        self.anchor_dict ={}
+        self._create_event_list()
+        self._create_anchor_list()
+
+    # def iter_context(self):
+    #     """ Iterate rows of context.
+    # 
+    #     Yields:
+    #         int:  position in the dataFrame
+    #         HedString: Context
+    # 
+    #     """
+    # 
+    #     for index in range(len(self.contexts)):
+    #         yield index, self.contexts[index]
+
+    def _create_anchor_list(self):
+        """ Populate the dictionary of def names to list of temporal events.
+
+        :raises HedFileError:
+            - If the hed_strings contain unmatched offsets.
+
+        Notes:
+
+        """
+        for index, events in enumerate(self.event_list):
+            for event in events:
+                index_list = self.anchor_dict.get(event.anchor, [])
+                index_list.append(event)
+                self.anchor_dict[event.anchor] = index_list
+
+    def _create_event_list(self):
+        """ Populate the event_list with the events with temporal extent indexed by event number.
+
+        :raises HedFileError:
+            - If the hed_strings contain unmatched offsets.
+
+        Notes:
+
+        """
+        onset_dict = {}  # Temporary dictionary keeping track of temporal events that haven't ended yet.
+        for event_index, hed in enumerate(self.hed_strings):
+            self._extract_temporal_events(hed, event_index, onset_dict)
+        # Now handle the events that extend to end of list
+        for item in onset_dict.values():
+            item.set_end(len(self.onsets), None)
+
+    def _extract_temporal_events(self, hed, event_index, onset_dict):
+        """ Extract the temporal events and remove them from the other HED strings.
+
+        Parameters:
+            hed (HedString):  The assembled HedString at position event_index in the data.
+            event_index (int): The position of this string in the data.
+            onset_dict (dict):  Running dict that keeps track of temporal events that haven't yet ended.
+
+        Note:
+            This removes the events of temporal extent from the HED string.
+
+         """
+        if not hed:
+            return
+        group_tuples = hed.find_top_level_tags(anchor_tags={DefTagNames.ONSET_KEY, DefTagNames.OFFSET_KEY},
+                                               include_groups=2)
+        to_remove = []
+        for tup in group_tuples:
+            anchor_tag = tup[1].find_def_tags(recursive=False, include_groups=0)[0]
+            anchor = anchor_tag.extension.lower()
+            if anchor in onset_dict or tup[0].short_base_tag.lower() == DefTagNames.OFFSET_KEY:
+                temporal_event = onset_dict.pop(anchor)
+                temporal_event.set_end(event_index, self.onsets[event_index])
+            if tup[0] == DefTagNames.ONSET_KEY:
+                new_event = TemporalEvent(tup[1], event_index, self.onsets[event_index])
+                self.event_list[event_index].append(new_event)
+                onset_dict[anchor] = new_event
+            to_remove.append(tup[1])
+        hed.remove(to_remove)
+
+    def _set_event_contexts(self):
+        """ Creates an event context for each hed string.
+
+        Notes:
+            The event context would be placed in an event context group, but is kept in a separate array without the
+            event context group or tag.
+
+        """
+        # contexts = [[] for _ in range(len(self.hed_strings))]
+        # for onset in self.onset_list:
+        #     for i in range(onset.start_index+1, onset.end_index):
+        #         contexts[i].append(onset.contents)
+        # for i in range(len(self.hed_strings)):
+        #     contexts[i] = HedString(",".join(contexts[i]), hed_schema=self.hed_schema)
+        # self.contexts = contexts
+        print("_set_event_contexts not implemented yet")
+
+    def _update_onset_list(self, group, onset_dict, event_index):
+        """ Process one onset or offset group to create onset_list.
+
+        Parameters:
+            group (HedGroup):  The HedGroup containing the onset or offset.
+            onset_dict (dict): A dictionary of OnsetGroup objects that keep track of span of an event.
+            event_index (int): The event number in the list.
+
+        :raises HedFileError:
+            - if an unmatched offset is encountered.
+
+        Notes:
+            - Modifies onset_dict and onset_list.
+        """
+        # def_tags = group.find_def_tags(recursive=False, include_groups=0)
+        # name = def_tags[0].extension
+        # onset_element = onset_dict.pop(name, None)
+        # if onset_element:
+        #     onset_element.end_index = event_index
+        #     self.onset_list.append(onset_element)
+        # elif is_offset:
+        #     raise HedFileError("UnmatchedOffset", f"Unmatched {name} offset at event {event_index}", " ")
+        # if not is_offset:
+        #     onset_element = TemporalEvent(name, group, event_index)
+        #     onset_dict[name] = onset_element

hed/tools/analysis/hed_context_manager.py

@@ -45,7 +45,6 @@ def __init__(self, hed_strings, hed_schema):
         self.onset_list = []
         self.onset_count = 0
         self.offset_count = 0
-        self.contexts = []
         self._create_onset_list()
         self._set_event_contexts()
 

hed/tools/analysis/hed_type_definitions.py

@@ -5,7 +5,16 @@
 
 
 class HedTypeDefinitions:
+    """
 
+    Properties:
+        def_map (dict):  keys are definition names, values are dict {type_values, description, tags}
+                         Example: A definition 'famous-face-cond' with contents 
+                         `(Condition-variable/Face-type,Description/A face that should be recognized by the participants,(Image,(Face,Famous)))`
+                         would have type_values ['face_type'].  All items are strings not objects.
+
+
+    """
     def __init__(self, definitions, hed_schema, type_tag='condition-variable'):
         """ Create a definition manager for a type of variable.
 
@@ -14,8 +23,6 @@ def __init__(self, definitions, hed_schema, type_tag='condition-variable'):
             hed_schema (Hedschema or HedSchemaGroup): The schema used for parsing.
             type_tag (str): Lower-case HED tag string representing the type managed.
 
-        # TODO: [Refactor] - should dict be allowed for definitions.
-
         """
 
         self.type_tag = type_tag.lower()
@@ -26,8 +33,8 @@ def __init__(self, definitions, hed_schema, type_tag='condition-variable'):
             self.definitions = definitions
         else:
             self.definitions = {}
-        self.def_map = self._extract_def_map()   # maps def names to conditions.
-        self.type_map = self._extract_type_map()
+        self.def_map = self._extract_def_map()   
+        self.type_map = self._extract_type_map() #
 
     def get_type_values(self, item):
         """ Return a list of type_tag values in item.