Fixing input/ouput files

Author: tainagdcoleman

wfcommons/wfgen/abstract_recipe.py

@@ -203,10 +203,13 @@ def _generate_task_files(self, task: Task) -> List[File]:
 
         # generate output files
         output_files_list = self._generate_files(task.task_id, task_recipe['output'], FileLink.OUTPUT)
-        task.files = self.tasks_files[task.task_id]
+        task.output_files = self.tasks_files[task.task_id]
+
+        
 
         # obtain input files from parents
         input_files = []
+        
         if task.name in self.tasks_parents.keys():
             for parent_task_name in self.tasks_parents[task.name]:
                 output_files = self._generate_task_files(self.tasks_map[parent_task_name])
@@ -215,14 +218,15 @@ def _generate_task_files(self, task: Task) -> List[File]:
                 input_files.extend(output_files)
 
         for input_file in input_files:
-            if input_file.name not in self.tasks_files_names[task.task_id]:
-                self.tasks_files[task.task_id].append(File(name=input_file.name,
-                                                        link=FileLink.INPUT,
-                                                        size=input_file.size))
-                self.tasks_files_names[task.task_id].append(input_file.name)
+            if input_file not in self.tasks_files_names[task.task_id]:
+                self.tasks_files[task.task_id].append(File(name=input_file,
+                                                           link=FileLink.INPUT,
+                                                            size=input_file.size))
+                self.tasks_files_names[task.task_id].append(input_file)
 
         # generate additional input files
         self._generate_files(task.task_id, task_recipe['input'], FileLink.INPUT)
+        task.input_files = [ifile for ifile in self.tasks_files[task.task_id] if ifile.link == FileLink.INPUT]
 
         return output_files_list
 

wfcommons/wfinstances/instance_analyzer.py

@@ -106,26 +106,29 @@ def build_summary(self,
             for task in self.tasks_summary[task_name]:
                 runtime_list.append(task.runtime)
 
-
                 # For each input_file and output_file, append the file size to the dictionary
                 for infile in task.input_files:
                     extension: str = path.splitext(infile.file_id)[1] if '.' in infile.file_id else infile.file_id
+                    # print(f"file {infile.file_id} extension: {extension}")
                     if extension[1:].isnumeric():
                         extension = path.splitext(infile.file_id.replace(extension, ''))[1]
 
                     # Check if the file is definetly an input 
                     assert infile.link == FileLink.INPUT, f"{infile.file_id} is not set as input"
                     _append_file_to_dict(extension, inputs_dict, infile.size)
+
 
                 for outfile in task.output_files:
                     extension: str = path.splitext(outfile.file_id)[1] if '.' in outfile.file_id else outfile.file_id
+                    # print(f"file {outfile.file_id} extension: {extension}")
                     if extension[1:].isnumeric():
                         extension = path.splitext(outfile.file_id.replace(extension, ''))[1]
 
                     # Check if the file is definetly an output
                     assert outfile.link == FileLink.OUTPUT, f"{outfile.file_id} is not set as output"
                     _append_file_to_dict(extension, outputs_dict, outfile.size) 
 
+            
             # Find the best fit distribution for each file type
             _best_fit_distribution_for_file(inputs_dict, include_raw_data)
             _best_fit_distribution_for_file(outputs_dict, include_raw_data)
@@ -198,7 +201,6 @@ def _append_file_to_dict(extension: str, dict_obj: Dict[str, Any], file_size: in
         dict_obj[extension] = {'data': [], 'distribution': None}
     dict_obj[extension]['data'].append(file_size)
 
-
 def _best_fit_distribution_for_file(dict_obj, include_raw_data) -> None:
     """
     Find the best fit distribution for a file.