Merge wfchef_updates into main

Author: tainagdcoleman

wfcommons/utils.py

@@ -7,7 +7,6 @@
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-
 import warnings
 warnings.filterwarnings('ignore')
 
@@ -25,6 +24,7 @@
 from typing import Any, Dict, Optional, List, Tuple
 
 
+
 class NoValue(Enum):
     def __repr__(self):
         return '<%s.%s>' % (self.__class__.__name__, self.name)

wfcommons/wfchef/chef.py

@@ -193,59 +193,20 @@ def ls_recipe():
     print(get_recipes())
 
 
-def uninstall_recipe(wf_name: str,
+def uninstall_recipe(module_name:str,
                      savedir: pathlib.Path = this_dir.joinpath("recipes")):
     """
     Uninstalls a recipe installed in the system.
     """
 
-    dst = pathlib.Path(savedir, f"{savedir.stem}_recipes", wf_name).resolve()
+    dst = f"wfcommons.wfchef.recipe.{savedir.stem}"
     try:
-        # Removing package from setup.py
-        with this_dir.joinpath(dst.parent.parent.joinpath("setup.py")).open("r") as fp:
-            setup_str = fp.read()
-
-        # Find and remove the specific line that added the package
-        for line in setup_str.split("\n"):
-            if wf_name in setup_str:
-                setup_str = setup_str.replace(line, "")
-
-        with this_dir.joinpath(dst.parent.parent.joinpath("setup.py")).open("w") as fp:
-            fp.write(setup_str)
-
-        # Removing the import line from __init__.py
-        init_file = dst.parent.joinpath("__init__.py")
-        if init_file.exists():
-            with init_file.open("r") as fp:
-                init_str = fp.read()
-
-            # Remove the line that imports the package
-
-            for line in init_str.split("\n"):
-                if wf_name in line:
-                    init_str = init_str.replace(line, "")
-                    break
-
-            with init_file.open("w") as fp:
-                fp.write(init_str)
-
-        # Removing the recipe directory 
-        if dst.exists():
-            print(f"Removing {dst}")
-            subprocess.Popen(["rm", "-rf", str(dst)]).wait()
-
-        # Find setup.py, clean it and reinstall it
-        path_to_setup = this_dir.parent.parent.resolve()
-        # Check if the setup.py is in the same directory
-        if path_to_setup.joinpath("setup.py").exists():
-            subprocess.Popen(["python", "setup.py", "clean", "-all"]).wait()
-            subprocess.Popen(["python", "setup.py", "sdist", "bdist_wheel"]).wait()
-            subprocess.Popen(["pip", "install", "."]).wait()
-            subprocess.Popen(["pip", "cache", "purge"]).wait()
-            
+        subprocess.run(["pip", "uninstall", "-y", dst])
+        traceback.print_exc()
+
     except Exception as e:
         traceback.print_exc()
-    # print(f"Could not find recipe with module name {module_name} installed")
+        print(f"Could not uninstall recipe for {module_name}")
 
 
 def create_recipe(path_to_instances: Union[str, pathlib.Path],

wfcommons/wfgen/abstract_recipe.py

@@ -203,10 +203,13 @@ def _generate_task_files(self, task: Task) -> List[File]:
 
         # generate output files
         output_files_list = self._generate_files(task.task_id, task_recipe['output'], FileLink.OUTPUT)
-        task.files = self.tasks_files[task.task_id]
+        task.output_files = self.tasks_files[task.task_id]
+
+        
 
         # obtain input files from parents
         input_files = []
+        
         if task.name in self.tasks_parents.keys():
             for parent_task_name in self.tasks_parents[task.name]:
                 output_files = self._generate_task_files(self.tasks_map[parent_task_name])
@@ -215,14 +218,15 @@ def _generate_task_files(self, task: Task) -> List[File]:
                 input_files.extend(output_files)
 
         for input_file in input_files:
-            if input_file.name not in self.tasks_files_names[task.task_id]:
-                self.tasks_files[task.task_id].append(File(name=input_file.name,
-                                                        link=FileLink.INPUT,
-                                                        size=input_file.size))
-                self.tasks_files_names[task.task_id].append(input_file.name)
+            if input_file not in self.tasks_files_names[task.task_id]:
+                self.tasks_files[task.task_id].append(File(name=input_file,
+                                                           link=FileLink.INPUT,
+                                                            size=input_file.size))
+                self.tasks_files_names[task.task_id].append(input_file)
 
         # generate additional input files
         self._generate_files(task.task_id, task_recipe['input'], FileLink.INPUT)
+        task.input_files = [ifile for ifile in self.tasks_files[task.task_id] if ifile.link == FileLink.INPUT]
 
         return output_files_list
 

wfcommons/wfinstances/instance_analyzer.py

@@ -12,6 +12,7 @@
 import math
 import numpy
 import scipy.stats
+import warnings
 
 from logging import Logger
 from matplotlib import pyplot
@@ -105,26 +106,29 @@ def build_summary(self,
             for task in self.tasks_summary[task_name]:
                 runtime_list.append(task.runtime)
 
-
                 # For each input_file and output_file, append the file size to the dictionary
                 for infile in task.input_files:
                     extension: str = path.splitext(infile.file_id)[1] if '.' in infile.file_id else infile.file_id
+                    # print(f"file {infile.file_id} extension: {extension}")
                     if extension[1:].isnumeric():
                         extension = path.splitext(infile.file_id.replace(extension, ''))[1]
 
                     # Check if the file is definetly an input 
                     assert infile.link == FileLink.INPUT, f"{infile.file_id} is not set as input"
                     _append_file_to_dict(extension, inputs_dict, infile.size)
+
 
                 for outfile in task.output_files:
                     extension: str = path.splitext(outfile.file_id)[1] if '.' in outfile.file_id else outfile.file_id
+                    # print(f"file {outfile.file_id} extension: {extension}")
                     if extension[1:].isnumeric():
                         extension = path.splitext(outfile.file_id.replace(extension, ''))[1]
 
                     # Check if the file is definetly an output
                     assert outfile.link == FileLink.OUTPUT, f"{outfile.file_id} is not set as output"
                     _append_file_to_dict(extension, outputs_dict, outfile.size) 
 
+            
             # Find the best fit distribution for each file type
             _best_fit_distribution_for_file(inputs_dict, include_raw_data)
             _best_fit_distribution_for_file(outputs_dict, include_raw_data)
@@ -197,7 +201,6 @@ def _append_file_to_dict(extension: str, dict_obj: Dict[str, Any], file_size: in
         dict_obj[extension] = {'data': [], 'distribution': None}
     dict_obj[extension]['data'].append(file_size)
 
-
 def _best_fit_distribution_for_file(dict_obj, include_raw_data) -> None:
     """
     Find the best fit distribution for a file.
@@ -207,14 +210,16 @@ def _best_fit_distribution_for_file(dict_obj, include_raw_data) -> None:
     :param include_raw_data:
     :type include_raw_data: bool
     """
-    for ext in dict_obj:
-        dict_obj[ext]['min'] = min(dict_obj[ext]['data'])
-        dict_obj[ext]['max'] = max(dict_obj[ext]['data'])
-        if dict_obj[ext]['min'] != dict_obj[ext]['max']:
-            dict_obj[ext]['distribution'] = _json_format_distribution_fit(
-                best_fit_distribution(dict_obj[ext]['data']))
-        if not include_raw_data:
-            del dict_obj[ext]['data']
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        for ext in dict_obj:
+            dict_obj[ext]['min'] = min(dict_obj[ext]['data'])
+            dict_obj[ext]['max'] = max(dict_obj[ext]['data'])
+            if dict_obj[ext]['min'] != dict_obj[ext]['max']:
+                dict_obj[ext]['distribution'] = _json_format_distribution_fit(
+                    best_fit_distribution(dict_obj[ext]['data']))
+            if not include_raw_data:
+                del dict_obj[ext]['data']
 
 
 def _json_format_distribution_fit(dist_tuple: Tuple) -> Dict[str, Any]: