code cleanup

Author: rafaelfsilva

wfcommons/__init__.py

@@ -8,14 +8,15 @@
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 
-
 from .version import __version__
 
 __author__ = 'WfCommons Team - https://wfcommons.org'
 __credits__ = 'University of Southern California, University of Hawaii at Manoa'
 
 import logging
 
+from .wfchef.recipes import BlastRecipe, BwaRecipe, CyclesRecipe, EpigenomicsRecipe, GenomeRecipe, MontageRecipe, \
+    SeismologyRecipe, SoykbRecipe, SrasearchRecipe
 from .wfgen import WorkflowGenerator
 from .trace import Trace, TraceAnalyzer, TraceElement
 

wfcommons/wfchef/duplicate.py

@@ -10,7 +10,7 @@
 
 import pathlib
 import json
-import pickle 
+import pickle
 import networkx as nx
 from typing import Set, Optional, List, Union, Dict
 from uuid import uuid4
@@ -19,16 +19,17 @@
 from .utils import draw
 import random
 import argparse
-import pandas as pd 
+import pandas as pd
 from functools import partial
 
 this_dir = pathlib.Path(__file__).resolve().parent
 
+
 class NoMicrostructuresError(Exception):
-    pass 
+    pass
+
 
 def duplicate_nodes(graph: nx.DiGraph, nodes: Set[str]) -> Dict:
-    
     """
     Replicates nodes of a graph.
 
@@ -46,7 +47,7 @@ def duplicate_nodes(graph: nx.DiGraph, nodes: Set[str]) -> Dict:
         graph.add_node(new_node, **graph.nodes[node])
         nx.set_node_attributes(graph, {new_node: node}, "duplicate_of")
         new_nodes[node] = new_node
-    
+
     for node, new_node in new_nodes.items():
         for parent, _ in graph.in_edges(node):
             if parent in new_nodes:
@@ -59,11 +60,12 @@ def duplicate_nodes(graph: nx.DiGraph, nodes: Set[str]) -> Dict:
                 graph.add_edge(new_node, new_nodes[child])
             else:
                 graph.add_edge(new_node, child)
-    
+
     return new_nodes
 
-def duplicate(path: pathlib.Path, 
-              base: Union[str, pathlib.Path], 
+
+def duplicate(path: pathlib.Path,
+              base: Union[str, pathlib.Path],
               num_nodes: int) -> nx.DiGraph:
     """
     Attaches replicated nodes to base graph.
@@ -90,29 +92,29 @@ def duplicate(path: pathlib.Path,
 
     graph = pickle.loads(base_path.joinpath("base_graph.pickle").read_bytes())
     if num_nodes < graph.order():
-        raise ValueError(f"Cannot create synthentic graph with {num_nodes} nodes from base graph with {graph.order()} nodes")
+        raise ValueError(
+            f"Cannot create synthentic graph with {num_nodes} nodes from base graph with {graph.order()} nodes")
 
     all_microstructures = json.loads(base_path.joinpath("microstructures.json").read_text())
     microstructures, freqs = map(list, zip(*[(ms, ms["frequency"]) for ms_hash, ms in all_microstructures.items()]))
-    
+
     p: List[float] = (np.array(freqs) / np.sum(freqs)).tolist()
     while graph.order() < num_nodes and microstructures:
         i = random.choice(range(len(microstructures)))
         ms = microstructures[i]
         while ms["nodes"]:
             j = random.choice(range(len(ms["nodes"])))
-            structure = ms["nodes"][j]      
+            structure = ms["nodes"][j]
             if graph.order() + len(structure) > num_nodes:
                 del ms["nodes"][j]
             else:
                 break
-        
-        if not ms["nodes"]: # delete microstructure
+
+        if not ms["nodes"]:  # delete microstructure
             del microstructures[i]
             del p[i]
             continue
-        
+
         duplicate_nodes(graph, structure)
 
     return graph
-

wfcommons/wfchef/find_microstructures.py

@@ -16,14 +16,15 @@
 import json
 from itertools import product
 from networkx.readwrite import read_gpickle, write_gpickle
-import numpy as np 
+import numpy as np
 from itertools import chain, combinations
 import argparse
 from .utils import create_graph, string_hash, type_hash, combine_hashes, annotate, draw
-import math 
+import math
 
 this_dir = pathlib.Path(__file__).resolve().parent
 
+
 def comb(n: int, k: int) -> int:
     """
     Calculates the combination of two integers.
@@ -38,11 +39,12 @@ def comb(n: int, k: int) -> int:
     """
     return math.factorial(n) / (math.factorial(k) * math.factorial(n - k))
 
+
 class ImbalancedMicrostructureError(Exception):
-    pass 
+    pass
+
 
 def get_children(graph: nx.DiGraph, node: str) -> List[str]:
-    
     """
     Gets the children of a node.
 
@@ -55,9 +57,9 @@ def get_children(graph: nx.DiGraph, node: str) -> List[str]:
     :rtype: List[str].
     """
     return [child for _, child in graph.out_edges(node)]
-    
+
+
 def get_parents(graph: nx.DiGraph, node: str) -> List[str]:
-    
     """
     Gets the parents of a node.
 
@@ -71,8 +73,8 @@ def get_parents(graph: nx.DiGraph, node: str) -> List[str]:
     """
     return [parent for parent, _ in graph.in_edges(node)]
 
+
 def get_relatives(graph: nx.DiGraph, node: str) -> Set[str]:
-    
     """
     Gets all node's relatives (children and parents).
 
@@ -86,6 +88,7 @@ def get_relatives(graph: nx.DiGraph, node: str) -> Set[str]:
     """
     return set(chain(get_children(graph, node), get_parents(graph, node)))
 
+
 def find_microstructure(graph: nx.DiGraph, n1: str, n2: str):
     """
     Detects a pattern (microstructure).
@@ -121,7 +124,6 @@ def find_microstructure(graph: nx.DiGraph, n1: str, n2: str):
         common_friends.update(n1_new_friends.intersection(n2_new_friends))
         all_friends.update(n1_new_friends.union(n2_new_friends))
 
-
         n1_new_friends -= common_friends
         n2_new_friends -= common_friends
 
@@ -130,6 +132,7 @@ def find_microstructure(graph: nx.DiGraph, n1: str, n2: str):
 
     return n1_friends, n2_friends, common_friends, all_friends
 
+
 def find_microstructures(graph: nx.DiGraph, verbose: bool = False):
     """
     Detects the patterns (microstructures) that are used for replication and graph expansion.
@@ -142,7 +145,7 @@ def find_microstructures(graph: nx.DiGraph, verbose: bool = False):
     :return: patterns (microstructures)
     :rtype: Set[str].
     """
-    
+
     if verbose:
         print("Sorting nodes by type hash and parent")
     nodes_by_type_hash: Dict[str, Set[str]] = {}
@@ -174,9 +177,9 @@ def find_microstructures(graph: nx.DiGraph, verbose: bool = False):
 
     return microstructures
 
+
 def sort_graphs(workflow_path: Union[pathlib.Path],
                 verbose: bool = False) -> List[nx.DiGraph]:
-    
     """
     Sort graphs in crescent order of number of tasks.
 
@@ -197,7 +200,7 @@ def sort_graphs(workflow_path: Union[pathlib.Path],
         annotate(graph)
         graph.graph["name"] = path.stem
         graphs.append(graph)
-    
+
     if not graphs:
         raise ValueError(f"No graphs found in {workflow_path}")
 
@@ -207,22 +210,23 @@ def sort_graphs(workflow_path: Union[pathlib.Path],
     sorted_graphs = sorted(graphs, key=lambda graph: len(graph.nodes))
     return sorted_graphs
 
-def save_microstructures(workflow_path: Union[pathlib.Path], 
-                         savedir: pathlib.Path, 
-                         verbose: bool = False, 
+
+def save_microstructures(workflow_path: Union[pathlib.Path],
+                         savedir: pathlib.Path,
+                         verbose: bool = False,
                          img_type: Optional[str] = 'png',
                          cutoff: int = 4000,
                          highlight_all_instances: bool = False) -> List[nx.DiGraph]:
     summary = {
         "frequencies": {},
         "base_graphs": {}
     }
-    
+
     for graph in sort_graphs(workflow_path, verbose):
         if graph.order() > cutoff:
             print(f'This and the next workflows have more than {cutoff} tasks')
             break
-    
+
         if verbose:
             print(f"Running for {graph.name}")
         g_savedir = savedir.joinpath(graph.name)
@@ -239,7 +243,7 @@ def save_microstructures(workflow_path: Union[pathlib.Path],
             base_graph_image_path = g_savedir.joinpath(f"base_graph")
             if verbose:
                 print(f"Drawing base graph to {base_graph_image_path}")
-            draw(graph, close=True, legend=False, extension= img_type, save=str(base_graph_image_path))
+            draw(graph, close=True, legend=False, extension=img_type, save=str(base_graph_image_path))
 
         if verbose:
             print("Finding microstructures")
@@ -259,18 +263,17 @@ def save_microstructures(workflow_path: Union[pathlib.Path],
             if img_type:
                 print(f"Drawing {ms_name}")
                 draw(
-                    graph, 
+                    graph,
                     subgraph=list(instances)[0] if not highlight_all_instances else set.union(*instances),
-                    with_labels=False, 
+                    with_labels=False,
                     extension=img_type,
-                    save=str(g_savedir.joinpath(ms_name)), 
+                    save=str(g_savedir.joinpath(ms_name)),
                     close=True
                 )
-            
+
         if verbose:
             print()
-                
-        g_savedir.joinpath("microstructures").with_suffix(".json").write_text(json.dumps(mdatas, indent=2)) 
 
-    savedir.joinpath("summary").with_suffix(".json").write_text(json.dumps(summary, indent=2)) 
-        
+        g_savedir.joinpath("microstructures").with_suffix(".json").write_text(json.dumps(mdatas, indent=2))
+
+    savedir.joinpath("summary").with_suffix(".json").write_text(json.dumps(summary, indent=2))