Merge pull request #81 from wfcommons/nextflow_improvements

rafaelfsilva · web-flow · commit 654ea9389b1c · 2025-06-19T10:19:20.000-04:00
Nextflow improvements
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,3 +1,6 @@
+graft wfcommons/wfchef/recipes/rnaseq/microstructures/**
+graft wfcommons/wfchef/recipes/rnaseq/microstructures
+graft wfcommons/wfchef/recipes/rnaseq
 graft wfcommons/wfchef/recipes/epigenomics/microstructures/**
 graft wfcommons/wfchef/recipes/epigenomics/microstructures
 graft wfcommons/wfchef/recipes/epigenomics
diff --git a/docs/source/dev_api_wfbench.rst b/docs/source/dev_api_wfbench.rst
@@ -15,7 +15,7 @@ wfcommons.wfbench.bench
    :noindex:
  
 wfcommons.wfbench.translator.nextflow
-------------------------------------
+-------------------------------------
 
 .. automodule:: wfcommons.wfbench.translator.nextflow
    :members:
diff --git a/docs/source/generating_workflows.rst b/docs/source/generating_workflows.rst
@@ -25,6 +25,7 @@ synthetic workflow instances. The current list of available workflow recipes inc
 - :class:`~wfcommons.wfchef.recipes.epigenomics.recipe.EpigenomicsRecipe`: :code:`from wfcommons.wfchef.recipes import EpigenomicsRecipe`
 - :class:`~wfcommons.wfchef.recipes.genome.recipe.GenomeRecipe`: :code:`from wfcommons.wfchef.recipes import GenomeRecipe`
 - :class:`~wfcommons.wfchef.recipes.montage.recipe.MontageRecipe`: :code:`from wfcommons.wfchef.recipes import MontageRecipe`
+- :class:`~wfcommons.wfchef.recipes.rnaseq.recipe.RnaseqRecipe`: :code:`from wfcommons.wfchef.recipes import RnaseqRecipe`
 - :class:`~wfcommons.wfchef.recipes.seismology.recipe.SeismologyRecipe`: :code:`from wfcommons.wfchef.recipes import SeismologyRecipe`
 - :class:`~wfcommons.wfchef.recipes.soykb.recipe.SoykbRecipe`: :code:`from wfcommons.wfchef.recipes import SoykbRecipe`
 - :class:`~wfcommons.wfchef.recipes.srasearch.recipe.SrasearchRecipe`: :code:`from wfcommons.wfchef.recipes import SrasearchRecipe`
diff --git a/docs/source/parsing_logs.rst b/docs/source/parsing_logs.rst
diff --git a/docs/source/user_api_wfchef.rst b/docs/source/user_api_wfchef.rst
@@ -54,6 +54,14 @@ wfcommons.wfchef.recipes.montage.recipe
    :undoc-members:
    :show-inheritance:
 
+wfcommons.wfchef.recipes.rnaseq.recipe
+--------------------------------------
+
+.. automodule:: wfcommons.wfchef.recipes.rnaseq.recipe
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 wfcommons.wfchef.recipes.seismology.recipe
 ------------------------------------------
 
diff --git a/setup.py b/setup.py
@@ -49,6 +49,7 @@ def run(self):
             'genome_recipe = wfcommons.wfchef.recipes:GenomeRecipe',
             'blast_recipe = wfcommons.wfchef.recipes:BlastRecipe',
             'bwa_recipe = wfcommons.wfchef.recipes:BwaRecipe',
+            'rnaseq_recipe = wfcommons.wfchef.recipes:RnaseqRecipe',
         ]
     },
 )
diff --git a/tests/unit/common/test_task.py b/tests/unit/common/test_task.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2024 The WfCommons Team.
+# Copyright (c) 2024-2025 The WfCommons Team.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -41,7 +41,7 @@ def task(self) -> Task:
             energy=100,
             avg_power=1.0,
             priority=100,
-            executedAt="2024-09-15T08:59:33.699321-04:00",
+            executed_at="2024-09-15T08:59:33.699321-04:00",
             task_type=TaskType.COMPUTE,
             launch_dir="/tmp",
         )
diff --git a/tests/unit/common/test_workflow.py b/tests/unit/common/test_workflow.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2024 The WfCommons Team.
+# Copyright (c) 2024-2025 The WfCommons Team.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -11,6 +11,7 @@
 import pathlib
 import pytest
 
+from datetime import datetime
 from wfcommons.common import Task, Workflow
 from wfcommons.version import __version__, __schema_version__
 
@@ -28,7 +29,8 @@ class TestWorkflow:
     def workflow(self) -> Workflow:
         return Workflow(
             name="Workflow Test",
-            makespan=100.0
+            makespan=100.0,
+            executed_at=str(datetime.now().astimezone().isoformat())
         )
 
     @pytest.mark.unit
diff --git a/wfcommons/common/task.py b/wfcommons/common/task.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2020-2023 The WfCommons Team.
+# Copyright (c) 2020-2025 The WfCommons Team.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -87,7 +87,7 @@ def __init__(self,
                  energy: Optional[int] = None,
                  avg_power: Optional[float] = None,
                  priority: Optional[int] = None,
-                 executedAt: Optional[str] = None,
+                 executed_at: Optional[str] = None,
                  task_type: Optional[TaskType] = None,
                  launch_dir: Optional[str] = None,
                  logger: Optional[Logger] = None,
@@ -114,7 +114,7 @@ def __init__(self,
         self.priority: Optional[int] = priority
         self.type: Optional[TaskType] = task_type
         self.launch_dir: Optional[str] = launch_dir
-        self.start_time: Optional[str] = str(datetime.now().astimezone().isoformat()) if not executedAt else executedAt
+        self.start_time: Optional[str] = str(datetime.now().astimezone().isoformat()) if not executed_at else executed_at
         self.logger.debug(
             f"created task {self.task_id}: runtime => {self.runtime} seconds.")
 
diff --git a/wfcommons/common/workflow.py b/wfcommons/common/workflow.py
@@ -84,7 +84,7 @@ def __init__(self,
         self.tasks_children = {}
         self.workflow_id: str = None
         self.workflow_json = {}
-        super().__init__(name=name, makespan=self.makespan, executedat=self.executed_at)
+        super().__init__(name=name, makespan=self.makespan, executed_at=self.executed_at)
 
     def add_task(self, task: Task) -> None:
         """
diff --git a/wfcommons/wfchef/chef.py b/wfcommons/wfchef/chef.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2021 The WfCommons Team.
+# Copyright (c) 2021-2025 The WfCommons Team.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -230,7 +230,7 @@ def create_recipe(path_to_instances: Union[str, pathlib.Path],
     :param verbose: when set, prints status messages.
     :type cutoff: bool
     :param verbose: number of times to repeat the err calculation process (due to randomization).
-    :type runs:bool
+    :type runs: bool
     """
     camelname = capitalcase(wf_name)
     savedir.mkdir(exist_ok=True, parents=True)
diff --git a/wfcommons/wfchef/recipes/__init__.py b/wfcommons/wfchef/recipes/__init__.py
@@ -1,20 +1,20 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2021 The WfCommons Team.
+# Copyright (c) 2021-2025 The WfCommons Team.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 
+from .blast import BlastRecipe
+from .bwa import BwaRecipe
+from .cycles import CyclesRecipe
 from .epigenomics import EpigenomicsRecipe
+from .genome import GenomeRecipe
 from .montage import MontageRecipe
-from .cycles import CyclesRecipe
+from .rnaseq import RnaseqRecipe
 from .seismology import SeismologyRecipe
 from .soykb import SoykbRecipe
 from .srasearch import SrasearchRecipe
-
-from .genome import GenomeRecipe
-from .blast import BlastRecipe
-from .bwa import BwaRecipe
diff --git a/wfcommons/wfchef/recipes/rnaseq/__init__.py b/wfcommons/wfchef/recipes/rnaseq/__init__.py
@@ -0,0 +1 @@
+from .recipe import RnaseqRecipe
diff --git a/wfcommons/wfchef/recipes/rnaseq/microstructures/metric/err.csv b/wfcommons/wfchef/recipes/rnaseq/microstructures/metric/err.csv
@@ -0,0 +1,4 @@
+,rnaseq-3l,rnaseq-6l
+rnaseq-t1l,0.004108531398080609,0.004108531398080609
+rnaseq-3l,0.0,0.0
+rnaseq-6l,,0.0
diff --git a/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-3l/base_graph.pickle b/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-3l/base_graph.pickle
diff --git a/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-3l/microstructures.json b/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-3l/microstructures.json
@@ -0,0 +1 @@
+{}
diff --git a/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-6l/base_graph.pickle b/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-6l/base_graph.pickle
diff --git a/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-6l/microstructures.json b/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-6l/microstructures.json
@@ -0,0 +1 @@
+{}
diff --git a/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-t1l/base_graph.pickle b/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-t1l/base_graph.pickle
diff --git a/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-t1l/microstructures.json b/wfcommons/wfchef/recipes/rnaseq/microstructures/rnaseq-t1l/microstructures.json
@@ -0,0 +1 @@
+{}
diff --git a/wfcommons/wfchef/recipes/rnaseq/microstructures/summary.json b/wfcommons/wfchef/recipes/rnaseq/microstructures/summary.json
@@ -0,0 +1,17 @@
+{
+  "frequencies": {},
+  "base_graphs": {
+    "rnaseq-t1l": {
+      "size": 126,
+      "order": 65
+    },
+    "rnaseq-3l": {
+      "size": 128,
+      "order": 66
+    },
+    "rnaseq-6l": {
+      "size": 128,
+      "order": 66
+    }
+  }
+}
diff --git a/wfcommons/wfchef/recipes/rnaseq/recipe.py b/wfcommons/wfchef/recipes/rnaseq/recipe.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2021 The WfCommons Team.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+import pathlib
+from logging import Logger
+from typing import Optional, Set
+import pathlib 
+from wfcommons.wfchef.wfchef_abstract_recipe import BaseMethod, WfChefWorkflowRecipe
+
+this_dir = pathlib.Path(__file__).resolve().parent
+
+
+class RnaseqRecipe(WfChefWorkflowRecipe):
+    """A Rnaseq workflow recipe class for creating synthetic workflow instances.
+
+    :param data_footprint: The upper bound for the workflow total data footprint (in bytes).
+    :type data_footprint: int
+    :param num_tasks: The upper bound for the total number of tasks in the workflow.
+    :type num_tasks: int
+    :param exclude_graphs:
+    :type exclude_graphs: Set
+    :param runtime_factor: The factor of which tasks runtime will be increased/decreased.
+    :type runtime_factor: float
+    :param input_file_size_factor: The factor of which tasks input files size will be increased/decreased.
+    :type input_file_size_factor: float
+    :param output_file_size_factor: The factor of which tasks output files size will be increased/decreased.
+    :type output_file_size_factor: float
+    :param logger: The logger where to log information/warning or errors (optional).
+    :type logger: Logger
+    """
+
+    def __init__(self,
+                 data_footprint: Optional[int] = 0,
+                 num_tasks: Optional[int] = 3,
+                 exclude_graphs: Set[str] = set(),
+                 runtime_factor: Optional[float] = 1.0,
+                 input_file_size_factor: Optional[float] = 1.0,
+                 output_file_size_factor: Optional[float] = 1.0,
+                 logger: Optional[Logger] = None,
+                 base_method: BaseMethod = BaseMethod.ERROR_TABLE,
+                 **kwargs) -> None:
+        super().__init__(
+            name="Rnaseq", 
+            data_footprint=data_footprint, 
+            num_tasks=num_tasks, 
+            exclude_graphs=exclude_graphs, 
+            runtime_factor=runtime_factor, 
+            input_file_size_factor=input_file_size_factor,
+            output_file_size_factor=output_file_size_factor, 
+            logger=logger, 
+            this_dir=this_dir, 
+            base_method=base_method,
+            **kwargs
+        )
diff --git a/wfcommons/wfchef/recipes/rnaseq/task_type_stats.json b/wfcommons/wfchef/recipes/rnaseq/task_type_stats.json
diff --git a/wfcommons/wfinstances/logs/nextflow.py b/wfcommons/wfinstances/logs/nextflow.py

Original file line number	Diff line number	Diff line change
`@@ -49,6 +49,7 @@ def run(self):`
`49`	`49`	`'genome_recipe = wfcommons.wfchef.recipes:GenomeRecipe',`
`50`	`50`	`'blast_recipe = wfcommons.wfchef.recipes:BlastRecipe',`
`51`	`51`	`'bwa_recipe = wfcommons.wfchef.recipes:BwaRecipe',`
	`52`	`+ 'rnaseq_recipe = wfcommons.wfchef.recipes:RnaseqRecipe',`
`52`	`53`	`]`
`53`	`54`	`},`
`54`	`55`	`)`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`#!/usr/bin/env python`
`2`	`2`	`# -- coding: utf-8 --`
`3`	`3`	`#`
`4`		`-# Copyright (c) 2024 The WfCommons Team.`
	`4`	`+# Copyright (c) 2024-2025 The WfCommons Team.`
`5`	`5`	`#`
`6`	`6`	`# This program is free software: you can redistribute it and/or modify`
`7`	`7`	`# it under the terms of the GNU General Public License as published by`
`@@ -41,7 +41,7 @@ def task(self) -> Task:`
`41`	`41`	`energy=100,`
`42`	`42`	`avg_power=1.0,`
`43`	`43`	`priority=100,`
`44`		`- executedAt="2024-09-15T08:59:33.699321-04:00",`
	`44`	`+ executed_at="2024-09-15T08:59:33.699321-04:00",`
`45`	`45`	`task_type=TaskType.COMPUTE,`
`46`	`46`	`launch_dir="/tmp",`
`47`	`47`	`)`