Merge branch 'main' into stress-ng_cpu_benchmark

Author: henricasanova

docs/source/dev_api_wfinstances.rst

@@ -50,6 +50,16 @@ wfcommons.wfinstances.logs.makeflow
    :private-members:
    :noindex:
 
+wfcommons.wfinstances.logs.taskvine
+-----------------------------------
+
+.. automodule:: wfcommons.wfinstances.logs.taskvine
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :private-members:
+   :noindex:
+
 wfcommons.wfinstances.logs.nextflow
 -----------------------------------
 

docs/source/user_api_wfbench.rst

@@ -38,6 +38,14 @@ wfcommons.wfbench.translator.cwl
    :undoc-members:
    :show-inheritance:
 
+wfcommons.wfbench.translator.streamflow
+--------------------------------
+
+.. automodule:: wfcommons.wfbench.translator.streamflow
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 wfcommons.wfbench.translator.dask
 ---------------------------------
 

pyproject.toml

@@ -26,6 +26,7 @@ license = {text = "GNU General Public License v3 (GPLv3)"}
 dependencies = [
     "jsonschema",
     "matplotlib",
+    "pygraphviz",
     "networkx",
     "numpy",
     "pandas",

tests/test_helpers.py

@@ -20,6 +20,11 @@
 
 from wfcommons.common import Workflow
 
+def _recursive_chmod(dirpath, permissions):
+            for directory, directory_name, filenames in os.walk(dirpath):
+                os.chmod(directory, permissions)
+                for filename in filenames:
+                    os.chmod(os.path.join(directory, filename), permissions)
 
 def _create_fresh_local_dir(path: str) -> pathlib.Path:
     dirpath = pathlib.Path(path)
@@ -89,6 +94,7 @@ def _start_docker_container(backend, mounted_dir, working_dir, bin_dir, command=
         volumes={mounted_dir: {'bind': mounted_dir, 'mode': 'rw'}},
         working_dir=working_dir,
         user="wfcommons",
+        privileged=True,
         tty=True,
         detach=True,
         init=True  # For zombies

tests/translators_loggers/Dockerfile.airflow

@@ -1,5 +1,5 @@
-# docker build -t wfcommons-dev -f Dockerfile.airflow .
-# docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev /bin/bash
+# docker build -t wfcommons-dev-airflow -f Dockerfile.airflow .
+# docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev-airflow /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -21,12 +21,14 @@ RUN apt-get -y install cmake-data
 RUN apt-get -y install sudo 
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
+RUN apt-get -y install graphviz libgraphviz-dev
+
 #RUN apt-get -y install gcc-multilib
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 

tests/translators_loggers/Dockerfile.bash

@@ -22,12 +22,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.cwl

@@ -1,5 +1,5 @@
-# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile.dask .
-# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-dask /bin/bash
+# docker build --platform amd64 -t wfcommons-dev-cwl -f Dockerfile.cwl .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-cwl /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -22,12 +22,13 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.dask

@@ -22,12 +22,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.makeflow

@@ -1,5 +1,5 @@
-# docker build --platform amd64 -t wfcommons-dev -f Dockerfile.parsl .
-# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev /bin/bash
+# docker build --platform amd64 -t wfcommons-dev-makeflow -f Dockerfile.makeflow .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-makeflow /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -23,12 +23,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.nextflow

@@ -23,12 +23,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN pip install --break-system-packages pathos pandas filelock 
-RUN pip install --break-system-packages networkx scipy matplotlib
+RUN pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools