tmp commit

Author: henricasanova

tests/translators_loggers/Dockerfile.streamflow

@@ -0,0 +1,47 @@
+# docker build --platform amd64 -t wfcommons-dev-streamflow -f Dockerfile.streamflow .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-streamflow /bin/bash
+
+FROM alphaunito/streamflow
+
+LABEL org.containers.image.authors="henric@hawaii.edu"
+
+# update repositories
+RUN apt-get update
+
+# set timezone
+RUN echo "America/Los_Angeles" > /etc/timezone && export DEBIAN_FRONTEND=noninteractive && apt-get install -y tzdata
+
+# install useful stuff
+RUN apt-get -y install pkg-config 
+RUN apt-get -y install git
+RUN apt-get -y install wget 
+RUN apt-get -y install make 
+RUN apt-get -y install cmake 
+RUN apt-get -y install cmake-data 
+RUN apt-get -y install sudo 
+RUN apt-get -y install vim --fix-missing
+RUN apt-get -y install gcc 
+RUN apt-get -y install gcc-multilib
+
+# Python stuff
+RUN apt-get -y install python3 python3-pip
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+RUN python3 -m pip install --break-system-packages pathos pandas filelock
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
+RUN python3 -m pip install --break-system-packages --upgrade setuptools
+
+# Stress-ng
+RUN apt-get -y install stress-ng
+
+# Add wfcommons user
+RUN useradd -ms /bin/bash wfcommons
+RUN adduser wfcommons sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+ENV PATH="$PATH:/home/wfcommons/.local/bin/"
+
+USER wfcommons
+WORKDIR /home/wfcommons
+# Making this directory world rwx to facilitate testing
+RUN chmod -R 777 /home/wfcommons
+

tests/translators_loggers/test_translators_loggers.py

@@ -114,6 +114,7 @@ def _additional_setup_swiftt(container):
     "taskvine": _additional_setup_taskvine,
     "makeflow": noop,
     "cwl": noop,
+    "streamflow": noop,
     "pegasus": _additional_setup_pegasus,
     "swiftt": _additional_setup_swiftt,
 }
@@ -189,6 +190,19 @@ def run_workflow_cwl(container, num_tasks, str_dirpath):
     # and there is a 2* because there is a message for the job and for the step)
     assert (output.decode().count("completed success") == 3 + 2 *num_tasks)
 
+def run_workflow_streamflow(container, num_tasks, str_dirpath):
+    # Run the workflow!
+    # Note that the input file is hardcoded and Blast-specific
+    sys.stderr.write("TODO: RUN THE STREAMFLOW WORKFLOW!!!")
+    time.sleep(100000)
+    # exit_code, output = container.exec_run(cmd="cwltool ./main.cwl --split_fasta_00000001_input ./data/workflow_infile_0001 ",
+    #                                        user="wfcommons", stdout=True, stderr=True)
+    # # Check sanity
+    # assert (exit_code == 0)
+    # # this below is ugly (the 3 is for "workflow", "compile_output_files" and "compile_log_files",
+    # # and there is a 2* because there is a message for the job and for the step)
+    # assert (output.decode().count("completed success") == 3 + 2 *num_tasks)
+
 def run_workflow_pegasus(container, num_tasks, str_dirpath):
     # Run the workflow!
     exit_code, output = container.exec_run(cmd="bash /home/wfcommons/run_workflow.sh",
@@ -217,6 +231,7 @@ def run_workflow_swiftt(container, num_tasks, str_dirpath):
     "taskvine": run_workflow_taskvine,
     "makeflow": run_workflow_makeflow,
     "cwl": run_workflow_cwl,
+    "streamflow": run_workflow_streamflow,
     "pegasus": run_workflow_pegasus,
     "swiftt": run_workflow_swiftt,
 }
@@ -231,6 +246,7 @@ def run_workflow_swiftt(container, num_tasks, str_dirpath):
     "taskvine": TaskVineTranslator,
     "makeflow": MakeflowTranslator,
     "cwl": CWLTranslator,
+    "streamflow": CWLTranslator,
     "pegasus": PegasusTranslator,
     "swiftt": SwiftTTranslator,
 }
@@ -241,17 +257,18 @@ class TestTranslators:
     @pytest.mark.parametrize(
         "backend",
         [
-           "swiftt",
-           "dask",
-           "parsl",
-           "nextflow",
-           "nextflow_subworkflow",
-           "airflow",
-           "bash",
-           "taskvine",
-           "makeflow",
-           "cwl",
-           "pegasus",
+           # "swiftt",
+           # "dask",
+           # "parsl",
+           # "nextflow",
+           # "nextflow_subworkflow",
+           # "airflow",
+           # "bash",
+           # "taskvine",
+           # "makeflow",
+           # "cwl",
+           "streamflow",
+           # "pegasus",
         ])
     @pytest.mark.unit
     # @pytest.mark.skip(reason="tmp")
@@ -305,6 +322,8 @@ def test_translator(self, backend) -> None:
             parser = TaskVineLogsParser(dirpath / "vine-run-info/most-recent/vine-logs", filenames_to_ignore=["cpu-benchmark","stress-ng", "wfbench"])
         elif backend == "makeflow":
             parser = MakeflowLogsParser(execution_dir = dirpath, resource_monitor_logs_dir = dirpath / "monitor_data/")
+        elif backend == "streamflow":
+            parsed =
         else:
             parser = None