Skip to content

Commit c368393

Browse files
henricasanovahenricasanova
committed
Implemented a Makeflow Translator
1 parent 46ccda1 commit c368393

3 files changed

Lines changed: 94 additions & 25 deletions

File tree

‎tests/translators_loggers/Dockerfile.makeflow‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# docker build --platform amd64 -t wfcommons-dev -f Dockerfile.parsl .
2+
# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev /bin/bash
3+
4+
FROM amd64/ubuntu:noble
5+
6+
LABEL org.containers.image.authors="henric@hawaii.edu"
7+
8+
# update repositories
9+
RUN apt-get update
10+
11+
# set timezone
12+
RUN echo "America/Los_Angeles" > /etc/timezone && export DEBIAN_FRONTEND=noninteractive && apt-get install -y tzdata
13+
14+
# install useful stuff
15+
RUN apt-get -y install pkg-config
16+
RUN apt-get -y install git
17+
RUN apt-get -y install wget
18+
RUN apt-get -y install curl
19+
RUN apt-get -y install make
20+
RUN apt-get -y install cmake
21+
RUN apt-get -y install cmake-data
22+
RUN apt-get -y install sudo
23+
RUN apt-get -y install vim --fix-missing
24+
RUN apt-get -y install gcc
25+
RUN apt-get -y install gcc-multilib
26+
27+
# Python stuff
28+
RUN apt-get -y install python3 python3-pip
29+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
30+
RUN python3 -m pip install --break-system-packages pathos pandas filelock
31+
RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
32+
RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
33+
RUN python3 -m pip install --break-system-packages --upgrade setuptools
34+
35+
# Stress-ng
36+
RUN apt-get -y install stress-ng
37+
38+
# Add wfcommons user
39+
RUN useradd -ms /bin/bash wfcommons
40+
RUN adduser wfcommons sudo
41+
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
42+
ENV PATH="$PATH:/home/wfcommons/.local/bin/"
43+
44+
USER wfcommons
45+
WORKDIR /home/wfcommons
46+
47+
# Install Miniforge
48+
RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
49+
bash Miniforge3.sh -b -p "${HOME}/conda" && \
50+
rm Miniforge3.sh
51+
52+
# Make sure conda is available in each shell session
53+
RUN echo "source ${HOME}/conda/etc/profile.d/conda.sh" >> ${HOME}/.bashrc
54+
RUN echo "conda activate" >> ${HOME}/.bashrc
55+
56+
# Install necessary packages
57+
RUN . ${HOME}/conda/etc/profile.d/conda.sh && \
58+
conda activate base && \
59+
conda install -c conda-forge ndcctools && \
60+
conda clean --all -f -y
61+

‎tests/translators_loggers/test_translators_loggers.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import sys
1515
import json
1616
import time
17+
import re
1718
import networkx
1819

1920
from tests.test_helpers import _create_fresh_local_dir
@@ -164,12 +165,12 @@ def run_workflow_taskvine(container, num_tasks, str_dirpath):
164165
assert (output.decode().count("completed") == num_tasks)
165166

166167
def run_workflow_makeflow(container, num_tasks, str_dirpath):
167-
# Run the workflow!
168-
exit_code, output = container.exec_run(cmd=["bash", "-c", "makeflow ./workflow.makeflow"], stdout=True, stderr=True)
168+
# Run the workflow (with full logging)
169+
exit_code, output = container.exec_run(cmd=["bash", "-c", "source ~/conda/etc/profile.d/conda.sh && conda activate && makeflow --log-verbose ./workflow.makeflow"], stdout=True, stderr=True)
169170
# Check sanity
170171
assert (exit_code == 0)
171-
sys.stderr.write("SHOULD BE CHECKING SANITY")
172-
# assert (output.decode().count("completed") == num_tasks)
172+
num_completed_jobs = len(re.findall(r'job \d+ completed', output.decode()))
173+
assert (num_completed_jobs == num_tasks)
173174

174175
def run_workflow_cwl(container, num_tasks, str_dirpath):
175176
# Run the workflow!

‎wfcommons/wfbench/translator/makeflow.py‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919

2020
this_dir = pathlib.Path(__file__).resolve().parent
2121

22-
2322
class MakeflowTranslator(Translator):
2423
"""
2524
A WfFormat parser for creating Makeflow workflow applications.
@@ -71,30 +70,40 @@ def _generate_code(self):
7170
make_clause = ""
7271
# output files
7372
for output_file in task.output_files:
74-
make_clause += f"{output_file.file_id} "
73+
make_clause += f"data/{output_file.file_id} "
7574
make_clause += ": "
7675
# input files
7776
for input_file in task.input_files:
78-
make_clause += f"{input_file.file_id} "
77+
make_clause += f"data/{input_file.file_id} "
7978
make_clause += "\n"
79+
# Command
8080
make_clause += "\t"
81-
make_clause += task.program + " " + " ".join(task.args)
82-
make_clause += "\n"
81+
make_clause += task.program + " "
82+
83+
input_spec = "\"["
84+
for file in task.input_files:
85+
input_spec += f"\\\\\"data/{file.file_id}\\\\\","
86+
input_spec = input_spec[:-1] + "]\""
87+
88+
output_spec = "\"{"
89+
for file in task.output_files:
90+
output_spec += f"\\\\\"data/{file.file_id}\\\\\":{str(file.size)},"
91+
output_spec = output_spec[:-1] + "}\""
92+
93+
args = []
94+
for a in task.args:
95+
if "--output-files" in a:
96+
args.append(f"--output-files {output_spec}")
97+
elif "--input-files" in a:
98+
args.append(f"--input-files {input_spec}")
99+
else:
100+
args.append(a)
101+
102+
args = " ".join(f"{a}" for a in args)
103+
make_clause += args + "\n"
83104
self._script += make_clause + "\n\n"
84105
return
85106

86-
# OLD CODE FOR TASK VINE...
87-
# args = []
88-
# for a in task.args:
89-
# if "--output-files" in a:
90-
# args.append(f"--output-files {output_spec}")
91-
# elif "--input-files" in a:
92-
# args.append(f"--input-files {input_spec}")
93-
# else:
94-
# args.append(a)
95-
# args = " ".join(f"{a}" for a in args)
96-
97-
98107
def _write_readme_file(self, output_folder: pathlib.Path) -> None:
99108
"""
100109
Write the README file.
@@ -105,7 +114,5 @@ def _write_readme_file(self, output_folder: pathlib.Path) -> None:
105114
readme_file_path = output_folder.joinpath("README")
106115
with open(readme_file_path, "w") as out:
107116
out.write(f"In directory {str(output_folder)}:\n")
108-
out.write(f" - The Makeflow input file: "
109-
f" workflow.makeflow\n")
110-
out.write(f" - Run the workflow: "
111-
f" makeflow workflow.makeflow\n")
117+
out.write(f" - The Makeflow input file: workflow.makeflow\n")
118+
out.write(f" - Run the workflow: makeflow workflow.makeflow\n")

0 commit comments

Comments
 (0)