Shoehorned that use case of hell

Author: chahuistle

com.workflowconversion.knime2grid/src/com/workflowconversion/knime2grid/export/node/impl/DefaultKnimeNodeConverter.java

@@ -169,9 +169,7 @@ public Job convert(final NativeNodeContainer nativeNodeContainer, final Workflow
 				// input.setMultiFile(true);
 				LOGGER.info("Creating FileInput");
 				nodeFactory = new MimeFileImporterNodeFactory();
-				inputSettings.add(new VariableSetting("FILENAME", inputFileKey, "tmpfile.txt"));
 				final String extensionKey = "extension" + currentInput;
-				inputSettings.add(new VariableSetting("FILE_EXTENSION", extensionKey));
 				final NodeContainer sourceNode = workflowManager.getNodeContainer(sourceNodeId);
 				final NodeOutPort sourcePort = sourceNode.getOutPort(connectionContainer.getSourcePort());
 				// make sure that the origin is indeed a IURIPortObject!
@@ -184,11 +182,15 @@ public Job convert(final NativeNodeContainer nativeNodeContainer, final Workflow
 				} else {
 					throw new RuntimeException("The port types of the source and destination port do not match");
 				}
+				inputSettings.add(new VariableSetting("FILE_EXTENSION", extensionKey));
+				// [hacking intensifies]
+				inputSettings.add(new VariableSetting("FILENAME", inputFileKey + extension, "tmpfile.txt"));
 			} else {
 				// not sure what the hell should we do here...
 				// TODO: is it ok to assume that model writer is fine?
 				LOGGER.info("PortType " + inPortObjectClass.getName());
 				nodeFactory = new PortObjectReaderNodeFactory(portType);
+				// [hacking intensifies]
 				inputSettings.add(new VariableSetting("filename", inputFileKey));
 			}
 			// an extension might have been added

com.workflowconversion.knime2grid/src/com/workflowconversion/knime2grid/export/node/impl/GenericKnimeNodeConverter.java

@@ -10,8 +10,10 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -78,14 +80,17 @@ public Job convert(final NativeNodeContainer nativeNodeContainer, final Workflow
 
 		// input/output ports are also handled as parameters in GKN, so we need to keep track of
 		// the ports that we've processed
-		final Set<String> processedPortNames = new TreeSet<String>();
+		final Set<String> processedPortNames = new TreeSet<>();
+		// since we will name converted job inputs using extensions, we need a map to relate GKN port names to converted
+		// inputs
+		final Map<String, com.workflowconversion.knime2grid.model.Port> gknPortToConvertedPort = new HashMap<>();
 
 		// all inputs and outputs of GKNs are URIs, so this simplifies the conversion
 		// we need to know how to actually execute this job on a command line environment
 		// so we need to access the command generator... however, at this point, there
 		// is no guarantee that all jobs have been converted so we need to use the WorkflowManager to
 		// gather information about other nodes
-		createInputsAndOutputsFromKnimeWorkflow(nativeNodeContainer, workflowManager, nodeConfiguration, job, processedPortNames);
+		createInputsAndOutputsFromKnimeWorkflow(nativeNodeContainer, workflowManager, nodeConfiguration, job, processedPortNames, gknPortToConvertedPort);
 
 		// process all non-input/non-output parameters AND the CTD, if any.
 		// this must be AFTER inputs/outputs have been processed, because we are adding
@@ -98,7 +103,7 @@ public Job convert(final NativeNodeContainer nativeNodeContainer, final Workflow
 					throw new ApplicationException(
 							"This job already has a CTD file. Only one CTD file per job is allowed. This is probably a bug and should be reported.");
 				}
-				addCTDInputPort(workflowManager, (CommandLineCTDFile) element, job, nodeConfiguration, nativeNodeContainer);
+				addCTDInputPort(workflowManager, (CommandLineCTDFile) element, job, nodeConfiguration, nativeNodeContainer, gknPortToConvertedPort);
 				ctdFound = true;
 			} else if (element instanceof ParametrizedCommandLineElement && !processedPortNames.contains(element.getKey())) {
 				// we need to process only true parameters, not flags or option identifiers
@@ -124,18 +129,20 @@ public Job convert(final NativeNodeContainer nativeNodeContainer, final Workflow
 	}
 
 	private void createInputsAndOutputsFromKnimeWorkflow(final NativeNodeContainer nativeNodeContainer, final WorkflowManager workflowManager,
-			final INodeConfiguration nodeConfiguration, final Job job, final Set<String> processedPortNames) {
+			final INodeConfiguration nodeConfiguration, final Job job, final Set<String> processedPortNames,
+			final Map<String, com.workflowconversion.knime2grid.model.Port> gknPortToConvertedPort) {
 		for (final ConnectionContainer connectionContainer : workflowManager.getIncomingConnectionsFor(nativeNodeContainer.getID())) {
 			final NodeID sourceNodeId = connectionContainer.getSource();
 			final Node sourceNode = ((NativeNodeContainer) workflowManager.getNodeContainer(connectionContainer.getSource())).getNode();
 			final int destPortNr = connectionContainer.getDestPort();
 			final Port destPort = nodeConfiguration.getInputPorts().get(ConverterUtils.convertFromKnimePort(destPortNr));
-			final Input input = new Input();
-			input.setSourceId(sourceNodeId);
-			input.setOriginalPortNr(destPortNr);
-			input.setName(destPort.getName() + getExtensionForPort(sourceNode, connectionContainer.getSourcePort()));
+			final Input convertedInput = new Input();
+			convertedInput.setSourceId(sourceNodeId);
+			convertedInput.setOriginalPortNr(destPortNr);
+			convertedInput.setName(destPort.getName() + getExtensionForPort(sourceNode, connectionContainer.getSourcePort()));
+			gknPortToConvertedPort.put(destPort.getName(), convertedInput);
 			// input.setMultiFile(destPort.isMultiFile());
-			job.addInput(input);
+			job.addInput(convertedInput);
 
 			processedPortNames.add(destPort.getName());
 		}
@@ -144,12 +151,13 @@ private void createInputsAndOutputsFromKnimeWorkflow(final NativeNodeContainer n
 			if (job.getOutputByOriginalPortNr(sourcePortNr) == null) {
 				// first time we see the output, we need to add it
 				final Port sourcePort = nodeConfiguration.getOutputPorts().get(ConverterUtils.convertFromKnimePort(sourcePortNr));
-				final Output newOutput = new Output();
+				final Output convertedOutput = new Output();
 				final Node sourceNode = ((NativeNodeContainer) workflowManager.getNodeContainer(connectionContainer.getSource())).getNode();
 				// newOutput.setMultiFile(sourcePort.isMultiFile());
-				newOutput.setName(sourcePort.getName() + getExtensionForPort(sourceNode, connectionContainer.getSourcePort()));
-				newOutput.setOriginalPortNr(sourcePortNr);
-				job.addOutput(newOutput);
+				convertedOutput.setName(sourcePort.getName() + getExtensionForPort(sourceNode, connectionContainer.getSourcePort()));
+				convertedOutput.setOriginalPortNr(sourcePortNr);
+				gknPortToConvertedPort.put(sourcePort.getName(), convertedOutput);
+				job.addOutput(convertedOutput);
 				processedPortNames.add(sourcePort.getName());
 			}
 		}
@@ -158,7 +166,8 @@ private void createInputsAndOutputsFromKnimeWorkflow(final NativeNodeContainer n
 	// when executing GKN in KNIME, each GKN generates an "on the fly" CTD file and uses it to
 	// execute the associated binary, but what we need here is to add a new input containing a CTD
 	private void addCTDInputPort(final WorkflowManager workflowManager, final CommandLineCTDFile element, final Job job,
-			final INodeConfiguration nodeConfiguration, final NativeNodeContainer nativeNodeContainer) throws IOException, InvalidCTDFileException {
+			final INodeConfiguration nodeConfiguration, final NativeNodeContainer nativeNodeContainer,
+			final Map<String, com.workflowconversion.knime2grid.model.Port> gknPortToConvertedPort) throws IOException, InvalidCTDFileException {
 		final Input ctdInput = new Input();
 		ctdInput.setName(CommandLineCTDFile.CTD_FILE_KEY);
 		ctdInput.setConnectionType(ConnectionType.UserProvided);
@@ -171,8 +180,21 @@ private void addCTDInputPort(final WorkflowManager workflowManager, final Comman
 				dumpConfiguration(clonedNodeConfiguration).getCanonicalPath());
 		ctdInput.setAssociatedFileParameter(ctdFileParameter);
 		job.addInput(ctdInput);
-		// fix the command line element!
+		// fix the command line element
 		element.setValue(ctdFileParameter);
+		// fix converted Input/Outputs
+		transferToConvertedPorts(clonedNodeConfiguration, gknPortToConvertedPort);
+	}
+
+	private void transferToConvertedPorts(final INodeConfiguration clonedNodeConfiguration,
+			final Map<String, com.workflowconversion.knime2grid.model.Port> gknPortToConvertedPort) {
+		for (final Map.Entry<String, com.workflowconversion.knime2grid.model.Port> entry : gknPortToConvertedPort.entrySet()) {
+			final Parameter<?> parameter = clonedNodeConfiguration.getParameter(entry.getKey());
+			if (parameter == null || !(parameter instanceof IFileParameter)) {
+				throw new ApplicationException("Invalid contents of map relating GKN ports to converted inputs. This is a bug and should be reported.");
+			}
+			entry.getValue().setAssociatedFileParameter((IFileParameter) parameter);
+		}
 	}
 
 	private File dumpConfiguration(final INodeConfiguration nodeConfiguration) throws IOException {
@@ -270,21 +292,18 @@ private void fixFilePathsFromAssociatedParameters(final WorkflowManager workflow
 						processedPorts.add(parameterName);
 					}
 				} else if (fileNames.size() > 1) {
-					if (associatedParameter.getValue() != null) {
-						// multifile
-						final List<String> fixedFilenames = new LinkedList<String>();
-						int fileNumber = 0;
-						for (final String fileName : fileNames) {
-							final String extension = FilenameUtils.getExtension(fileName);
-							fixedFilenames.add(ConverterUtils.generateFileNameForExport(parameterName, extension, fileNumber));
-							fileNumber++;
-						}
-						((FileListParameter) associatedParameter).setValue(fixedFilenames);
-						processedPorts.add(parameterName);
+					final List<String> fixedFilenames = new LinkedList<String>();
+					int fileNumber = 0;
+					for (final String fileName : fileNames) {
+						final String extension = FilenameUtils.getExtension(fileName);
+						fixedFilenames.add(ConverterUtils.generateFileNameForExport(parameterName, extension, fileNumber));
+						fileNumber++;
 					}
+					((FileListParameter) associatedParameter).setValue(fixedFilenames);
+					processedPorts.add(parameterName);
 				} else {
-					// 0 inputs?
-					throw new RuntimeException("Invalid association between parameters and input files. This is probably a bug, please report it.");
+					// 0 inputs!
+					// TODO: what does this mean???
 				}
 			}
 		}

com.workflowconversion.knime2grid/src/com/workflowconversion/knime2grid/export/workflow/impl/guse/GuseKnimeWorkflowExporter.java

@@ -67,6 +67,7 @@
 import com.workflowconversion.knime2grid.model.ConnectionType;
 import com.workflowconversion.knime2grid.model.Input;
 import com.workflowconversion.knime2grid.model.Job;
+import com.workflowconversion.knime2grid.model.JobType;
 import com.workflowconversion.knime2grid.model.Output;
 import com.workflowconversion.knime2grid.model.Port;
 import com.workflowconversion.knime2grid.model.Workflow;
@@ -84,6 +85,8 @@ public class GuseKnimeWorkflowExporter implements KnimeWorkflowExporter {
 	private static final String INPUT_PORTS_WITH_FILELIST_SCRIPT_KEY = "@@INPUT_PORTS_WITH_FILELIST@@";
 	private static final String OUTPUT_PORTS_WITH_FILELIST_SCRIPT_KEY = "@@OUTPUT_PORTS_WITH_FILELIST@@";
 	private static final String COMMAND_LINE_PARAMETERS_SCRIPT_KEY = "@@COMMAND_LINE_PARAMETERS@@";
+	private static final String INPUT_FILENAME_TRANSLATION_SCRIPT_KEY = "@@INPUT_FILENAME_TRANSLATION@@";
+	private static final String FILENAME_TRANSLATION_VAR_PREFIX = "KNIME2GRID_VAR_";
 	private static final String QUOTE_REGEX = "\"";
 	private static final String QUOTE_REPLACEMENT_FOR_BASH_SCRIPT = "\\\"";
 	private static final String LOCAL_EXECUTOR_TYPE = "local";
@@ -251,7 +254,7 @@ private void fixDuplicateJobName(final Map<String, Integer> nameOccurrenceMap, f
 	}
 
 	private void fixCollectorGeneratorJob(final Job job) {
-		// set to use the local executor
+		// "hardcode" to set to use the local executor
 		switch (job.getJobType()) {
 			case Generator :
 			case Collector :
@@ -311,40 +314,64 @@ private void writeExecuteBin(final String rootEntryName, final ZipOutputStream z
 	}
 
 	private String generateGeneratorScript(final Job job) throws IOException {
-		return loadScript("zip_loop_start.sh", "@@PORT_NAME@@", job.getInputByPortNr(0).getName());
+		// TODO: this is hackish, we know (assume) that generator jobs have one input and one output
+		final Input input = job.getInputs().iterator().next();
+		final Output output = job.getOutputs().iterator().next();
+		return loadScript("zip_loop_start.sh", "@@INPUT_PORT_NAME@@", fixPortName(input), "@@OUTPUT_BASE_NAME@@", fixPortName(output));
 	}
 
 	private String generateCollectorScript(final Job job) throws IOException {
-		return loadScript("zip_loop_end.sh", "@@BASE_PORT_NAME@@", job.getInputByPortNr(0).getName());
+		// TODO: this is hackish, we know (assume) that collector jobs have one input and one output
+		final Input input = job.getInputs().iterator().next();
+		final Output output = job.getOutputs().iterator().next();
+		return loadScript("zip_loop_end.sh", "@@INPUT_BASE_NAME@@", fixPortName(input), "@@OUTPUT_PORT_NAME@@", fixPortName(output));
 	}
 
 	private String generateDefaultScript(final Job job) throws IOException {
 		final StringBuilder fileListInputs = new StringBuilder();
 		final StringBuilder fileListOutputs = new StringBuilder();
+		final StringBuilder fileNameTranslation = new StringBuilder();
+		// see comment on job_wrapper.sh
+		final StringBuilder majorHackett = new StringBuilder();
 
+		// not related to port number, this is just a hack for the wrapper script
+		int scriptPortIndex = 0;
 		for (final Input input : job.getInputs()) {
 			if (input.isMultiFile()) {
 				if (fileListInputs.length() > 0) {
 					// not the first element, we can prepend a space
 					fileListInputs.append(' ');
 				}
-				fileListInputs.append(input.getName());
+				fileListInputs.append(fixPortName(input));
+				fileNameTranslation.append(FILENAME_TRANSLATION_VAR_PREFIX).append(scriptPortIndex).append("=\"");
+				// only place outside method fixPortName where we use the "base name" of the port!
+				fileNameTranslation.append(input.getName());
+				// gUSE runs on Linux, it might be incorrect to use System.getProperty("line.separator")
+				fileNameTranslation.append("\"\n");
+				scriptPortIndex++;
 			}
 		}
 		for (final Output output : job.getOutputs()) {
 			if (output.isMultiFile()) {
 				if (fileListOutputs.length() > 0) {
 					fileListOutputs.append(' ');
 				}
-				fileListOutputs.append(output.getName());
+				fileListOutputs.append(fixPortName(output));
+			}
+			if (job.getJobType() == JobType.KnimeInternal) {
+				if (majorHackett.length() > 0) {
+					majorHackett.append(' ');
+				}
+				majorHackett.append(fixPortName(output));
 			}
 		}
 
 		// script handles empty variables for input/output ports with filelist, make sure to escape the command line,
 		// which is not 100% under the control of this class
 		return loadScript("job_wrapper.sh", EXECUTABLE_SCRIPT_KEY, job.getRemoteApplication().getPath(), INPUT_PORTS_WITH_FILELIST_SCRIPT_KEY,
 				fileListInputs.toString(), OUTPUT_PORTS_WITH_FILELIST_SCRIPT_KEY, fileListOutputs.toString(), COMMAND_LINE_PARAMETERS_SCRIPT_KEY,
-				generateCommandLine(job).replace(QUOTE_REGEX, QUOTE_REPLACEMENT_FOR_BASH_SCRIPT));
+				generateCommandLine(job).replace(QUOTE_REGEX, QUOTE_REPLACEMENT_FOR_BASH_SCRIPT), INPUT_FILENAME_TRANSLATION_SCRIPT_KEY,
+				fileNameTranslation.toString(), "@@MAJOR_HACKETT@@", majorHackett.toString());
 	}
 
 	// loads a script from file,
@@ -565,15 +592,15 @@ private void addConcreteInputPortProperties(final Element inputElement, final In
 				addConcretePortProperty(inputElement, "file", "knime2grid.file");
 				addConcretePortProperty(inputElement, "eparam", "0");
 				addConcretePortProperty(inputElement, "pequaltype", "0");
-				addConcretePortProperty(inputElement, "intname", input.getName());
+				addConcretePortProperty(inputElement, "intname", fixPortName(input));
 				addConcretePortProperty(inputElement, "dpid", Integer.toString(input.getPortNr()));
 				break;
 			case Collector :
 				addConcretePortProperty(inputElement, "waitingtmp", "all");
 				addConcretePortProperty(inputElement, "waiting", "all");
 				addConcretePortProperty(inputElement, "eparam", "1");
 				addConcretePortProperty(inputElement, "pequaltype", "0");
-				addConcretePortProperty(inputElement, "intname", input.getName());
+				addConcretePortProperty(inputElement, "intname", fixPortName(input));
 				addConcretePortProperty(inputElement, "dpid", Integer.toString(input.getPortNr()));
 				break;
 			default :

com.workflowconversion.knime2grid/src/com/workflowconversion/knime2grid/export/workflow/impl/guse/job_wrapper.sh

@@ -3,25 +3,63 @@
 
 # contains names of input ports that take filelists, separated by whitespace
 INPUT_PORTS_WITH_FILELIST="@@INPUT_PORTS_WITH_FILELIST@@"
+
+# we cannot assume that the contents of the archive contain the input files named exactly as expected
+# this section relates input port names to a (base) filename to use to change the names of the extracted files, e.g.,
+#   KNIME2GRID_VAR_N="bar"
+# signifies that the the contents of the N-th input will be extracted and renamed to 0_bar, 1_bar, ...
+##### start filename translation variables 
+@@INPUT_FILENAME_TRANSLATION@@
+##### end
+
 # contains names of output ports that generate filelists, separated by whitespace
 OUTPUT_PORTS_WITH_FILELIST="@@OUTPUT_PORTS_WITH_FILELIST@@"
 EXECUTABLE="@@EXECUTABLE@@"
 COMMAND_LINE_PARAMETERS="@@COMMAND_LINE_PARAMETERS@@"
 
+# KNIME 3.6 seems to have a race condition, sometimes KNIME reports that the node "Table Reader" is not available, but it is...
+# This is probably not best practice, not regular practice, hell, this should not be practice, but basically what we do here
+# is to wait for ALL outputs...
+# Major Hackett made me do it, I am sorry. The chain of command is sacred.
+MAJOR_HACKETT="@@MAJOR_HACKETT@@"
+# Hackett out!
+
+ARCHIVE_INDEX=0
 if [ -n "$INPUT_PORTS_WITH_FILELIST" ]; then
 	for input_port in ${INPUT_PORTS_WITH_FILELIST}; do
 		echo "expanding ${input_port}"
-		echo tar xfz ${input_port}
+		# extract files individually and rename them
+		FILE_INDEX=0
+		BASENAME_VARIABLE_NAME="KNIME2GRID_VAR_${FILE_INDEX}"
+		for input_file in `tar tfz ${input_port}`; do
+			# use basename and index to rename the file as its written to stdout
+			tar xOfz ${input_port} ${input_file} > ${FILE_INDEX}_${!BASENAME_VARIABLE_NAME}
+			FILE_INDEX=$(expr ${FILE_INDEX} + 1)
+		done
+		ARCHIVE_INDEX=$(expr ${ARCHIVE_INDEX} + 1)
 	done
 fi
 
-# execute the tool
-echo "Executing: ${EXECUTABLE} ${COMMAND_LINE_PARAMETERS}"
-${EXECUTABLE} ${COMMAND_LINE_PARAMETERS}
+# execute the tool using Major Hackett's approach
+N_ATTEMPTS=1
+HACKETT_OUT=""
+while [  -z "${HACKETT_OUT}"  ]; do
+	echo "Executing(${N_ATTEMPTS}): ${EXECUTABLE} ${COMMAND_LINE_PARAMETERS}"
+	N_ATTEMPTS=$(expr ${N_ATTEMPTS} + 1)
+	${EXECUTABLE} ${COMMAND_LINE_PARAMETERS}
+	HACKETT_OUT="yes"
+	for hackettinno in ${MAJOR_HACKETT}; do
+		if [ ! -s ${hackettinno} ]; then
+			# expected output not found or size zero
+			HACKETT_OUT=""
+			sleep 2
+			break;
+		fi
+	done
+done
+
 
-# compress the multi-file outputs and make sure to name the archive using port name and .tar.gz as extension
-# see: com.workflowconversion.knime2grid.export.workflow.ConverterUtils.generateFileNameForExport(String, String, int) and
-#      com.workflowconversion.knime2grid.export.workflow.impl.guse.GuseKnimeWorkflowExporter.fixPortName(Port)
+# compress the multi-file outputs
 if [ -n "$OUTPUT_PORTS_WITH_FILELIST" ]; then
 	for output_port in ${OUTPUT_PORTS_WITH_FILELIST}; do
 		echo "compressing outputs for ${output_port}"