Add more support for scatter/gather

Author: bbimber

mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java

@@ -422,14 +422,14 @@ private Map<String, String> getSampleToAlias(File input) throws PipelineJobExcep
     }
 
     @Override
-    public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List<File> orderedScatterOutputs) throws PipelineJobException
+    public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List<File> orderedScatterOutputs, List<String> orderedJobDirs) throws PipelineJobException
     {
         job.getLogger().info("Merging additional track VCFs");
         Map<String, List<String>> trackToSamples = parseSampleMap(getSampleNameFile(getPipelineCtx().getSourceDirectory(true)));
         for (String trackName : trackToSamples.keySet())
         {
-            List<File> toConcat = orderedScatterOutputs.stream().map(f -> {
-                f = getOutputVcf(trackName, f.getParentFile());
+            List<File> toConcat = orderedJobDirs.stream().map(dirName -> {
+                File f = getOutputVcf(trackName, new File(ctx.getWorkingDirectory(), dirName));
                 if (!f.exists())
                 {
                     throw new IllegalStateException("Missing file: " + f.getPath());

mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java

@@ -83,7 +83,7 @@ public GroupCompareStep create(PipelineContext ctx)
         }
 
         @Override
-        public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List<File> orderedScatterOutputs) throws PipelineJobException
+        public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List<File> orderedScatterOutputs, List<String> orderedJobDirs) throws PipelineJobException
         {
             job.getLogger().info("Merging variant tables");
             List<File> toConcat = orderedScatterOutputs.stream().map(f -> {

mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java

@@ -110,7 +110,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
     }
 
     @Override
-    public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List<File> orderedScatterOutputs) throws PipelineJobException
+    public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List<File> orderedScatterOutputs, List<String> orderedJobDirs) throws PipelineJobException
     {
         job.getLogger().info("Merging missing sites VCFs");
         List<File> toConcat = orderedScatterOutputs.stream().map(f -> {