Add more logging in ETL

Author: bbimber

mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java

@@ -131,6 +131,10 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S
 
             File subDir = getLocalSubdir(folderName);
             File localCopy = doFileCopy(f, subDir, name);
+            if (localCopy == null)
+            {
+                // TODO
+            }
 
             //first create the ExpData
             ExpData d = ExperimentService.get().getExpDataByURL(localCopy, getContainerUser().getContainer());
@@ -203,7 +207,11 @@ protected File getLocalSubdir(Object folderName) throws PipelineJobException
 
     protected File doFileCopy(File f, File subdir, @Nullable String name) throws PipelineJobException
     {
-        getStatusLogger().info("preparing to copy file: " + f.getPath());
+        getStatusLogger().info("preparing to copy file: " + f.getPath() + ", with name: " + name);
+        if (f.getName().equals("write.lock"))
+        {
+            return LuceneIndexTransform.doLuceneCopy(f, subdir, name, getStatusLogger(), getContainerUser().getContainer());
+        }
 
         //Copy file locally, plus index if exists:
         File localCopy = new File(subdir, name == null || f.getName().startsWith("mGap.v") ? f.getName() : FileUtil.makeLegalName(name).replaceAll(" ", "_") + ".vcf.gz");
@@ -227,6 +235,10 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
                 try
                 {
                     Files.delete(localCopy.toPath());
+                    if (localCopy.exists())
+                    {
+                        throw new PipelineJobException("Unable to delete file: " + localCopy.getPath());
+                    }
                 }
                 catch (IOException e)
                 {
@@ -241,20 +253,24 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
 
         if (doCopy)
         {
-            getStatusLogger().info("queueing file copy: " + localCopy.getPath());
+            getStatusLogger().info("Creating symlink: " + f.getPath() + " / " + localCopy.getPath());
             try
             {
                 if (!Files.isReadable(f.toPath()))
                 {
                     throw new PipelineJobException("Unable to read file: " + f.getPath());
                 }
 
+                if (localCopy.exists())
+                {
+                    throw new PipelineJobException("File should have been deleted: " + localCopy.getPath());
+                }
+
                 Files.createSymbolicLink(f.toPath(), localCopy.toPath());
             }
             catch (IOException e)
             {
-                getStatusLogger().error("Failed to create symlink: " + localCopy.getPath(), e);
-                return null;
+                throw new PipelineJobException("Failed to create symlink: " + localCopy.getPath(), e);
             }
         }
 
@@ -270,16 +286,20 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
 
             if (!indexLocal.exists())
             {
-                getStatusLogger().info("queueing copy of index: " + indexLocal.getPath());
+                getStatusLogger().info("Creating symlink copy of VCF index: " + index.getPath() + " / " + indexLocal.getPath());
                 try
                 {
-                    Files.createSymbolicLink(f.toPath(), localCopy.toPath());
+                    Files.createSymbolicLink(index.toPath(), indexLocal.toPath());
                 }
                 catch (IOException e)
                 {
-                    getStatusLogger().error("Failed to create symlink: " + localCopy.getPath(), e);
+                    getStatusLogger().error("Failed to create symlink: " + indexLocal.getPath(), e);
                 }
             }
+            else
+            {
+                getStatusLogger().info("Local index already exists: " + indexLocal.getPath());
+            }
         }
 
         return localCopy;

mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java

@@ -1,6 +1,8 @@
 package org.labkey.mgap.columnTransforms;
 
+import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.Container;
 import org.labkey.api.jbrowse.JBrowseService;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.mgap.etl.EtlQueueManager;
@@ -21,11 +23,18 @@ protected Object doTransform(Object inputValue)
     @Override
     protected File doFileCopy(File f, File subdir, @Nullable String name) throws PipelineJobException
     {
+        return doLuceneCopy(f, subdir, name, getStatusLogger(), getContainerUser().getContainer());
+    }
+
+    public static File doLuceneCopy(File f, File subdir, @Nullable String name, Logger log, Container container) throws PipelineJobException
+    {
+        log.info("preparing to copy lucene index: " + f.getPath() + ", with name: " + name);
+
         // NOTE: lucene is a special case since the DB tracks one file, but we need this whole folder:
         File sourceDir = f.getParentFile();
         File targetDir = new File(subdir, "LuceneIndex");
         JBrowseService.get().clearLuceneCacheEntry(targetDir);
-        EtlQueueManager.get().queueRsyncCopy(getContainerUser().getContainer(), sourceDir, targetDir);
+        EtlQueueManager.get().queueRsyncCopy(container, sourceDir, targetDir);
 
         return new File(targetDir, sourceDir.getName() + "/" + f.getName());
     }