Allow long-running upgrade code to run separate from update

Author: bbimber

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java

@@ -5254,4 +5254,38 @@ public void setDataFileUrl(String dataFileUrl)
             _dataFileUrl = dataFileUrl;
         }
     }
+
+    @UtilityAction(label = "Migrate Sequence Files", description = "This will start a background process to migrate sequence files from the flat .sequences folder into a hashed multi-folder scheme")
+    @RequiresSiteAdmin
+    public static class MigrateSequenceFilesAction extends ConfirmAction<Object>
+    {
+        @Override
+        public ModelAndView getConfirmView(Object o, BindException errors) throws Exception
+        {
+            setTitle("Migrate Sequence Files");
+
+            return(HtmlView.of("This will start a background process to migrate sequence files from the flat .sequences folder into a hashed multi-folder scheme.  Do you want to continue?"));
+        }
+
+        @Override
+        public boolean handlePost(Object o, BindException errors) throws Exception
+        {
+            SequenceAnalysisUpgradeCode.doSequenceMigration(getUser(), _log);
+
+            return true;
+        }
+
+        @Override
+        public void validateCommand(Object o, Errors errors)
+        {
+
+        }
+
+        @NotNull
+        @Override
+        public URLHelper getSuccessURL(Object o)
+        {
+            return getContainer().getStartURL(getUser());
+        }
+    }
 }

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisUpgradeCode.java

@@ -20,7 +20,9 @@
 import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.module.ModuleContext;
 import org.labkey.api.query.FieldKey;
+import org.labkey.api.security.User;
 import org.labkey.api.sequenceanalysis.RefNtSequenceModel;
+import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.PageFlowUtil;
 
 import java.io.File;
@@ -234,68 +236,81 @@ public void updateBarcodeRC(final ModuleContext moduleContext)
     @SuppressWarnings({"UnusedDeclaration"})
     @DeferredUpgrade
     public void migrateSequenceDirs(final ModuleContext moduleContext)
+    {
+        doSequenceMigration(moduleContext.getUpgradeUser(), _log);
+    }
+
+    public static void doSequenceMigration(User u, Logger log)
     {
         try
         {
             TableInfo ti = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_REF_NT_SEQUENCES);
             TableSelector ts = new TableSelector(ti);
             List<RefNtSequenceModel> nts = ts.getArrayList(RefNtSequenceModel.class);
-            _log.info(nts.size() + " total sequences to migrate");
+            log.info(nts.size() + " total sequences to migrate");
             int processed = 0;
             int totalMigrated = 0;
             for (RefNtSequenceModel nt : nts)
             {
                 processed++;
-
                 if (processed % 1000 == 0)
                 {
-                    _log.info("{} of {} sequence files migrated", processed, nts.size());
+                    log.info("{} of {} sequence files migrated", processed, nts.size());
                 }
 
                 ExpData legacyExpData = ExperimentService.get().getExpData(nt.getSequenceFile());
                 if (legacyExpData == null)
                 {
-                    _log.error("Missing ExpData for NT sequence: {}", nt.getSequenceFile());
+                    log.error("Missing ExpData for NT sequence: {}", nt.getSequenceFile());
                     continue;
                 }
 
                 File legacyFile = legacyExpData.getFile();
-                if (!legacyFile.exists())
+                if (!RefNtSequenceModel.BASE_DIRNAME.equals(legacyFile.getParentFile().getName()))
                 {
-                    _log.error("Missing file for NT sequence: {}", legacyFile.getPath());
+                    // NOTE: this includes sequences imported to custom locations, such as refSequenceImport pipeline jobs
                     continue;
                 }
 
-                if (!RefNtSequenceModel.BASE_DIRNAME.equals(legacyFile.getParentFile().getName()))
+                File newLocation = nt.getExpectedSequenceFile(null);
+                if (legacyFile.equals(newLocation))
                 {
-                    // NOTE: this includes sequences imported to custom locations, such as refSequenceImport pipeline jobs
                     continue;
                 }
 
-                File newLocation = nt.getExpectedSequenceFile(null);
+                if (!legacyFile.exists())
+                {
+                    log.error("Missing file for NT sequence: {}", legacyFile.getPath());
+                    continue;
+                }
+
                 if (!newLocation.getParentFile().exists())
                 {
-                    newLocation.getParentFile().mkdirs();
+                    FileUtil.mkdirs(newLocation.getParentFile());
                 }
 
                 if (newLocation.exists())
                 {
-                    _log.error("Target location for migrated sequence file exists, this might indicate a retry after a filed move: {}", newLocation.getPath());
-                    continue;
+                    if (newLocation.length() == legacyFile.length())
+                    {
+                        continue;
+                    }
+
+                    log.error("Target location for migrated sequence file exists, but file size is smaller. this might indicate a retry after a filed move. deleting the target and retrying: {}", newLocation.getPath());
+                    FileUtils.delete(newLocation);
                 }
 
-                totalMigrated++;
-                FileUtils.copyFile(legacyFile, newLocation);
+                FileUtils.moveFile(legacyFile, newLocation);
                 legacyExpData.setDataFileURI(newLocation.toURI());
-                legacyExpData.save(moduleContext.getUpgradeUser());
-                legacyFile.delete();
+                legacyExpData.save(u);
+                totalMigrated++;
             }
 
-            _log.info("Total sequences migrated: {}", totalMigrated);
+            log.info("Total sequences migrated: {}", totalMigrated);
         }
         catch (Exception e)
         {
-            _log.error("Error upgrading sequenceanalysis module", e);
+            log.error("Error migrating sequence files", e);
         }
     }
 }