Merge pull request #166 from bimberlabinternal/24.3_fb_merge

Merge discvr-23.11 to discvr-24.3

Author: bbimber

mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java

@@ -1,14 +1,11 @@
 package org.labkey.mgap.columnTransforms;
 
-import org.apache.commons.io.FileUtils;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.jbrowse.JBrowseService;
 import org.labkey.api.pipeline.PipelineJobException;
-import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
 import org.labkey.mgap.etl.EtlQueueManager;
 
 import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
 
 public class LuceneIndexTransform extends OutputFileTransform
 {
@@ -27,6 +24,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
         // NOTE: lucene is a special case since the DB tracks one file, but we need this whole folder:
         File sourceDir = f.getParentFile();
         File targetDir = new File(subdir, "LuceneIndex");
+        JBrowseService.get().clearLuceneCacheEntry(targetDir);
         EtlQueueManager.get().queueRsyncCopy(getContainerUser().getContainer(), sourceDir, targetDir);
 
         return new File(targetDir, sourceDir.getName() + "/" + f.getName());

mcc/resources/etls/mcc.xml

@@ -38,7 +38,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -56,7 +56,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="kinship" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="kinship" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -72,7 +72,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="weight" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="weight" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -89,7 +89,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="genomicDatasets" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="genomicDatasets" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -106,7 +106,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="clinical_observations" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="clinical_observations" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -127,7 +127,7 @@
                     <column>marmosetsShipped</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="mcc" queryName="census" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="mcc" queryName="census" targetOption="truncate" bulkLoad="true" batchSize="5000">
 
             </destination>
         </transform>

mcc/resources/queries/study/demographics.js

@@ -111,6 +111,13 @@ function onUpsert(helper, scriptErrors, row, oldRow){
         }
     }
 
+    if (oldRow && oldRow.Id) {
+        var existingId = triggerHelper.getMccAlias(oldRow.Id);
+        if (existingId) {
+            idToMccAlias[row.Id] = existingId;
+        }
+    }
+
     if (!row.date) {
         row.date = new Date();
     }

mcc/src/org/labkey/mcc/query/TriggerHelper.java

@@ -260,6 +260,22 @@ public void possiblySendRabNotification(int reviewerId)
         }
     }
 
+    private TableInfo _mappingTable = null;
+
+    private TableInfo getMappingTable()
+    {
+        if (_mappingTable == null)
+        {
+            _mappingTable = QueryService.get().getUserSchema(_user, _container, MccSchema.NAME).getTable(MccSchema.TABLE_ANIMAL_MAPPING);
+        }
+
+        return _mappingTable;
+    }
+
+    public @Nullable String getMccAlias(String id) {
+        return new TableSelector(getMappingTable(), PageFlowUtil.set("externalAlias"), new SimpleFilter(FieldKey.fromString("subjectname"), id, CompareType.EQUAL), null).getObject(String.class);
+    }
+
     public int ensureMccAliasExists(Collection<String> rawIds, Map<Object, Object> existingAliases)
     {
         // NOTE: The incoming object can convert numeric IDs from strings to int, so manually convert:
@@ -273,7 +289,7 @@ public int ensureMccAliasExists(Collection<String> rawIds, Map<Object, Object> e
         SimpleFilter filter = new SimpleFilter(FieldKey.fromString("subjectname"), idMap.values(), CompareType.IN);
 
         final Set<String> aliasesFound = new HashSet<>();
-        TableInfo ti = QueryService.get().getUserSchema(_user, _container, MccSchema.NAME).getTable(MccSchema.TABLE_ANIMAL_MAPPING);
+        TableInfo ti = getMappingTable();
         new TableSelector(ti, PageFlowUtil.set("subjectname", "externalAlias"), filter, null).forEachResults(rs -> {
             aliasesFound.add(rs.getString(FieldKey.fromString("subjectname")));
             if (ciExistingAliases.containsKey(rs.getString(FieldKey.fromString("subjectname")))) {