Merge pull request #217 from LabKey/fb_merge_24.7_to_develop

Merge discvr-24.7 to develop

Author: bbimber

mcc/resources/queries/mcc/genomicDatasetsSource.sql

@@ -7,4 +7,4 @@ SELECT
 
 FROM sequenceanalysis.sequence_readsets r
 
-WHERE r.subjectid LIKE 'MCC%' AND r.subjectid NOT LIKE 'MCC_%'
+WHERE r.subjectid LIKE 'MCC%' AND r.subjectid NOT LIKE 'MCC[_]%'

mcc/resources/queries/study/demographicsMostRecentWeight.query.xml

@@ -13,7 +13,7 @@
                             query.queryName=Weight&
                             query.id~eq=${id}
                         </url>
-                        <formatString>0.##</formatString>
+                        <formatString>0.####</formatString>
                     </column>
                     <column columnName="MostRecentWeightGrams">
                         <columnTitle>Current Weight (g)</columnTitle>

mcc/resources/queries/study/demographicsMostRecentWeight.sql

@@ -12,7 +12,7 @@ timestampdiff('SQL_TSI_DAY', w.MostRecentWeightDate, now()) AS DaysSinceWeight,
 null as weightField,
 --NOTE: we need to be careful in case duplicate weights are entered on the same time
 cast((
-    SELECT round(cast(AVG(w2.weight) as double), 2) AS _expr
+    SELECT round(cast(AVG(w2.weight) as double), 4) AS _expr
     FROM study.weight w2
     WHERE w.id=w2.id AND w.MostRecentWeightDate=w2.date
 ) as double) AS MostRecentWeight,

mcc/resources/queries/study/weight.query.xml

@@ -38,8 +38,8 @@
                         </fk>
                     </column>
                     <column columnName="weight">
-                      <columnTitle>Weight (kg)</columnTitle>
-                        <formatString>0.###</formatString>
+                        <columnTitle>Weight (kg)</columnTitle>
+                        <formatString>0.####</formatString>
                     </column>
                 </columns>
             </table>

mcc/src/org/labkey/mcc/etl/NprcObservationStep.java

@@ -36,6 +36,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -133,7 +134,7 @@ private void processFile(PipelineJob job) throws PipelineJobException
                     String objectId = new TableSelector(clinicalObs, PageFlowUtil.set("objectid"), filter, null).getObject(String.class);
                     if (observation == null && objectId != null)
                     {
-                        toDelete.add(row);
+                        toDelete.add(new CaseInsensitiveHashMap<>(Map.of("objectid", objectId)));
                         continue;
                     }
 

mcc/src/org/labkey/mcc/notification/MCCDataNotification.java

@@ -14,8 +14,6 @@
 import org.labkey.mcc.MccManager;
 
 import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
 
 /**
  * User: bimber
@@ -81,6 +79,7 @@ public String getMessageBodyHTML(Container c, User u)
         doU24AssignedCheck(mccData, u , msg);
         doMissingIdCheck(mccData, u, msg);
         doZeroWeightCheck(mccData, u, msg);
+        doDuplicationCheck(mccData, u, msg);
 
         //since we dont want to trigger an email if there's no alerts, conditionally append the title
         if (msg.length() > 0)
@@ -174,4 +173,37 @@ protected void doParentSexCheck(final Container c, User u, final StringBuilder m
             msg.append("<hr>\n\n");
         }
     }
+
+    protected void doDuplicationCheck(final Container c, User u, final StringBuilder msg)
+    {
+        TableInfo ti = getUserSchemaByName(c, u, "mcc").getTable("duplicateDemographics");
+        TableSelector ts = new TableSelector(ti);
+        long count = ts.getRowCount();
+        if (count > 0)
+        {
+            msg.append("<b>WARNING: There are ").append(count).append(" demographics records with duplicated MCC IDs\n");
+            msg.append("<p><a href='").append(getExecuteQueryUrl(c, "mcc", "duplicateDemographics", null)).append("'>Click here to view them</a><br>\n\n");
+            msg.append("<hr>\n\n");
+        }
+
+        ti = getUserSchemaByName(c, u, "mcc").getTable("duplicatedAggregatedDemographics");
+        ts = new TableSelector(ti);
+        count = ts.getRowCount();
+        if (count > 0)
+        {
+            msg.append("<b>WARNING: There are ").append(count).append(" aggregated demographics records with duplicated MCC IDs\n");
+            msg.append("<p><a href='").append(getExecuteQueryUrl(c, "mcc", "duplicatedAggregatedDemographics", null)).append("'>Click here to view them</a><br>\n\n");
+            msg.append("<hr>\n\n");
+        }
+
+        ti = getUserSchemaByName(c, u, "mcc").getTable("duplicatedAggregatedDemographicsParents");
+        ts = new TableSelector(ti);
+        count = ts.getRowCount();
+        if (count > 0)
+        {
+            msg.append("<b>WARNING: There are ").append(count).append(" aggregated demographics parent records with duplicated MCC IDs\n");
+            msg.append("<p><a href='").append(getExecuteQueryUrl(c, "mcc", "duplicatedAggregatedDemographicsParents", null)).append("'>Click here to view them</a><br>\n\n");
+            msg.append("<hr>\n\n");
+        }
+    }
 }

primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java

@@ -98,8 +98,8 @@ public Integer getMaxRequestCpus(PipelineJob job)
 
         if (isSequenceNormalizationTask(job))
         {
-            job.getLogger().debug("setting max CPUs to 8");
-            return 8;
+            job.getLogger().debug("setting max CPUs to 4");
+            return 4;
         }
 
         if (isLuceneIndexJob(job))
@@ -125,14 +125,14 @@ public Integer getMaxRequestCpus(PipelineJob job)
             }
             else if (totalFileSize < 20e9)
             {
-                job.getLogger().debug("file size less than 20gb, lowering CPUs to 16");
+                job.getLogger().debug("file size less than 20gb, lowering CPUs to 12");
 
-                return 16;
+                return 12;
             }
 
-            job.getLogger().debug("file size greater than 20gb, using 24 CPUs");
+            job.getLogger().debug("file size greater than 20gb, using 12 CPUs");
 
-            return 24;
+            return 12;
         }
 
         return null;
@@ -155,8 +155,8 @@ public Integer getMaxRequestMemory(PipelineJob job)
 
         if (isSequenceNormalizationTask(job))
         {
-            job.getLogger().debug("setting memory to 48");
-            return 48;
+            job.getLogger().debug("setting memory to 18");
+            return 18;
         }
 
         if (isGeneticsTask(job))
@@ -167,8 +167,8 @@ public Integer getMaxRequestMemory(PipelineJob job)
 
         if (isCacheAlignerIndexesTask(job))
         {
-            job.getLogger().debug("setting memory to 48");
-            return 48;
+            job.getLogger().debug("setting memory to 12");
+            return 12;
         }
 
         if (isLuceneIndexJob(job))
@@ -314,7 +314,7 @@ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine eng
     }
 
     @Override
-    public Map<String, Object> getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine)
+    public @NotNull Map<String, Object> getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine)
     {
         Map<String, Object> ret = new HashMap<>();
 

tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java

@@ -512,6 +512,12 @@ else if (discordantBarcodes.contains(barcode))
         for (File f : Arrays.asList(consensusFastaGD, allFastaGD))
         {
             _log.info("processing G/D FASTA: " + f.getPath());
+            if (!SequencePipelineService.get().hasMinLineCount(f, 2))
+            {
+                _log.info("too few lines, skipping FASTA:" + f.getPath());
+                continue;
+            }
+
             try (FastaDataLoader loader = new FastaDataLoader(f, false))
             {
                 loader.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter());