Switch ETLs to log row count discrepancies

Author: bbimber

PMR/resources/etls/prime-blooddraws.xml

@@ -34,6 +34,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="blood"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>

PMR/resources/etls/prime-chemistryResults.xml

@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="chemistryResults"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>

PMR/resources/etls/prime-clinpathRuns.xml

@@ -41,6 +41,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="clinpathRuns"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>

PMR/resources/etls/prime-hematologyResults.xml

@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="hematologyResults"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>

PMR/resources/etls/prime-histology.xml

@@ -39,6 +39,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="histology"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>

PMR/resources/etls/prime-microbiology.xml

@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="microbiology"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>

PMR/resources/etls/prime-pathologyDiagnoses.xml

@@ -38,6 +38,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="pathologyDiagnoses"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>

PMR/resources/etls/prime-weight.xml

@@ -37,6 +37,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="weight"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>

primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java

@@ -1,6 +1,7 @@
 package org.labkey.primeseq.etl;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
@@ -50,7 +51,8 @@ private enum Settings
         destSchema(true),
         destQuery(true),
         destColumn(true),
-        destAdditionalFilters(false);
+        destAdditionalFilters(false),
+        reportOnly(false);
 
         private final boolean _isRequired;
 
@@ -106,6 +108,11 @@ public void setSettings(Map<String, String> settings)
         _settings.putAll(settings);
     }
 
+    private boolean isReportOnly()
+    {
+        return _settings.containsKey(Settings.reportOnly.name()) && Boolean.parseBoolean(_settings.get(Settings.reportOnly.name()));
+    }
+
     private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Container c, Logger log) throws IllegalStateException
     {
         DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, c, log);
@@ -259,7 +266,13 @@ private void verifyRows(PipelineJob job) throws PipelineJobException
 
         if (source != dest)
         {
-            job.getLogger().error("Row counts do not match (source: {}, dest: {})!", source, dest);
+            if (isReportOnly()) {
+                job.getLogger().info("Row counts do not match (source: {}, dest: {})!", source, dest);
+            }
+            else
+            {
+                job.getLogger().error("Row counts do not match (source: {}, dest: {})!", source, dest);
+            }
         }
     }
 }