Ensure PopulateIds runs prior to center ETLs

bbimber · bbimber · commit e51a71976716 · 2024-07-14T16:25:31.000-07:00
diff --git a/mcc/resources/etls/snprc-datasets.xml b/mcc/resources/etls/snprc-datasets.xml
@@ -178,11 +178,17 @@
             </taskref>
         </transform>
 
+        <!--This will create MCC aliases as-needed-->
+        <transform id="step0" type="TaskRefTransformStep">
+            <taskref ref="org.labkey.mcc.etl.PopulateIdsStep">
+                <settings>
+                </settings>
+            </taskref>
+        </transform>
     </transforms>
 
     <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >
-        <!--TODO: need to fix LK FilterStrategy.init() bug before this can be enabled-->
-        <!--<deletedRowsSource schemaName="mcc" queryName="rowsToDelete" targetKeyColumnName="objectid" deletedSourceKeyColumnName="objectid" timestampColumnName="source_modified"/>-->
+        <deletedRowsSource schemaName="mcc" queryName="rowsToDelete" targetKeyColumnName="objectid" deletedSourceKeyColumnName="objectid" timestampColumnName="source_modified"/>
     </incrementalFilter>
     <schedule>
         <!--11PM-->
diff --git a/mcc/resources/etls/wnprc-datasets.xml b/mcc/resources/etls/wnprc-datasets.xml
@@ -140,10 +140,16 @@
         <!--</destination>-->
         <!--</transform>-->
 
+        <!--This will create MCC aliases as-needed-->
+        <transform id="step0" type="TaskRefTransformStep">
+            <taskref ref="org.labkey.mcc.etl.PopulateIdsStep">
+                <settings>
+                </settings>
+            </taskref>
+        </transform>
     </transforms>
 
     <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified">
-        <!--TODO: need to fix LK FilterStrategy.init() bug before this can be enabled-->
-        <!--<deletedRowsSource schemaName="mcc" queryName="rowsToDelete" targetKeyColumnName="objectId" deletedSourceKeyColumnName="objectId" timestampColumnName="source_modified"/>-->
+        <deletedRowsSource schemaName="mcc" queryName="rowsToDelete" targetKeyColumnName="objectId" deletedSourceKeyColumnName="objectId" timestampColumnName="source_modified"/>
     </incrementalFilter>
 </etl>
diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.sql b/mcc/resources/queries/mcc/genomicDatasetsSource.sql
@@ -7,4 +7,4 @@ SELECT
 
 FROM sequenceanalysis.sequence_readsets r
 
-WHERE r.subjectid LIKE 'MCC%'
+WHERE r.subjectid LIKE 'MCC%' AND r.subjectid NOT LIKE 'MCC_%'

Original file line number	Diff line number	Diff line change
`@@ -7,4 +7,4 @@ SELECT`
`7`	`7`
`8`	`8`	`FROM sequenceanalysis.sequence_readsets r`
`9`	`9`
`10`		`-WHERE r.subjectid LIKE 'MCC%'`
	`10`	`+WHERE r.subjectid LIKE 'MCC%' AND r.subjectid NOT LIKE 'MCC_%'`