BimberLabKeyModules/PMR/resources/etls/prime-histology.xml at 6db0795d08ff1ffc523249dac6e2271a20c2b2b2 · LabKey/BimberLabKeyModules · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
<?xml version="1.0" encoding="UTF-8"?>
<etl xmlns="http://labkey.org/etl/xml" standalone="true">
    <name>PRIME_Histology_Data</name>
    <description>PRIMe Histology Data</description>
    <transforms>
        <transform id="select" type="RemoteQueryTransformStep">
            <source schemaName="study" queryName="histology" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
                <sourceColumns>
                    <column>Id</column>
                    <column>date</column>
                    <column>ageAtTime</column>
                    <column>sort_order</column>
                    <column>tissue</column>
                    <column>codes</column>
                    <column>objectid</column>
                    <column>created</column>
                    <column>modified</column>
                    <column>QCState/Label</column>
                </sourceColumns>
                <sourceFilters>
                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
                </sourceFilters>
            </source>
            <destination schemaName="study" queryName="histology" targetOption="merge" bulkLoad="true" batchSize="1000">
                <alternateKeys>
                    <column name="objectid"/>
                </alternateKeys>
            </destination>
        </transform>
        <transform id="rc" type="TaskrefTransformStep">
            <taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
                <settings>
                    <setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
                    <setting name="sourceSchema" value="study"/>
                    <setting name="sourceQuery" value="histology"/>
                    <setting name="sourceColumn" value="objectId"/>
                    <setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>

                    <setting name="destSchema" value="study"/>
                    <setting name="destQuery" value="histology"/>
                    <setting name="destColumn" value="objectId"/>

                    <setting name="reportOnly" value="false"/>
                </settings>
            </taskref>
        </transform>
    </transforms>
    <incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
        <deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
            <sourceFilters>
                <sourceFilter column="datasetid/Name" operator="eq" value="histology" />
                <sourceFilter column="Comment" operator="contains" value="Delete"/>
            </sourceFilters>
            <sourceColumns>
                <column>primaryKey</column>
            </sourceColumns>
        </deletedRowsSource>
    </incrementalFilter>
</etl>