Skip to content

Commit 48bd615

Browse files
bbimberbbimber
committed
Many improvements to EHR-related ETLs
1 parent 61ee678 commit 48bd615

22 files changed

Lines changed: 1182 additions & 553 deletions

‎PMR/resources/etls/pmr-datasets.xml‎

Lines changed: 0 additions & 226 deletions
This file was deleted.

‎PMR/resources/etls/pmrClinicalData.xml‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,91 @@
66
<transform id="demographics" type="TaskrefTransformStep">
77
<taskref ref="org.labkey.di.steps.QueueJobTask">
88
<settings>
9-
<setting name="transformId" value="{PMR}/pmr-demographics"/>
9+
<setting name="transformId" value="{PMR}/prime-demographics"/>
1010
</settings>
1111
</taskref>
1212
</transform>
1313

14-
<transform id="datasets" type="TaskrefTransformStep">
14+
<transform id="birth" type="TaskrefTransformStep">
1515
<taskref ref="org.labkey.di.steps.QueueJobTask">
1616
<settings>
17-
<setting name="transformId" value="{PMR}/pmr-datasets"/>
17+
<setting name="transformId" value="{PMR}/prime-birth"/>
1818
</settings>
1919
</taskref>
2020
</transform>
21+
22+
<transform id="blooddraws" type="TaskrefTransformStep">
23+
<taskref ref="org.labkey.di.steps.QueueJobTask">
24+
<settings>
25+
<setting name="transformId" value="{PMR}/prime-blooddraws"/>
26+
</settings>
27+
</taskref>
28+
</transform>
29+
30+
<transform id="chemistryResults" type="TaskrefTransformStep">
31+
<taskref ref="org.labkey.di.steps.QueueJobTask">
32+
<settings>
33+
<setting name="transformId" value="{PMR}/prime-chemistryResults"/>
34+
</settings>
35+
</taskref>
36+
</transform>
37+
38+
<transform id="clinpathRuns" type="TaskrefTransformStep">
39+
<taskref ref="org.labkey.di.steps.QueueJobTask">
40+
<settings>
41+
<setting name="transformId" value="{PMR}/prime-clinpathRuns"/>
42+
</settings>
43+
</taskref>
44+
</transform>
45+
46+
<transform id="hematologyResults" type="TaskrefTransformStep">
47+
<taskref ref="org.labkey.di.steps.QueueJobTask">
48+
<settings>
49+
<setting name="transformId" value="{PMR}/prime-hematologyResults"/>
50+
</settings>
51+
</taskref>
52+
</transform>
53+
54+
<transform id="histology" type="TaskrefTransformStep">
55+
<taskref ref="org.labkey.di.steps.QueueJobTask">
56+
<settings>
57+
<setting name="transformId" value="{PMR}/prime-histology"/>
58+
</settings>
59+
</taskref>
60+
</transform>
61+
62+
<transform id="microbiology" type="TaskrefTransformStep">
63+
<taskref ref="org.labkey.di.steps.QueueJobTask">
64+
<settings>
65+
<setting name="transformId" value="{PMR}/prime-microbiology"/>
66+
</settings>
67+
</taskref>
68+
</transform>
69+
70+
<transform id="parentage" type="TaskrefTransformStep">
71+
<taskref ref="org.labkey.di.steps.QueueJobTask">
72+
<settings>
73+
<setting name="transformId" value="{PMR}/prime-parentage"/>
74+
</settings>
75+
</taskref>
76+
</transform>
77+
78+
<transform id="pathologyDiagnoses" type="TaskrefTransformStep">
79+
<taskref ref="org.labkey.di.steps.QueueJobTask">
80+
<settings>
81+
<setting name="transformId" value="{PMR}/prime-pathologyDiagnoses"/>
82+
</settings>
83+
</taskref>
84+
</transform>
85+
86+
<transform id="weight" type="TaskrefTransformStep">
87+
<taskref ref="org.labkey.di.steps.QueueJobTask">
88+
<settings>
89+
<setting name="transformId" value="{PMR}/prime-weight"/>
90+
</settings>
91+
</taskref>
92+
</transform>
93+
2194
</transforms>
2295
<schedule>
2396
<cron expression="0 30 1 * * ?"/>

‎PMR/resources/etls/prime-birth.xml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<etl xmlns="http://labkey.org/etl/xml" standalone="true">
3+
<name>PRIME_Birth_Data</name>
4+
<description>PRIMe Birth Data</description>
5+
<transforms>
6+
<transform id="select" type="RemoteQueryTransformStep">
7+
<source schemaName="study" queryName="birth" remoteSource="EHR_ClinicalSource" timestampColumnName="modified" sourceTimeout="0">
8+
<sourceColumns>
9+
<column>Id</column>
10+
<column>date</column>
11+
<column>date_type</column>
12+
<column>birth_condition</column>
13+
<column>dam</column>
14+
<column>gender</column>
15+
<column>weight</column>
16+
<column>wdate</column>
17+
<column>type</column>
18+
<column>species</column>
19+
<column>geographic_origin</column>
20+
<column>sire</column>
21+
<column>estimated</column>
22+
<column>conception</column>
23+
<column>conceptualDay</column>
24+
<column>objectid</column>
25+
<column>created</column>
26+
<column>QCState/Label</column>
27+
</sourceColumns>
28+
<sourceFilters>
29+
<sourceFilter column="QCState/Label" operator="eq" value="Completed"/>
30+
</sourceFilters>
31+
</source>
32+
<destination schemaName="study" queryName="birth" targetOption="merge" bulkLoad="true" batchSize="1000">
33+
<alternateKeys>
34+
<column name="objectid"/>
35+
</alternateKeys>
36+
</destination>
37+
</transform>
38+
39+
<transform id="rc" type="TaskrefTransformStep">
40+
<taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
41+
<settings>
42+
<setting name="sourceRemoteSource" value="EHR_ClinicalSource"/>
43+
<setting name="sourceSchema" value="study"/>
44+
<setting name="sourceQuery" value="birth"/>
45+
<setting name="sourceColumn" value="objectId"/>
46+
<setting name="sourceAdditionalFilters" value="qcstate/label~eq=Completed"/>
47+
48+
<setting name="destSchema" value="study"/>
49+
<setting name="destQuery" value="birth"/>
50+
<setting name="destColumn" value="objectId"/>
51+
</settings>
52+
</taskref>
53+
</transform>
54+
</transforms>
55+
<incrementalFilter timestampColumnName="modified" pkColumnName="objectid" className="ModifiedSinceFilterStrategy" >
56+
<deletedRowsSource remoteSource="EHR_ClinicalSource" schemaName="AuditSummary" queryName="DatasetUpdateAuditLog" timestampColumnName="Created" deletedSourceKeyColumnName="primaryKey" targetKeyColumnName="objectid">
57+
<sourceFilters>
58+
<sourceFilter column="datasetid/Name" operator="eq" value="birth" />
59+
<sourceFilter column="Comment" operator="contains" value="Delete"/>
60+
</sourceFilters>
61+
<sourceColumns>
62+
<column>primaryKey</column>
63+
</sourceColumns>
64+
</deletedRowsSource>
65+
</incrementalFilter>
66+
</etl>

0 commit comments

Comments
 (0)