Skip to content

Commit d353e90

Browse files
bbimberbbimber
committed
Bugfixes to SubjectScopedSelect
1 parent b2e8037 commit d353e90

5 files changed

Lines changed: 45 additions & 15 deletions

File tree

‎SivStudies/resources/etls/siv-studies.xml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<setting name="dataRemoteSource" value="PRIMe"/>
1414
<setting name="dataSourceSchema" value="study"/>
1515
<setting name="dataSourceQuery" value="weight"/>
16+
<setting name="dataSourceSubjectColumn" value="Id"/>
1617
<setting name="dataSourceColumns" value="Id,date,weight,objectid"/>
1718

1819
<setting name="targetSchema" value="study"/>
@@ -32,7 +33,8 @@
3233
<setting name="dataSourceContainerPath" value="Internal/MHC_Typing"/>
3334
<setting name="dataSourceSchema" value="geneticscore"/>
3435
<setting name="dataSourceQuery" value="mhc_data"/>
35-
<setting name="dataSourceColumns" value="subjectid,assaytype,marker,result,score"/>
36+
<setting name="dataSourceSubjectColumn" value="subjectid"/>
37+
<setting name="dataSourceColumns" value="subjectid,date,assaytype,marker,result,score"/>
3638
<setting name="dataSourceColumnMapping" value="subjectId=Id"/>
3739
<setting name="dataSourceColumnDefaults" value="category=MHC Typing"/>
3840

@@ -53,6 +55,7 @@
5355
<setting name="dataRemoteSource" value="PRIMe"/>
5456
<setting name="dataSourceSchema" value="study"/>
5557
<setting name="dataSourceQuery" value="hematologyResults"/>
58+
<setting name="dataSourceSubjectColumn" value="Id"/>
5659
<setting name="dataSourceColumns" value="Id,date,testid,result,units,qualresult,objectid"/>
5760
<setting name="dataSourceColumnMapping" value="testid=test"/>
5861
<setting name="dataSourceAdditionalFilters" value="qcstate/publicdata~eq=true"/>
@@ -76,6 +79,7 @@
7679
<setting name="dataRemoteSource" value="PRIMe"/>
7780
<setting name="dataSourceSchema" value="study"/>
7881
<setting name="dataSourceQuery" value="chemistryResults"/>
82+
<setting name="dataSourceSubjectColumn" value="Id"/>
7983
<setting name="dataSourceColumns" value="Id,date,testid,result,units,qualresult,objectid"/>
8084
<setting name="dataSourceColumnMapping" value="testid=test"/>
8185
<setting name="dataSourceAdditionalFilters" value="qcstate/publicdata~eq=true"/>
@@ -98,7 +102,8 @@
98102

99103
<setting name="dataRemoteSource" value="PRIMe"/>
100104
<setting name="dataSourceSchema" value="study"/>
101-
<setting name="dataSourceQuery" value="clinicalEncounters"/>
105+
<setting name="dataSourceQuery" value="encounters"/>
106+
<setting name="dataSourceSubjectColumn" value="Id"/>
102107
<setting name="dataSourceColumns" value="Id,date,type,procedureid/name,objectid"/>
103108
<setting name="dataSourceColumnMapping" value="procedureid/name=procedure,type=category"/>
104109
<setting name="dataSourceAdditionalFilters" value="qcstate/publicdata~eq=true;procedureid/name~isnonblank="/>

‎SivStudies/resources/folderTypes/SIV Studies.folderType.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<ft:folderType xmlns:ft="http://labkey.org/data/xml/folderType">
2-
<ft:name>Studies Overview</ft:name>
2+
<ft:name>SIV Studies Overview</ft:name>
33
<ft:description>The default folder layout for Studies</ft:description>
44
<ft:folderTabs>
55
<!-- <ft:folderTab>-->
@@ -61,7 +61,7 @@
6161
</ft:requiredWebParts>
6262
<ft:preferredWebParts>
6363
<ft:webPart>
64-
<ft:name>Studies Admin</ft:name>
64+
<ft:name>SIV Studies Admin</ft:name>
6565
<ft:location>body</ft:location>
6666
</ft:webPart>
6767
</ft:preferredWebParts>

‎SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,9 @@
201201
<datatype>entityid</datatype>
202202
<isKeyField>true</isKeyField>
203203
</column>
204-
204+
<column columnName="category">
205+
<datatype>varchar</datatype>
206+
</column>
205207
<column columnName="test">
206208
<datatype>varchar</datatype>
207209
</column>

‎SivStudies/src/org/labkey/sivstudies/SivStudiesController.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public boolean handlePost(Object o, BindException errors) throws Exception
6666
StudiesService.get().importFolderDefinition(getContainer(), getUser(), ModuleLoader.getInstance().getModule(SivStudiesModule.NAME), new Path("referenceStudy"));
6767

6868
Module m = ModuleLoader.getInstance().getModule(SivStudiesModule.NAME);
69-
StudiesService.get().loadTsv(m.getModuleResource("data/lookup_sets.tsv"), SivStudiesSchema.NAME, getUser(), getContainer());
69+
StudiesService.get().loadTsv(m.getModuleResource("data/lookup_sets.tsv"), "studies", getUser(), getContainer());
7070

7171
Resource r = m.getModuleResource("data");
7272
r.list().forEach(tsv -> {
@@ -79,7 +79,7 @@ public boolean handlePost(Object o, BindException errors) throws Exception
7979
{
8080
case "reports.tsv" -> "laboratory";
8181
case "species.tsv" -> "laboratory";
82-
default -> SivStudiesSchema.NAME;
82+
default -> "studies";
8383
};
8484

8585
StudiesService.get().loadTsv(tsv, schemaName, getUser(), getContainer());

‎SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import java.util.Collection;
4444
import java.util.Collections;
4545
import java.util.HashMap;
46+
import java.util.HashSet;
4647
import java.util.List;
4748
import java.util.Map;
4849
import java.util.Set;
@@ -65,6 +66,7 @@ private enum Settings
6566
dataSourceContainerPath(false),
6667
dataSourceSchema(true),
6768
dataSourceQuery(true),
69+
dataSourceSubjectColumn(true),
6870
dataSourceColumns(true),
6971
dataSourceColumnMapping(false),
7072
dataSourceAdditionalFilters(false),
@@ -120,6 +122,11 @@ private void processBatch(List<String> subjects, Logger log)
120122
additionalFilters.forEach(subjectFilter::addCondition);
121123
}
122124

125+
if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null)
126+
{
127+
throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name()));
128+
}
129+
123130
Collection<Map<String, Object>> existingRows = new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection();
124131
if (!existingRows.isEmpty())
125132
{
@@ -132,11 +139,16 @@ private void processBatch(List<String> subjects, Logger log)
132139
}
133140

134141
// Query data and import
135-
List<Map<String, Object>> toImport = getRowsToImport(log);
142+
List<Map<String, Object>> toImport = getRowsToImport(subjects, log);
136143
if (!toImport.isEmpty())
137144
{
138145
log.info("inserting " + toImport.size() + " rows");
139-
qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImport, new BatchValidationException(), null, null);
146+
BatchValidationException bve = new BatchValidationException();
147+
qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toImport, bve, null, null);
148+
if (bve.hasErrors())
149+
{
150+
throw bve;
151+
}
140152
}
141153
else
142154
{
@@ -158,7 +170,7 @@ private List<CompareType.CompareClause> parseAdditionalFilters(String rawVal)
158170
}
159171

160172
SimpleFilter filter = new SimpleFilter();
161-
String[] filters = rawVal.split(",");
173+
String[] filters = rawVal.split(";");
162174
for (String queryParam : filters)
163175
{
164176
filter.addUrlFilters(new ActionURL().setRawQuery(queryParam), null);
@@ -234,7 +246,7 @@ private Map<String, String> parseColumnDefaultMap(String rawVal)
234246
return colMap;
235247
}
236248

237-
private List<Map<String, Object>> getRowsToImport(Logger log)
249+
private List<Map<String, Object>> getRowsToImport(List<String> subjects, Logger log)
238250
{
239251
if (_settings.get(Settings.dataSourceColumns.name()) == null)
240252
{
@@ -250,6 +262,7 @@ private List<Map<String, Object>> getRowsToImport(Logger log)
250262
DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.dataRemoteSource.name()), log);
251263
SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.dataSourceSchema.name()), _settings.get(Settings.dataSourceQuery.name()));
252264
sr.setColumns(sourceColumns);
265+
sr.addFilter(_settings.get(Settings.dataSourceSubjectColumn.name()), StringUtils.join(subjects, ";"), Filter.Operator.IN);
253266
if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null)
254267
{
255268
List<CompareType.CompareClause> additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name()));
@@ -269,7 +282,13 @@ else if (f.getParamVals().length == 1)
269282
value = StringUtils.join(f.getParamVals(), ";");
270283
}
271284

272-
sr.addFilter(new Filter(f.getFieldKey().toString(), value, Filter.Operator.valueOf(f.getCompareType().getFilterValueText())));
285+
Filter.Operator o = Filter.Operator.getOperatorFromUrlKey(f.getCompareType().getPreferredUrlKey());
286+
if (o == null)
287+
{
288+
throw new IllegalStateException("Unknown operator: " + f.getCompareType().getPreferredUrlKey() + ", raw filter: " + f.getCompareType().name());
289+
}
290+
291+
sr.addFilter(new Filter(f.getFieldKey().toString(), value, o));
273292
}
274293
}
275294

@@ -326,15 +345,19 @@ else if (f.getParamVals().length == 1)
326345
}
327346
}
328347

348+
if (sourceTable.getColumn(_settings.get(Settings.dataSourceSubjectColumn.name())) == null)
349+
{
350+
throw new IllegalStateException("Table is missing column: " + _settings.get(Settings.dataSourceSubjectColumn.name()));
351+
}
329352

330-
final SimpleFilter filter = new SimpleFilter();
353+
final SimpleFilter filter = new SimpleFilter(_settings.get(Settings.dataSourceSubjectColumn.name()), subjects, CompareType.IN);
331354
if (_settings.get(Settings.dataSourceAdditionalFilters.name()) != null)
332355
{
333356
List<CompareType.CompareClause> additionalFilters = parseAdditionalFilters(_settings.get(Settings.dataSourceAdditionalFilters.name()));
334357
additionalFilters.forEach(filter::addCondition);
335358
}
336359

337-
TableSelector ts = new TableSelector(sourceTable, PageFlowUtil.set(_settings.get(Settings.subjectSourceColumn.name())), filter, null);
360+
TableSelector ts = new TableSelector(sourceTable, new HashSet<>(sourceColumns), filter, null);
338361

339362
return doNameMapping(new ArrayList<>(ts.getMapCollection()), sourceToDestColumMap, columnToDefaultMap);
340363
}
@@ -400,7 +423,7 @@ public List<ValidationError> preFlightCheck(Container c)
400423
List<ValidationError> errors = new ArrayList<>();
401424
for (String setting : getRequiredSettings())
402425
{
403-
if (_settings.get(setting) == null)
426+
if (_settings.get(setting) == null || StringUtils.isEmpty(_settings.get(setting)))
404427
{
405428
errors.add(new SimpleValidationError("Missing required setting: " + setting));
406429
}

0 commit comments

Comments
 (0)