Skip to content

Commit 417dcb0

Browse files
bbimberbbimber
authored
Merge pull request #204 from LabKey/fb_merge_24.3_to_develop
Merge discvr-24.3 to develop
2 parents 59054c9 + 5c222c4 commit 417dcb0

20 files changed

Lines changed: 562 additions & 104 deletions

File tree

‎LabPurchasing/resources/queries/labpurchasing/purchases.js‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,31 @@ var console = require("console");
44
var triggerHelper = new org.labkey.labpurchasing.LabPurchasingTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id);
55

66
function beforeInsert(row, errors){
7+
beforeUpsert(row, errors)
8+
}
9+
10+
function beforeUpdate(row, oldRow, errors){
11+
beforeUpsert(row, errors)
12+
}
13+
14+
function beforeUpsert(row, errors){
15+
// Validate requestor:
16+
if (row.requestor) {
17+
if (!isNaN(row.requestor) && !triggerHelper.isValidUserId(row.requestor)) {
18+
errors.requestor = 'Unknown userId for requestor: ' + row.requestor;
19+
}
20+
// Try to resolve strings:
21+
else if (isNaN(row.requestor)) {
22+
var id = triggerHelper.resolveUserId(String(row.requestor));
23+
if (!id) {
24+
errors.requestor = 'Unknown userId for requestor: ' + row.requestor;
25+
}
26+
else {
27+
row.requestor = id;
28+
}
29+
}
30+
}
31+
732
// The purpose of this is to allow the user to provide a string value for
833
// vendorId or vendorName, and attempt to resolve this against known vendors:
934
if (!row.vendorId || isNaN(row.vendorId)) {

‎LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,16 @@ public void sendNotifications(List<Integer> rowIds) {
112112
_log.error("Unable to send purchasing email", e);
113113
}
114114
}
115+
116+
public boolean isValidUserId(int userId)
117+
{
118+
return UserManager.getUser(userId) != null;
119+
}
120+
121+
public Integer resolveUserId(String userNameOrEmail)
122+
{
123+
User u = UserManager.getUserByDisplayName(userNameOrEmail);
124+
125+
return u == null ? null : u.getUserId();
126+
}
115127
}

‎mGAP/resources/etls/prime-seq.xml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,13 @@
33
<name>PRIMe-Seq ETLs</name>
44
<description>Syncs Anonymized Data PRIMe-Seq to mGAP</description>
55
<transforms>
6+
<transform id="step0" type="TaskRefTransformStep">
7+
<taskref ref="org.labkey.mgap.etl.ClearEtlWorkQueueTask">
8+
<settings>
9+
10+
</settings>
11+
</taskref>
12+
</transform>
613
<transform id="step1" type="RemoteQueryTransformStep">
714
<description>Copy to local table</description>
815
<source schemaName="mGAP" queryName="subjectsSource" remoteSource="PRIMESEQ">
@@ -190,6 +197,14 @@
190197
</columnTransforms>
191198
</destination>
192199
</transform>
200+
201+
<transform id="step7" type="TaskRefTransformStep">
202+
<taskref ref="org.labkey.mgap.etl.PerformQueuedEtlWorkTask">
203+
<settings>
204+
205+
</settings>
206+
</taskref>
207+
</transform>
193208
</transforms>
194209
<schedule>
195210
<cron expression="0 30 1 * * ?"/>

‎mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java‎

Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package org.labkey.mgap.columnTransforms;
22

3-
import org.apache.commons.io.FileUtils;
43
import org.apache.commons.lang3.StringUtils;
54
import org.labkey.api.collections.CaseInsensitiveHashMap;
65
import org.labkey.api.data.Results;
@@ -21,13 +20,12 @@
2120
import org.labkey.api.query.QueryService;
2221
import org.labkey.api.util.FileUtil;
2322
import org.labkey.api.util.PageFlowUtil;
23+
import org.labkey.mgap.etl.EtlQueueManager;
2424
import org.labkey.mgap.mGAPManager;
2525

2626
import java.io.File;
27-
import java.io.IOException;
2827
import java.net.URI;
2928
import java.sql.SQLException;
30-
import java.util.Arrays;
3129
import java.util.Date;
3230
import java.util.HashMap;
3331
import java.util.List;
@@ -107,7 +105,7 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S
107105
{
108106
if (dataFileUrl == null)
109107
{
110-
throw new IllegalArgumentException("DataFileUrl was null.");
108+
throw new IllegalArgumentException("DataFileUrl was null");
111109
}
112110

113111
URI uri = new URI(String.valueOf(dataFileUrl));
@@ -215,20 +213,8 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx
215213

216214
if (doCopy)
217215
{
218-
getStatusLogger().info("copying file locally: " + localCopy.getPath());
219-
if (localCopy.exists())
220-
{
221-
localCopy.delete();
222-
}
223-
224-
try
225-
{
226-
FileUtils.copyFile(f, localCopy);
227-
}
228-
catch (IOException e)
229-
{
230-
throw new PipelineJobException(e);
231-
}
216+
getStatusLogger().info("queueing file copy: " + localCopy.getPath());
217+
EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), f, localCopy);
232218
}
233219

234220
File index = new File(f.getPath() + ".tbi");
@@ -243,15 +229,8 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx
243229

244230
if (!indexLocal.exists())
245231
{
246-
getStatusLogger().info("copying index locally: " + indexLocal.getPath());
247-
try
248-
{
249-
FileUtils.copyFile(index, indexLocal);
250-
}
251-
catch (IOException e)
252-
{
253-
throw new PipelineJobException(e);
254-
}
232+
getStatusLogger().info("queueing copy of index: " + indexLocal.getPath());
233+
EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), index, indexLocal);
255234
}
256235
}
257236

‎mGAP/src/org/labkey/mgap/columnTransforms/ExpDataTransform.java‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import org.jetbrains.annotations.Nullable;
55
import org.labkey.api.pipeline.PipelineJobException;
66
import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
7+
import org.labkey.mgap.etl.EtlQueueManager;
78

89
import java.io.File;
910
import java.io.IOException;
@@ -26,12 +27,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
2627
// NOTE: lucene is a special case since the DB tracks one file, but we need this whole folder:
2728
File sourceDir = f.getParentFile();
2829
File targetDir = new File(subdir, "LuceneIndex");
29-
30-
// NOTE: rsync should no-op if there are no source changes
31-
getStatusLogger().info("Copying lucene index dir to: " + targetDir.getPath());
32-
new SimpleScriptWrapper(getStatusLogger()).execute(Arrays.asList(
33-
"rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), targetDir.getPath()
34-
));
30+
EtlQueueManager.get().queueRsyncCopy(getContainerUser().getContainer(), sourceDir, targetDir);
3531

3632
return new File(targetDir, sourceDir.getName() + "/" + f.getName());
3733
}

‎mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package org.labkey.mgap.etl;
2+
3+
import org.apache.xmlbeans.XmlException;
4+
import org.jetbrains.annotations.NotNull;
5+
import org.labkey.api.di.TaskRefTask;
6+
import org.labkey.api.pipeline.PipelineJob;
7+
import org.labkey.api.pipeline.PipelineJobException;
8+
import org.labkey.api.pipeline.RecordedActionSet;
9+
import org.labkey.api.writer.ContainerUser;
10+
11+
import java.util.List;
12+
import java.util.Map;
13+
14+
public class ClearEtlWorkQueueTask implements TaskRefTask
15+
{
16+
private ContainerUser _containerUser = null;
17+
18+
@Override
19+
public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException
20+
{
21+
EtlQueueManager.get().clearQueue(_containerUser.getContainer(), job.getLogger());
22+
23+
return new RecordedActionSet();
24+
}
25+
26+
@Override
27+
public List<String> getRequiredSettings()
28+
{
29+
return null;
30+
}
31+
32+
@Override
33+
public void setSettings(Map<String, String> settings) throws XmlException
34+
{
35+
36+
}
37+
38+
@Override
39+
public void setContainerUser(ContainerUser containerUser)
40+
{
41+
_containerUser = containerUser;
42+
}
43+
}

‎mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package org.labkey.mgap.etl;
2+
3+
import org.apache.commons.io.FileUtils;
4+
import org.apache.commons.lang3.tuple.Pair;
5+
import org.apache.logging.log4j.Logger;
6+
import org.labkey.api.data.Container;
7+
import org.labkey.api.pipeline.PipelineJobException;
8+
import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
9+
10+
import java.io.File;
11+
import java.io.IOException;
12+
import java.util.ArrayList;
13+
import java.util.Arrays;
14+
import java.util.HashMap;
15+
import java.util.List;
16+
import java.util.Map;
17+
18+
public class EtlQueueManager
19+
{
20+
private static EtlQueueManager _instance = new EtlQueueManager();
21+
22+
private Map<Container, List<Pair<File, File>>> _pendingFileCopy = new HashMap<>();
23+
24+
private Map<Container, List<Pair<File, File>>> _pendingRsyncCopy = new HashMap<>();
25+
26+
private EtlQueueManager()
27+
{
28+
29+
}
30+
31+
public static EtlQueueManager get()
32+
{
33+
return _instance;
34+
}
35+
36+
public void clearQueue(Container container, Logger log)
37+
{
38+
if (_pendingFileCopy.containsKey(container) && !_pendingFileCopy.get(container).isEmpty())
39+
{
40+
log.error("The file copy queue was not empty!");
41+
}
42+
43+
if (_pendingRsyncCopy.containsKey(container) && !_pendingRsyncCopy.get(container).isEmpty())
44+
{
45+
log.error("The rsync copy queue was not empty!");
46+
}
47+
48+
_pendingFileCopy.clear();
49+
_pendingRsyncCopy.clear();
50+
}
51+
52+
public void performQueuedWork(Container container, Logger log)
53+
{
54+
List<Pair<File, File>> queue = _pendingFileCopy.get(container);
55+
if (queue != null && !queue.isEmpty())
56+
{
57+
queue.forEach(x -> copyFile(x.getLeft(), x.getRight(), log));
58+
}
59+
_pendingFileCopy.clear();
60+
61+
List<Pair<File, File>> rsyncQueue = _pendingRsyncCopy.get(container);
62+
if (rsyncQueue != null && !rsyncQueue.isEmpty())
63+
{
64+
rsyncQueue.forEach(x -> doRsyncCopy(x.getLeft(), x.getRight(), log));
65+
}
66+
_pendingRsyncCopy.clear();
67+
}
68+
69+
public void queueFileCopy(Container c, File source, File destination)
70+
{
71+
if (!_pendingFileCopy.containsKey(c))
72+
{
73+
_pendingFileCopy.put(c, new ArrayList<>());
74+
}
75+
76+
_pendingFileCopy.get(c).add(Pair.of(source, destination));
77+
}
78+
79+
public void queueRsyncCopy(Container c, File source, File destination)
80+
{
81+
if (!_pendingRsyncCopy.containsKey(c))
82+
{
83+
_pendingRsyncCopy.put(c, new ArrayList<>());
84+
}
85+
86+
_pendingRsyncCopy.get(c).add(Pair.of(source, destination));
87+
}
88+
89+
private void doRsyncCopy(File sourceDir, File destination, Logger log)
90+
{
91+
// NOTE: rsync should no-op if there are no source changes
92+
log.info("Performing rsync from: " + sourceDir.getPath() + " to " + destination.getPath());
93+
try
94+
{
95+
new SimpleScriptWrapper(log).execute(Arrays.asList(
96+
"rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), destination.getPath()
97+
));
98+
}
99+
catch (PipelineJobException e)
100+
{
101+
log.error("Error running rsync", e);
102+
}
103+
}
104+
105+
private void copyFile(File source, File destination, Logger log)
106+
{
107+
if (destination.exists())
108+
{
109+
destination.delete();
110+
}
111+
112+
try
113+
{
114+
log.info("Copying file: " + source.getPath());
115+
FileUtils.copyFile(source, destination);
116+
}
117+
catch (IOException e)
118+
{
119+
log.error("Error copying file", e);
120+
}
121+
}
122+
}

0 commit comments

Comments
 (0)