Skip to content

Commit 8fe4edb

Browse files
bbimberbbimber
committed
Switch sequence migration code to pipeline job to allow to run in background
1 parent dd372e1 commit 8fe4edb

5 files changed

Lines changed: 163 additions & 4 deletions

File tree

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@
157157
import org.labkey.sequenceanalysis.pipeline.SequenceAlignmentJob;
158158
import org.labkey.sequenceanalysis.pipeline.SequenceConcatPipelineJob;
159159
import org.labkey.sequenceanalysis.pipeline.SequenceJob;
160+
import org.labkey.sequenceanalysis.pipeline.SequenceMigrationPipelineJob;
160161
import org.labkey.sequenceanalysis.pipeline.SequenceOutputHandlerJob;
161162
import org.labkey.sequenceanalysis.pipeline.SequenceReadsetHandlerJob;
162163
import org.labkey.sequenceanalysis.pipeline.SequenceTaskHelper;
@@ -5270,7 +5271,8 @@ public ModelAndView getConfirmView(Object o, BindException errors) throws Except
52705271
@Override
52715272
public boolean handlePost(Object o, BindException errors) throws Exception
52725273
{
5273-
SequenceAnalysisUpgradeCode.doSequenceMigration(getUser(), _log);
5274+
PipeRoot pr = PipelineService.get().findPipelineRoot(getContainer());
5275+
PipelineService.get().queueJob(new SequenceMigrationPipelineJob(getContainer(), getUser(), getViewContext().getActionURL(), pr));
52745276

52755277
return true;
52765278
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
import org.labkey.sequenceanalysis.pipeline.SequenceAlignmentTask;
8484
import org.labkey.sequenceanalysis.pipeline.SequenceJob;
8585
import org.labkey.sequenceanalysis.pipeline.SequenceJobSupportImpl;
86+
import org.labkey.sequenceanalysis.pipeline.SequenceMigrationPipelineJob;
8687
import org.labkey.sequenceanalysis.pipeline.SequenceOutputHandlerPipelineProvider;
8788
import org.labkey.sequenceanalysis.pipeline.SequencePipelineProvider;
8889
import org.labkey.sequenceanalysis.pipeline.SequenceReadsetHandlerPipelineProvider;
@@ -484,6 +485,7 @@ public Set<ExperimentRunType> getExperimentRunTypes(@Nullable Container containe
484485
PipelineService.get().registerPipelineProvider(new OrphanFilePipelineProvider(this));
485486
PipelineService.get().registerPipelineProvider(new SequencePipelineProvider(this));
486487
PipelineService.get().registerPipelineProvider(new CacheGenomePipelineJob.Provider(this));
488+
PipelineService.get().registerPipelineProvider(new SequenceMigrationPipelineJob.Provider(this));
487489

488490
LDKService.get().registerQueryButton(new ReprocessLibraryButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_REF_LIBRARIES);
489491

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisUpgradeCode.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -237,17 +237,17 @@ public void updateBarcodeRC(final ModuleContext moduleContext)
237237
@DeferredUpgrade
238238
public void migrateSequenceDirs(final ModuleContext moduleContext)
239239
{
240-
doSequenceMigration(moduleContext.getUpgradeUser(), _log);
240+
doSequenceMigration(moduleContext.getUpgradeUser(), _log, 100000);
241241
}
242242

243-
public static void doSequenceMigration(User u, Logger log)
243+
public static void doSequenceMigration(User u, Logger log, int maxSequences)
244244
{
245245
try
246246
{
247247
TableInfo ti = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_REF_NT_SEQUENCES);
248248
TableSelector ts = new TableSelector(ti);
249249
List<RefNtSequenceModel> nts = ts.getArrayList(RefNtSequenceModel.class);
250-
if (nts.size() > 100000)
250+
if (maxSequences > 0 && nts.size() > maxSequences)
251251
{
252252
log.info("There are too many sequences to migrate on startup. Please run MigrateSequenceFilesAction");
253253
return;

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceMigrationPipelineJob.java‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package org.labkey.sequenceanalysis.pipeline;
2+
3+
import org.jetbrains.annotations.NotNull;
4+
import org.labkey.api.data.Container;
5+
import org.labkey.api.files.FileUrls;
6+
import org.labkey.api.module.Module;
7+
import org.labkey.api.pipeline.AbstractTaskFactory;
8+
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
9+
import org.labkey.api.pipeline.PipeRoot;
10+
import org.labkey.api.pipeline.PipelineDirectory;
11+
import org.labkey.api.pipeline.PipelineJob;
12+
import org.labkey.api.pipeline.PipelineJobException;
13+
import org.labkey.api.pipeline.PipelineJobService;
14+
import org.labkey.api.pipeline.PipelineProvider;
15+
import org.labkey.api.pipeline.RecordedActionSet;
16+
import org.labkey.api.pipeline.TaskId;
17+
import org.labkey.api.pipeline.TaskPipeline;
18+
import org.labkey.api.security.User;
19+
import org.labkey.api.util.FileType;
20+
import org.labkey.api.util.FileUtil;
21+
import org.labkey.api.util.PageFlowUtil;
22+
import org.labkey.api.view.ActionURL;
23+
import org.labkey.api.view.ViewBackgroundInfo;
24+
import org.labkey.api.view.ViewContext;
25+
import org.labkey.sequenceanalysis.SequenceAnalysisUpgradeCode;
26+
27+
import java.io.File;
28+
import java.util.Collections;
29+
import java.util.List;
30+
31+
public class SequenceMigrationPipelineJob extends PipelineJob
32+
{
33+
public static class Provider extends PipelineProvider
34+
{
35+
public static final String NAME = "sequenceMigrationPipeline";
36+
37+
public Provider(Module owningModule)
38+
{
39+
super(NAME, owningModule);
40+
}
41+
42+
@Override
43+
public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirectory directory, boolean includeAll)
44+
{
45+
46+
}
47+
}
48+
49+
// Default constructor for serialization
50+
protected SequenceMigrationPipelineJob()
51+
{
52+
}
53+
54+
public SequenceMigrationPipelineJob(Container c, User u, ActionURL url, PipeRoot pipeRoot)
55+
{
56+
super(Provider.NAME, new ViewBackgroundInfo(c, u, url), pipeRoot);
57+
58+
File subdir = FileUtil.appendName(pipeRoot.getRootPath(), Provider.NAME);
59+
if (!subdir.exists())
60+
{
61+
subdir.mkdirs();
62+
}
63+
64+
setLogFile(FileUtil.appendName(subdir, FileUtil.makeFileNameWithTimestamp("sequenceMigration", "log")));
65+
66+
}
67+
68+
@Override
69+
public ActionURL getStatusHref()
70+
{
71+
return PageFlowUtil.urlProvider(FileUrls.class).urlBegin(getContainer());
72+
}
73+
74+
@Override
75+
public String getDescription()
76+
{
77+
return "Migrate Sequence Files";
78+
}
79+
80+
@Override
81+
public TaskPipeline<?> getTaskPipeline()
82+
{
83+
return PipelineJobService.get().getTaskPipeline(new TaskId(SequenceMigrationPipelineJob.class));
84+
}
85+
86+
public static class Task extends PipelineJob.Task<Task.Factory>
87+
{
88+
protected Task(Factory factory, PipelineJob job)
89+
{
90+
super(factory, job);
91+
}
92+
93+
public static class Factory extends AbstractTaskFactory<AbstractTaskFactorySettings, Factory>
94+
{
95+
public Factory()
96+
{
97+
super(Task.class);
98+
}
99+
100+
@Override
101+
public List<FileType> getInputTypes()
102+
{
103+
return Collections.emptyList();
104+
}
105+
106+
@Override
107+
public String getStatusName()
108+
{
109+
return PipelineJob.TaskStatus.running.toString();
110+
}
111+
112+
@Override
113+
public List<String> getProtocolActionNames()
114+
{
115+
return List.of("Migrate Sequence Files");
116+
}
117+
118+
@Override
119+
public PipelineJob.Task<?> createTask(PipelineJob job)
120+
{
121+
return new Task(this, job);
122+
}
123+
124+
@Override
125+
public boolean isJobComplete(PipelineJob job)
126+
{
127+
return false;
128+
}
129+
}
130+
131+
private SequenceMigrationPipelineJob getPipelineJob()
132+
{
133+
return (SequenceMigrationPipelineJob)getJob();
134+
}
135+
136+
@Override
137+
public @NotNull RecordedActionSet run() throws PipelineJobException
138+
{
139+
SequenceAnalysisUpgradeCode.doSequenceMigration(getPipelineJob().getUser(), getPipelineJob().getLogger(), -1);
140+
141+
return new RecordedActionSet();
142+
}
143+
}
144+
}

‎SequenceAnalysis/webapp/WEB-INF/SequenceAnalysis/sequenceanalysisContext.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@
129129
<bean class="org.labkey.sequenceanalysis.pipeline.CacheGenomePipelineJob.CacheGenomesTask$Factory">
130130
<property name="location" value="webserver" />
131131
</bean>
132+
<bean class="org.labkey.sequenceanalysis.pipeline.SequenceMigrationPipelineJob.Task$Factory">
133+
<property name="location" value="webserver-high-priority" />
134+
</bean>
132135
</list>
133136
</property>
134137
<property name="pipelines">
@@ -234,6 +237,14 @@
234237
</list>
235238
</property>
236239
</bean>
240+
<bean class="org.labkey.api.pipeline.TaskPipelineSettings">
241+
<constructor-arg type="java.lang.Class" value="org.labkey.sequenceanalysis.pipeline.SequenceMigrationPipelineJob"/>
242+
<property name="taskProgressionSpec">
243+
<list>
244+
<value type="java.lang.Class">org.labkey.sequenceanalysis.pipeline.SequenceMigrationPipelineJob.Task</value>
245+
</list>
246+
</property>
247+
</bean>
237248
</list>
238249
</property>
239250
</bean>

0 commit comments

Comments
 (0)