Skip to content

Commit 5d255aa

Browse files
bbimberbbimber
committed
Fix merge conflicts
2 parents 0323ed3 + 62ffcda commit 5d255aa

15 files changed

Lines changed: 1400 additions & 1714 deletions

File tree

‎SequenceAnalysis/resources/schemas/dbscripts/postgresql/sequenceanalysis-12.333-12.334.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE sequenceanalysis.barcodes DROP CONSTRAINT IF EXISTS UNIQUE_barcodes;

‎SequenceAnalysis/resources/schemas/dbscripts/sqlserver/sequenceanalysis-12.333-12.334.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE sequenceanalysis.barcodes DROP CONSTRAINT IF EXISTS UNIQUE_barcodes;

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@
208208
import java.util.List;
209209
import java.util.Map;
210210
import java.util.Set;
211+
import java.util.stream.Collectors;
211212
import java.util.zip.GZIPInputStream;
212213
import java.util.zip.ZipEntry;
213214
import java.util.zip.ZipOutputStream;
@@ -717,7 +718,7 @@ else if (SequenceAnalysisSchema.TABLE_OUTPUTFILES.equals(_table.getName()))
717718
msg.append("Folder: ").unsafeAppend("<a href='" + url.toString() + "' target='_blank'>" + h(target.getPath()) + "</a><br><input type='hidden' name='jobIds' value='" + h(sf.getRowId()) + "'/>");
718719
}
719720
}
720-
msg.append("<br>");
721+
msg.unsafeAppend("<br>");
721722
}
722723
}
723724
}
@@ -1914,7 +1915,7 @@ else if (!d.getFile().exists())
19141915
{
19151916
throw new PipelineValidationException("Missing file for data: " + o.get("dataId"));
19161917
}
1917-
else if (d.getContainer().hasPermission(u, ReadPermission.class))
1918+
else if (!d.getContainer().hasPermission(u, ReadPermission.class))
19181919
{
19191920
throw new UnauthorizedException("You do not have permission to read data: " + o.get("dataId"));
19201921
}
@@ -5161,14 +5162,13 @@ public ApiResponse execute(ArchiveReadsetsForm form, BindException errors) throw
51615162

51625163
if (!toUpdate.isEmpty())
51635164
{
5164-
List<Map<String, Object>> keys = new ArrayList<>();
5165-
toUpdate.forEach(row -> {
5166-
keys.add(new CaseInsensitiveHashMap<>(Map.of("rowid", row.get("rowid"))));
5167-
});
5165+
// Remove duplicates:
5166+
List<Map<String, Object>> uniqueToUpdate = toUpdate.stream().distinct().toList();
5167+
List<Map<String, Object>> keys = uniqueToUpdate.stream().map(row -> new CaseInsensitiveHashMap<>(Map.of("rowid", row.get("rowid")))).collect(Collectors.toList());
51685168

51695169
try
51705170
{
5171-
readData.getUpdateService().updateRows(getUser(), getContainer(), toUpdate, keys, null, null);
5171+
readData.getUpdateService().updateRows(getUser(), getContainer(), uniqueToUpdate, keys, null, null);
51725172
}
51735173
catch (Exception e)
51745174
{

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java‎

Lines changed: 295 additions & 226 deletions
Large diffs are not rendered by default.

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ public String getName()
213213
@Override
214214
public Double getSchemaVersion()
215215
{
216-
return 12.333;
216+
return 12.334;
217217
}
218218

219219
@Override

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
import java.util.List;
6969
import java.util.Map;
7070
import java.util.Set;
71+
import java.util.concurrent.Future;
7172

7273
import static org.labkey.api.util.IntegerUtils.asInteger;
7374

@@ -467,7 +468,7 @@ public RecordedActionSet run() throws PipelineJobException
467468
{
468469
getJob().getLogger().info("running genome trigger: " + t.getName());
469470
final int libraryId = rowId;
470-
jr.execute(new Job()
471+
Future<?> future = jr.execute(new Job()
471472
{
472473
@Override
473474
public void run()
@@ -481,11 +482,12 @@ public void run()
481482
t.onRecreate(getJob().getContainer(), getJob().getUser(), getJob().getLogger(), libraryId);
482483
}
483484
}
484-
});
485+
}, 0);
486+
487+
// Wait for this job:
488+
future.get();
485489
}
486490
}
487-
488-
jr.waitForCompletion();
489491
}
490492
}
491493
catch (Exception e)

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
import java.util.List;
8282
import java.util.Map;
8383
import java.util.Set;
84+
import java.util.concurrent.Future;
8485

8586
import static org.labkey.api.util.IntegerUtils.asInteger;
8687

@@ -163,18 +164,19 @@ public boolean isJobComplete(PipelineJob job)
163164
if (t.isAvailable(genomeContainer))
164165
{
165166
getJob().getLogger().info("running genome trigger: " + t.getName());
166-
jr.execute(new Job()
167+
Future<?> future = jr.execute(new Job()
167168
{
168169
@Override
169170
public void run()
170171
{
171172
t.onTrackAdd(genomeContainer, getJob().getUser(), getJob().getLogger(), libraryId, trackId);
172173
}
173-
});
174+
}, 0);
175+
176+
// Wait for the job:
177+
future.get();
174178
}
175179
}
176-
177-
jr.waitForCompletion();
178180
}
179181

180182
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import org.labkey.sequenceanalysis.ReadDataImpl;
7272
import org.labkey.sequenceanalysis.SequenceReadsetImpl;
7373
import org.labkey.sequenceanalysis.run.RestoreSraDataHandler;
74+
import org.labkey.sequenceanalysis.run.SamtoolsMerger;
7475
import org.labkey.sequenceanalysis.run.bampostprocessing.SortSamStep;
7576
import org.labkey.sequenceanalysis.run.preprocessing.TrimmomaticWrapper;
7677
import org.labkey.sequenceanalysis.run.util.AddOrReplaceReadGroupsWrapper;
@@ -1306,7 +1307,6 @@ private File doAlignThenMerge(ReferenceGenome referenceGenome, Readset rs, Map<R
13061307
RecordedAction mergeAction = new RecordedAction(MERGE_ALIGNMENT_ACTIONNAME);
13071308
Date start = new Date();
13081309
mergeAction.setStartTime(start);
1309-
MergeSamFilesWrapper mergeSamFilesWrapper = new MergeSamFilesWrapper(getJob().getLogger());
13101310
List<File> bams = new ArrayList<>();
13111311
for (File o : alignOutputs)
13121312
{
@@ -1316,20 +1316,45 @@ private File doAlignThenMerge(ReferenceGenome referenceGenome, Readset rs, Map<R
13161316
getHelper().getFileManager().addIntermediateFile(SequenceAnalysisService.get().getExpectedBamOrCramIndex(o));
13171317
}
13181318

1319-
bam = new File(alignOutputs.get(0).getParent(), FileUtil.getBaseName(alignOutputs.get(0).getName()) + ".merged.bam");
1319+
bam = new File(alignOutputs.getFirst().getParent(), FileUtil.getBaseName(alignOutputs.getFirst().getName()) + ".merged.bam");
13201320
getHelper().getFileManager().addOutput(mergeAction, "Merged BAM", bam);
1321+
Set<SAMFileHeader.SortOrder> sortOrders = alignOutputs.stream().map(x -> {
1322+
try
1323+
{
1324+
return SequenceUtil.getBamSortOrder(x);
1325+
}
1326+
catch (IOException e)
1327+
{
1328+
throw new RuntimeException(e);
1329+
}
1330+
}).collect(Collectors.toSet());
1331+
13211332
//NOTE: merged BAMs will be deleted as intermediate files, and if we delete too early this breaks job resume
1322-
mergeSamFilesWrapper.execute(bams, bam, false);
1323-
getHelper().getFileManager().addCommandsToAction(mergeSamFilesWrapper.getCommandsExecuted(), mergeAction);
1333+
String toolName;
1334+
if (sortOrders.size() > 1 || sortOrders.iterator().next() != SAMFileHeader.SortOrder.coordinate)
1335+
{
1336+
toolName = "MergeSamFiles";
1337+
MergeSamFilesWrapper merger = new MergeSamFilesWrapper(getPipelineJob().getLogger());
1338+
merger.execute(bams, bam, false);
1339+
getHelper().getFileManager().addCommandsToAction(merger.getCommandsExecuted(), mergeAction);
1340+
}
1341+
else
1342+
{
1343+
// This will be faster, but requires sorted input:
1344+
toolName = "Samtools merge";
1345+
SamtoolsMerger merger = new SamtoolsMerger(getPipelineJob().getLogger());
1346+
merger.mergeBams(bams, bam);
1347+
getHelper().getFileManager().addCommandsToAction(merger.getCommandsExecuted(), mergeAction);
1348+
}
13241349

13251350
Date end = new Date();
13261351
mergeAction.setEndTime(end);
1327-
getJob().getLogger().info("MergeSamFiles Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true));
1352+
getJob().getLogger().info(toolName + " Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true));
13281353
alignActions.add(mergeAction);
13291354
}
13301355
else
13311356
{
1332-
bam = alignOutputs.get(0);
1357+
bam = alignOutputs.getFirst();
13331358
}
13341359

13351360
return bam;

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/SamtoolsMerger.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package org.labkey.sequenceanalysis.run;
2+
3+
import org.apache.logging.log4j.Logger;
4+
import org.labkey.api.pipeline.PipelineJobException;
5+
import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner;
6+
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
7+
8+
import java.io.File;
9+
import java.util.ArrayList;
10+
import java.util.List;
11+
12+
public class SamtoolsMerger extends SamtoolsRunner
13+
{
14+
private static final String COMMAND = "merge";
15+
16+
public SamtoolsMerger(Logger log)
17+
{
18+
super(log);
19+
}
20+
21+
public File mergeBams(List<File> inputBams, File outputFile) throws PipelineJobException
22+
{
23+
getLogger().info("Merging SAM/BAM(s):");
24+
25+
List<String> params = new ArrayList<>();
26+
params.add(getSamtoolsPath().getPath());
27+
params.add(COMMAND);
28+
29+
Integer threads = SequencePipelineService.get().getMaxThreads(getLogger());
30+
if (threads != null)
31+
{
32+
params.add("--threads");
33+
params.add(String.valueOf(threads));
34+
}
35+
36+
params.add("-o");
37+
params.add(outputFile.getPath());
38+
39+
inputBams.forEach(f -> params.add(f.getPath()));
40+
41+
execute(params);
42+
43+
File idx = SequencePipelineService.get().ensureBamIndex(outputFile, getLogger(), false);
44+
if (!idx.exists())
45+
{
46+
throw new PipelineJobException("Unable to find BAM index: " + idx.getPath());
47+
}
48+
49+
return outputFile;
50+
}
51+
}

0 commit comments

Comments
 (0)