public void GetIndelPositions()
{
var readPair = TestHelpers.GetPair("3M2D3M", "3M2I1M1D1M");
var indelPositions = OverlappingIndelHelpers.GetIndelPositions(readPair.Read1, out int totalIndelBasesR1);
Assert.Equal(1.0, indelPositions.Count);
Assert.Equal(2.0, totalIndelBasesR1);
var expectedR1DelStart = readPair.Read1.Position + 3 - 1;
var expectedR1DelEnd = expectedR1DelStart + 2 + 1;
Assert.Equal(expectedR1DelStart, indelPositions[0].PreviousMappedPosition);
Assert.Equal(expectedR1DelEnd, indelPositions[0].NextMappedPosition);
var indelPositions2 = OverlappingIndelHelpers.GetIndelPositions(readPair.Read2, out int totalIndelBasesR2);
Assert.Equal(2, indelPositions2.Count);
Assert.Equal(3, totalIndelBasesR2);
var expectedR2InsStart = readPair.Read2.Position + 3 - 1;
var expectedR2InsEnd = expectedR2InsStart + 1;
var expectedR2DelStart = readPair.Read2.Position + 4 - 1;
var expectedR2DelEnd = expectedR2DelStart + 1 + 1;
Assert.Equal(expectedR2InsStart, indelPositions2[0].PreviousMappedPosition);
Assert.Equal(expectedR2InsEnd, indelPositions2[0].NextMappedPosition);
Assert.Equal(expectedR2DelStart, indelPositions2[1].PreviousMappedPosition);
Assert.Equal(expectedR2DelEnd, indelPositions2[1].NextMappedPosition);
}
private PairResult HandleIndelPairIfStitchUnallowed(ReadPair readPair, int numMismatchesInR1, int numMismatchesInR2, bool r1HasIndels, bool r2HasIndels)
{
IEnumerable <BamAlignment> bamAlignmentList = null;
PairClassification classification;
var read1 = readPair.Read1;
var read2 = readPair.Read2;
if (read1.EndPosition >= read2.Position)
{
bamAlignmentList = OverlappingIndelHelpers.IndelsDisagreeWithStrongMate(
read1, read2, out bool disagree, 3, false);
// TODO allow to stitch if they don't disagree, as they may not necessarily get the chance later (either user is not using realigner, or there are no indels strong enough to realign against)
// Alternatively, if there are no indels to realign against, still stitch stuff if we can! (handle this in the realigner)
// For the cases where we want to skip realignment, either tell it to stitch here (configurable), or have it still go through realigner but not realign?
if (disagree)
{
classification = PairClassification.Disagree;
}
else
{
classification = PairClassification.UnstitchIndel;
}
}
else
{
classification = PairClassification.UnstitchIndel;
}
return(HandlePairContainingIndels(readPair, r1HasIndels, r2HasIndels, numMismatchesInR1, numMismatchesInR2,
r1HasIndels || r2HasIndels, classification, false, bamAlignmentList));
}
private void CheckR1IndelCoveredInMate(string read1Cigar, string read2Cigar, bool expectedResult, int r2Offset = 0)
{
var readPair = TestHelpers.GetPair(read1Cigar, read2Cigar, read2Offset: r2Offset);
var indelPositions = OverlappingIndelHelpers.GetIndelPositions(readPair.Read1, out int totalIndelBasesR1);
var coveredInMate = OverlappingIndelHelpers.AnyIndelCoveredInMate(indelPositions, readPair.Read2, readPair.Read1, anchorSize: 0);
Assert.Equal(expectedResult, coveredInMate > 0);
}
private void CheckReadsDisagreeTest(ReadPair readPair, bool shouldDisagree, string expectedCigarR1,
string expectedCigarR2, bool softclipWeakOne = false)
{
var result =
OverlappingIndelHelpers.IndelsDisagreeWithStrongMate(readPair.Read1, readPair.Read2, out bool disagree, 1, softclipWeakOne: softclipWeakOne);
Assert.Equal(shouldDisagree, disagree);
Assert.Equal(expectedCigarR1, result[0].CigarData.ToString());
Assert.Equal(expectedCigarR2, result[1].CigarData.ToString());
}
private List <BamAlignment> ShouldRestitch(ReadPair pair)
{
var result = OverlappingIndelHelpers.IndelsDisagreeWithStrongMate(pair.Read1, pair.Read2, out bool disagree);
if (disagree)
{
return(result);
}
return(null);
}
private void HandleFailedRealignment(BamAlignment origBamAlignment, ref bool forcedSoftclip, List <PreIndel> existingIndels,
RealignmentResult realignResult, bool hasExistingUnsanctionedIndels,
List <PreIndel> existingMatches)
{
_statusCounter.AddStatusCount("INDEL STATUS\tRejected\t" + realignResult.Indels);
_statusCounter.AppendStatusStringTag("RX", "Did not accept: " + realignResult.Indels, origBamAlignment);
// TODO could this be happening because of a low-ranked indel? Maybe we should be allowing to realign against all indels...
// TODO STILL should this actually be happening also to reads that had no indels to realign around (i.e. started with weak indel, and couldn't go anywhere), not just the ones that were changed?
if (_softclipUnknownIndels && hasExistingUnsanctionedIndels)
{
var unsanctioned = existingIndels.Where(x => !existingMatches.Contains(x));
foreach (var preIndel in unsanctioned.OrderBy(x => x.ReferencePosition))
{
var reverseClip = false;
var clipLength = preIndel.RightAnchor;
if (preIndel.LeftAnchor < preIndel.RightAnchor)
{
reverseClip = true;
clipLength = preIndel.LeftAnchor;
}
// TODO arbitrary number here...
// If it's pretty well-anchored, don't remove the indel
if (clipLength > 20)
{
continue;
}
forcedSoftclip = true;
_statusCounter.AddStatusCount("Softclipped out bad indel");
_statusCounter.AppendStatusStringTag("RX",
$"Softclipped out bad indel({origBamAlignment.CigarData},{string.Join(",", existingIndels)}...{realignResult?.Indels}",
origBamAlignment);
_statusCounter.AddStatusCount("INDEL STATUS\tRemoved\t" + string.Join("|", existingIndels));
OverlappingIndelHelpers.SoftclipAfterIndel(origBamAlignment,
reverseClip, preIndel.ReferencePosition);
}
}
}
public PairResult GetBamAlignmentsAndClassification(ReadPair readPair, IReadPairHandler pairHandler)
{
if (readPair.PairStatus == PairStatus.Duplicate)
{
// TODO hasIndels and numMismatches and split don't have meaning in an unusable dup, but it's a bit misleading to set them..
// Also, it's kind of silly to extract those alignments if we're going to set it to unusable anyway.
var alignments = readPair.GetAlignments().ToList();
return(new PairResult(alignments: alignments, readPair: readPair, classification: PairClassification.Duplicate, hasIndels: false,
isSplit: false, numMismatchesInSingleton: 0, softclipLengthForIndelRead: 0)
{
IsReputableIndelContaining = false
});
}
var classification = PairClassification.Unknown;
IEnumerable <BamAlignment> bamAlignmentList = null;
int?numMismatchesInR1 = null;
int?numMismatchesInR2 = null;
var r1HasIndels = OverlappingIndelHelpers.ReadContainsIndels(readPair.Read1);
var r2HasIndels = OverlappingIndelHelpers.ReadContainsIndels(readPair.Read2);
var hasIndels = r1HasIndels || r2HasIndels;
if (IsCompletedPairedPair(readPair))
{
if (BothReadsHighQuality(readPair))
{
numMismatchesInR1 = readPair.Read1.GetIntTag("NM");
numMismatchesInR2 = readPair.Read2.GetIntTag("NM");
var tryStitch = true;
if (hasIndels)
{
if (numMismatchesInR1 == null && numMismatchesInR2 == null)
{
Logger.WriteWarningToLog(
$"Found indel-containing read without NM: '{readPair.Name}', likely indicating that NM is not set on any read. Consider preprocessing the BAM to calculate NM tags for best results.");
}
return(HandleIndelPairIfStitchUnallowed(readPair, numMismatchesInR1 ?? 0,
numMismatchesInR2 ?? 0, r1HasIndels, r2HasIndels));
}
else
{
// TODO if not realigning anything (or not realigning imperfects), go ahead and stitch immediately
// TODO why are we using this bool multiple times
// ^ Because there's no point checking for imperfectinos if we don't care about softclips -- we already know there are no indels because this is in the else of if(hasIndels)
if (!_trustSoftclips && (ReadContainsImperfections(readPair.Read1, _trustSoftclips) ||
ReadContainsImperfections(readPair.Read2, _trustSoftclips)))
{
tryStitch = false;
classification =
ClassifySoftclipContainingPairGivenSoftclipDistrust(readPair, numMismatchesInR1,
numMismatchesInR2);
bamAlignmentList = readPair.GetAlignments();
}
else
{
if (numMismatchesInR1 == null)
{
numMismatchesInR1 = readPair.Read1.GetIntTag("NM");
}
if (numMismatchesInR2 == null)
{
numMismatchesInR2 = readPair.Read2.GetIntTag("NM");
}
if (numMismatchesInR1 >= _numMismatchesToBeConsideredMessy ||
numMismatchesInR2 >= _numMismatchesToBeConsideredMessy)
{
classification = PairClassification.UnstitchMessy;
tryStitch = false;
if (numMismatchesInR1 <= 1 || numMismatchesInR2 <= 1)
{
// One of the reads is clean
tryStitch = false;
if (numMismatchesInR1 <= 1)
{
// R1 is the clean one.
if (readPair.Read2.IsReverseStrand())
{
classification = PairClassification.UnstitchReverseMessy;
}
else
{
classification = PairClassification.UnstitchForwardMessy;
}
}
else
{
if (readPair.Read1.IsReverseStrand())
{
classification = PairClassification.UnstitchReverseMessy;
}
else
{
classification = PairClassification.UnstitchForwardMessy;
}
}
}
bamAlignmentList = readPair.GetAlignments();
}
else if (numMismatchesInR1 + numMismatchesInR2 == 0)
{
classification = PairClassification.UnstitchPerfect;
bamAlignmentList = readPair.GetAlignments();
}
else if (numMismatchesInR1 <= 1 && numMismatchesInR2 <= 1)
{
classification = PairClassification.UnstitchSingleMismatch;
bamAlignmentList = readPair.GetAlignments();
}
else
{
classification = PairClassification.UnstitchImperfect;
bamAlignmentList = readPair.GetAlignments();
}
}
classification = AdjustClassificationForMultimapper(readPair, classification);
}
if (classification == PairClassification.UnstitchMessySuspiciousRead)
{
tryStitch = false;
}
if (_skipStitch)
{
tryStitch = false;
}
if (classification != PairClassification.UnstitchPerfect)
{
//For now we can't stitch anything else because we can't properly calculate NM!!
tryStitch = false;
}
if (!tryStitch)
{
if (bamAlignmentList == null)
{
bamAlignmentList = readPair.GetAlignments().ToList();
classification = PairClassification.Unstitchable;
}
}
else
{
bamAlignmentList = TryStitch(readPair, pairHandler, out classification);
}
}
else if (OneReadIsHighQuality(readPair))
{
classification = PairClassification.Split;
if (hasIndels)
{
numMismatchesInR1 = numMismatchesInR1 ?? readPair.Read1?.GetIntTag("NM") ?? 0;
numMismatchesInR2 = numMismatchesInR2 ?? readPair.Read2?.GetIntTag("NM") ?? 0;
return(HandlePairContainingIndels(readPair, r1HasIndels, r2HasIndels, numMismatchesInR1.Value,
numMismatchesInR2.Value, true, PairClassification.Split, true));
}
}
else
{
classification = PairClassification.Unusable;
bamAlignmentList = readPair.GetAlignments().ToList();
}
}
else
{
numMismatchesInR1 = numMismatchesInR1 ?? readPair.Read1?.GetIntTag("NM") ?? 0;
numMismatchesInR2 = numMismatchesInR2 ?? readPair.Read2?.GetIntTag("NM") ?? 0;
return(ClassifyIncompletePair(readPair, r1HasIndels, r2HasIndels, numMismatchesInR1.Value, numMismatchesInR2.Value));
}
// TODO - not sure why I originally had this double-check on whether pairs were split? shouldn't this already be evident from the pair status?
//var isSplit = bamAlignmentList?.Count() > 0 && bamAlignmentList?.Select(x => x.RefID).Distinct().Count() > 1;
var isSplit = false;
if (isSplit || classification == PairClassification.Split || readPair.PairStatus == PairStatus.SplitChromosomes ||
readPair.PairStatus == PairStatus.MateNotFound || readPair.PairStatus == PairStatus.MateUnmapped)
{
return(HandleSplitNonIndelPair(readPair, bamAlignmentList, hasIndels, isSplit));
}
var pr = new PairResult(bamAlignmentList.ToList(), readPair, classification, hasIndels, isSplit);
if (classification == PairClassification.UnstitchMessy || classification == PairClassification.UnstitchMessySuspiciousRead)
{
if (_checkMd && HasSuspiciousMd(readPair, numMismatchesInR1, numMismatchesInR2, pr))
{
classification = PairClassification.UnstitchMessySuspiciousMd;
pr.Classification = classification;
}
}
return(pr);
}
public BamAlignment GetFinalAlignment(BamAlignment origBamAlignment, out bool changed, out bool forcedSoftclip, out bool confirmed, out bool sketchy,
List <PreIndel> selectedIndels = null, List <PreIndel> existingIndels = null,
bool assumeImperfect = true, List <HashableIndel> confirmedAccepteds = null, List <PreIndel> mateIndels = null)
{
sketchy = false;
forcedSoftclip = false;
bool forcedAlignment = false;
var presumeStartPositionForForcedAlignment = 0;
if (origBamAlignment.CigarData.Count == 0)
{
// This was something weird that came up in the halo dataset... mapq is 0 but is still mapped, no cigar
if (origBamAlignment.Position <= 0 && origBamAlignment.FragmentLength != 0) // No sense trying to fiddle with the position otherwise
{
// TODO does this really even move the needle? Is it helping enough to outweigh its weirdness?
var presumedEndPosition = origBamAlignment.MatePosition < origBamAlignment.Position
? origBamAlignment.MatePosition - origBamAlignment.FragmentLength
: origBamAlignment.MatePosition + origBamAlignment.FragmentLength;
presumeStartPositionForForcedAlignment = presumedEndPosition - origBamAlignment.Bases.Length;
forcedAlignment = true;
}
else
{
presumeStartPositionForForcedAlignment = origBamAlignment.Position;
forcedAlignment = true;
}
}
var anyIndelsAtAll = _regionFilterer.AnyIndelsNearby(origBamAlignment.Position);
bool isRealignable = true;
if (anyIndelsAtAll)
{
var isImperfectRead = false || ((origBamAlignment.ContainsDisallowedCigarOps(_suspectCigarOps) ||
origBamAlignment.GetIntTag("NM") > 0 || forcedAlignment));
var isReadWorthCaringAbout = !origBamAlignment.IsDuplicate() && !origBamAlignment.IsSecondary();
isRealignable = isImperfectRead && isReadWorthCaringAbout && origBamAlignment.Bases.Distinct().Count() > 1;
}
else
{
_statusCounter.AddStatusCount("No indels nearby at all");
isRealignable = false;
}
if (!isRealignable)
{
confirmed = false;
changed = false;
sketchy = false;
return(origBamAlignment);
}
// TODO maybe flag (or return all) if there's a lot or high quality stuff that we're missing! Esp with pair specific
var indels = _indelSource.GetRelevantIndels(forcedAlignment ? presumeStartPositionForForcedAlignment : origBamAlignment.Position,
mateIndels, confirmedAccepteds);
// Don't realign around single indels if we already have them
bool hasExistingUnsanctionedIndels = false;
bool existingSanctionedIndelIsBest = false;
bool hasVeryGoodIndel = false;
bool hasHardToCallIndel = false;
var existingMatches = new List <PreIndel>();
HashableIndel existingConfirmedIndel = new HashableIndel();
var existingMatchHashables = new List <HashableIndel>();
if (indels.Any() && existingIndels != null && existingIndels.Any())
{
var topScore = (float)(indels.Max(x => x.Key.Score));
var matchesFound = 0;
var nonPreExistingIndels = new List <KeyValuePair <HashableIndel, GenomeSnippet> >();
var index = 0;
foreach (var kvp in indels)
{
var indel = kvp.Key;
var matches = existingIndels.Where(e => Helper.IsMatch(e, indel));
var isMatch = matches.Any();
if (isMatch)
{
matchesFound++;
if (!indel.InMulti && index == 0)
{
existingSanctionedIndelIsBest = true;
existingConfirmedIndel = indel;
}
var proportionOfTopScore = indel.Score / (float)topScore;
if (proportionOfTopScore >= 0.75)
{
hasVeryGoodIndel = true;
}
if (indel.HardToCall)
{
hasHardToCallIndel = true;
}
existingMatches.AddRange(matches);
// TODO do we need special handling of multis?
existingMatchHashables.Add(indel);
}
if (!isMatch || indel.InMulti)
{
nonPreExistingIndels.Add(kvp);
}
index++;
}
// TODO do we actually want to replace indels with non-pre-existing only?
indels = nonPreExistingIndels;
if (matchesFound == 0)
{
hasExistingUnsanctionedIndels = true;
}
}
// TODO this precludes us from having good multis
if (existingSanctionedIndelIsBest)
{
// If it already had the top ranked indel, there's not really any point in trying to realign around others (here we assume that it's also the best fitting indel for the read, hence why it was originally called by the regular aligner).
_statusCounter.AddStatusCount("Existing indel is already the best available");
changed = false;
confirmed = true;
UpdateOutcomeForConfirmed(existingConfirmedIndel);
if (confirmedAccepteds == null)
{
confirmedAccepteds = new List <HashableIndel>();
}
confirmedAccepteds.Add(existingConfirmedIndel);
return(origBamAlignment);
}
if (!indels.Any() || origBamAlignment.EndPosition - origBamAlignment.Position > 500)
{
if (!indels.Any())
{
// TODO maybe do the forced softclip here if the read did have indels?
_statusCounter.AddStatusCount("No indels to realign to");
_statusCounter.AppendStatusStringTag("RX", $"{origBamAlignment.GetStringTag("RX")},No indels to realign to", origBamAlignment);
}
else
{
_statusCounter.AddStatusCount("Alignment reference span longer than we can realign to");
}
changed = false;
confirmed = false;
return(origBamAlignment);
}
// TODO this should relate to cap on indel size... introducing too large of an indel will make us go beyond this context.
var context = indels.First().Value;
var orderedIndels = indels.Select(x => x.Key).ToList();
var numIndels = orderedIndels.Count;
_statusCounter.AddStatusCount("Realigning to " + numIndels);
var bamAlignment = new BamAlignment(origBamAlignment);
if (forcedAlignment)
{
bamAlignment.CigarData = new CigarAlignment(origBamAlignment.Bases.Length + "M");
bamAlignment.Position = presumeStartPositionForForcedAlignment;
}
var realignResult = _readRealigner.Realign(new Read(_chromosome, bamAlignment),
orderedIndels, indels.ToDictionary(x => x.Key, x => x.Value), confirmedAccepteds != null && confirmedAccepteds.Any());
var acceptedIndels = realignResult?.AcceptedIndels;
var hasAnyIndels = acceptedIndels != null && acceptedIndels.Any();
if (realignResult != null)
{
_statusCounter.AddStatusCount("Able to realign at all (may still be worse than original)");
_statusCounter.AppendStatusStringTag("RX", "Able to realign at all(may still be worse than original)", bamAlignment);
}
else
{
_statusCounter.AddStatusCount("Not able to realign at all");
_statusCounter.AppendStatusStringTag("RX", "Not able to realign at all", origBamAlignment);
}
AlignmentSummary originalAlignmentSummary = null;
var realignmentUnchanged = true;
if (realignResult != null)
{
originalAlignmentSummary =
Extensions.GetAlignmentSummary((new Read(_chromosome, origBamAlignment)), context.Sequence,
_trackActualMismatches, _checkSoftclipsForMismatches, context.StartPosition);
realignmentUnchanged = _judger.RealignmentIsUnchanged(realignResult, origBamAlignment);
if (originalAlignmentSummary.NumMismatches > 0)
{
// TODO PERF do we still want to use this ever?
var sumMismatch = Helper.GetSumOfMismatchQualities(origBamAlignment.Qualities,
origBamAlignment.Bases, new Read(_chromosome, origBamAlignment).PositionMap, context.Sequence,
context.StartPosition);
originalAlignmentSummary.SumOfMismatchingQualities = sumMismatch;
}
// Within this logic also checking the same as "!realignmentUnchanged" above.. consolidate this.
if (selectedIndels != null &&
(_judger.RealignmentBetterOrEqual(realignResult, originalAlignmentSummary, confirmedAccepteds != null && confirmedAccepteds.Any())) ||
ResultIsGoodEnough(realignResult, origBamAlignment, originalAlignmentSummary,
realignmentUnchanged, confirmedAccepteds != null && confirmedAccepteds.Any()))
{
UpdateIndelOutcomes(numIndels, orderedIndels, hasAnyIndels, acceptedIndels, confirmedAccepteds, true, realignResult);
if (realignResult.IsSketchy)
{
sketchy = true;
}
return(AcceptRealignment(origBamAlignment, out changed, selectedIndels, existingIndels, realignResult, originalAlignmentSummary, bamAlignment, hasExistingUnsanctionedIndels, out confirmed));
}
}
// At this point, any good realignment would have been returned. If it's realigned and changed now, it's an unaccepted (not good enough) realignment.
// If it had an indel to begin with, it's basically a vote that we don't trust that indel. Optionally softclip it out.
if (!realignmentUnchanged)
{
changed = false;
confirmed = false;
HandleFailedRealignment(origBamAlignment, ref forcedSoftclip, existingIndels, realignResult, hasExistingUnsanctionedIndels, existingMatches);
if ((hasVeryGoodIndel || (hasHardToCallIndel && _judger.IsVeryConfident(originalAlignmentSummary))) && !hasExistingUnsanctionedIndels && existingMatchHashables.Any())
{
// It didn't have the tip-top indel, but it had one that was very close, and we tried realigning around the top guys and failed - this one looks better. Give it credit.
confirmed = true;
foreach (var indel in existingMatchHashables)
{
UpdateOutcomeForConfirmed(indel);
if (confirmedAccepteds != null)
{
confirmedAccepteds.Add(indel);
}
}
}
UpdateIndelOutcomes(numIndels, orderedIndels, hasAnyIndels, acceptedIndels, confirmedAccepteds, false, realignResult);
}
else
{
if (acceptedIndels != null)
{
foreach (var indelNum in acceptedIndels)
{
var indel = orderedIndels[indelNum];
UpdateOutcomeForConfirmed(indel);
}
}
_statusCounter.AddStatusCount("INDEL STATUS\tUnchanged\t" + realignResult?.Indels);
_statusCounter.AppendStatusStringTag("RX", "Unchanged: " + realignResult?.Indels, origBamAlignment);
confirmed = true;
changed = false;
return(origBamAlignment);
}
if (realignResult == null)
{
if (_softclipUnknownIndels && hasExistingUnsanctionedIndels)
{
var unsanctioned = existingIndels.Where(x => !existingMatches.Contains(x));
foreach (var preIndel in unsanctioned.OrderBy(x => x.ReferencePosition))
{
var reverseClip = false;
var clipLength = preIndel.RightAnchor;
if (preIndel.LeftAnchor < preIndel.RightAnchor)
{
reverseClip = true;
clipLength = preIndel.LeftAnchor;
}
// TODO arbitrary number here...
// If it's pretty well-anchored, don't remove the indel
if (clipLength > 20)
{
continue;
}
forcedSoftclip = true;
_statusCounter.AddStatusCount("Softclipped out bad indel");
_statusCounter.AppendStatusStringTag("RX",
$"Softclipped out bad indel({origBamAlignment.CigarData},{string.Join(",", existingIndels)}... No realignment",
origBamAlignment);
_statusCounter.AddStatusCount("INDEL STATUS\tRemoved\t" + string.Join("|", existingIndels));
OverlappingIndelHelpers.SoftclipAfterIndel(origBamAlignment,
reverseClip, preIndel.ReferencePosition);
}
}
}
_statusCounter.AppendStatusStringTag("RX", "Realignment failed", origBamAlignment);
_statusCounter.AddStatusCount("Realignment failed");
return(origBamAlignment);
}
