public IEnumerable <ISimplePosition> Recompose(List <ISimplePosition> simplePositions, List <int> functionBlockRanges)
{
var positionSet = PositionSet.CreatePositionSet(simplePositions, functionBlockRanges);
var alleleSet = positionSet.AlleleSet;
var alleleIndexBlockToSampleIndex = positionSet.AlleleBlockToSampleHaplotype;
int numSamples = positionSet.NumSamples;
_sequenceProvider.LoadChromosome(alleleSet.Chromosome);
int regionStart = alleleSet.Starts[0];
string lastRefAllele = alleleSet.VariantArrays.Last()[0];
int regionEnd = alleleSet.Starts.Last() + lastRefAllele.Length + 100; // make it long enough
if (regionEnd > _sequenceProvider.Sequence.Length)
{
regionEnd = _sequenceProvider.Sequence.Length;
}
string totalRefSequence = _sequenceProvider.Sequence.Substring(regionStart - 1, regionEnd - regionStart); // VCF positions are 1-based
var recomposedAlleleSet = new RecomposedAlleleSet(positionSet.ChrName, numSamples);
foreach (var(alleleIndexBlock, sampleAlleles) in alleleIndexBlockToSampleIndex)
{
(int start, _, string refAllele, string altAllele, var varPosIndexesInAlleleBlock, List <string> decomposedVids) = GetPositionsAndRefAltAlleles(alleleIndexBlock, alleleSet, totalRefSequence, regionStart, simplePositions);
var variantSite = new VariantSite(start, refAllele);
if (!recomposedAlleleSet.RecomposedAlleles.TryGetValue(variantSite, out var variantInfo))
{
variantInfo = GetVariantInfo(positionSet, alleleIndexBlock);
recomposedAlleleSet.RecomposedAlleles[variantSite] = variantInfo;
}
variantInfo.AddAllele(altAllele, sampleAlleles, decomposedVids);
variantInfo.UpdateSampleFilters(varPosIndexesInAlleleBlock, sampleAlleles);
}
return(recomposedAlleleSet.GetRecomposedPositions(_sequenceProvider.RefNameToChromosome));
}
private bool IsNRegion(IChromosome chrom, int start, int end)
{
if (_refProvider == null)
{
return(false);
}
_refProvider.LoadChromosome(chrom);
var sequence = _refProvider.Sequence.Substring(start - 1, end - start + 1);
if (sequence == null)
{
return(false);
}
foreach (char c in sequence)
{
if (c != 'N' && c != 'n')
{
return(false);
}
}
_nRegionSize += end - start + 1;
return(true);
}
public IVariant[] CreateVariants(IChromosome chromosome, int start, int end, string refAllele,
string[] altAlleles, IInfoData infoData, bool[] isDecomposed, bool isRecomposed, List <string>[] linkedVids, string globalMajorAllele)
{
string firstAltAllele = altAlleles[0];
bool isReference = globalMajorAllele != null;
bool isSymbolicAllele = IsSymbolicAllele(firstAltAllele);
var variantCategory = GetVariantCategory(firstAltAllele, isReference, isSymbolicAllele, infoData.SvType);
if (isReference)
{
return new[] { GetVariant(chromosome, start, end, refAllele, firstAltAllele, infoData, variantCategory, isDecomposed[0], isRecomposed, linkedVids?[0]?.ToArray(), globalMajorAllele) }
}
;
_sequenceProvider.LoadChromosome(chromosome);
var variants = new List <IVariant>();
for (var i = 0; i < altAlleles.Length; i++)
{
#if (!NI_ALLELE)
if (VcfCommon.IsNonInformativeAltAllele(altAlleles[i]))
{
continue;
}
#endif
bool isDecomposedVar = isDecomposed[i];
(int shiftedStart, string shiftedRef, string shiftedAlt) =
VariantUtils.TrimAndLeftAlign(start, refAllele, altAlleles[i], _sequenceProvider.Sequence);
variants.Add(GetVariant(chromosome, shiftedStart, end - (start - shiftedStart), shiftedRef, shiftedAlt, infoData, variantCategory, isDecomposedVar, isRecomposed, linkedVids?[i]?.ToArray(), null));
}
return(variants.Count == 0 ? null : variants.ToArray());
}
public static (ImmutableDictionary <IChromosome, List <int> > PositionsByChromosome, int Count) GetPositions(Stream vcfStream, GenomicRange genomicRange,
ISequenceProvider sequenceProvider, IRefMinorProvider refMinorProvider)
{
var positionsByChromosome = new Dictionary <IChromosome, List <int> >();
var rangeChecker = new GenomicRangeChecker(genomicRange);
var refNameToChrom = sequenceProvider.RefNameToChromosome;
using (var reader = new StreamReader(vcfStream))
{
string line;
string currentReferenceName = "";
IChromosome chromosome = null;
while ((line = reader.ReadLine()) != null)
{
if (line.StartsWith('#'))
{
continue;
}
string[] cols = line.OptimizedSplit('\t');
string referenceName = cols[VcfCommon.ChromIndex];
if (referenceName != currentReferenceName)
{
if (!refNameToChrom.TryGetValue(referenceName, out chromosome))
{
continue;
}
currentReferenceName = referenceName;
}
(int position, bool foundError) = cols[VcfCommon.PosIndex].OptimizedParseInt32();
if (foundError)
{
throw new InvalidDataException($"Unable to convert the VCF position to an integer: {cols[VcfCommon.PosIndex]}");
}
if (rangeChecker.OutOfRange(chromosome, position))
{
break;
}
string refAllele = cols[VcfCommon.RefIndex];
string altAllele = cols[VcfCommon.AltIndex];
if (altAllele == "." && !IsRefMinor(refMinorProvider, chromosome, position))
{
continue;
}
sequenceProvider.LoadChromosome(chromosome);
TryAddPosition(positionsByChromosome, chromosome, position, refAllele, altAllele, sequenceProvider.Sequence);
}
}
int count = SortPositionsAndGetCount(positionsByChromosome);
return(positionsByChromosome.ToImmutableDictionary(), count);
}
private List <ClinVarItem> GetValidVariants(List <ClinVarItem> clinVarItems)
{
var shiftedItems = new List <ClinVarItem>();
foreach (var item in clinVarItems)
{
_sequenceProvider.LoadChromosome(item.Chromosome);
if (!ValidateRefAllele(item))
{
continue;
}
string refAllele = item.ReferenceAllele, altAllele = item.AlternateAllele;
if (string.IsNullOrEmpty(item.ReferenceAllele) && item.VariantType == "Deletion")
{
refAllele = GetReferenceAllele(item, _sequenceProvider.Sequence);
}
if (string.IsNullOrEmpty(item.ReferenceAllele) && item.VariantType == "Indel" && !string.IsNullOrEmpty(item.AlternateAllele))
{
refAllele = GetReferenceAllele(item, _sequenceProvider.Sequence);
}
if (string.IsNullOrEmpty(item.AlternateAllele) && item.VariantType == "Duplication")
{
altAllele = GetAltAllele(item, _sequenceProvider.Sequence);
}
if (string.IsNullOrEmpty(refAllele) && string.IsNullOrEmpty(altAllele))
{
continue;
}
int start;
(start, refAllele, altAllele) = LeftShift(item.Start, refAllele, altAllele);
shiftedItems.Add(new ClinVarItem(item.Chromosome,
start,
item.Stop,
item.AlleleOrigins,
altAllele,
item.VariantType,
item.Id,
item.ReviewStatus,
item.MedGenIDs,
item.OmimIDs,
item.OrphanetIDs,
item.Phenotypes,
refAllele,
item.Significance,
item.PubmedIds,
item.LastUpdatedDate));
}
shiftedItems.Sort();
return(shiftedItems);
}
public int Write(IEnumerable <ISupplementaryDataItem> saItems)
{
var itemsMinHeap = new MinHeap <ISupplementaryDataItem>(SuppDataUtilities.CompareTo);
var chromIndex = ushort.MaxValue;
var currentEnsemblName = "";
_count = 0;
var benchmark = new Benchmark();
foreach (var saItem in saItems)
{
if (chromIndex != saItem.Chromosome.Index)
{
if (chromIndex != ushort.MaxValue)
{
//flushing out the remaining items in buffer
WriteUptoPosition(itemsMinHeap, int.MaxValue);
Flush(chromIndex);
Console.WriteLine($"Chromosome {currentEnsemblName} completed in {Benchmark.ToHumanReadable(benchmark.GetElapsedTime())}");
benchmark.Reset();
}
chromIndex = saItem.Chromosome.Index;
currentEnsemblName = saItem.Chromosome.EnsemblName;
_refProvider.LoadChromosome(saItem.Chromosome);
}
// the items come in sorted order of the pre-trimmed position.
// So when writing out, we have to make sure that we do not write past this position.
// Once a position has been seen in the stream, we can safely write all positions before that.
var writeToPos = saItem.Position;
string refSequence = _refProvider.Sequence.Substring(saItem.Position - 1, saItem.RefAllele.Length);
if (!string.IsNullOrEmpty(saItem.RefAllele) && saItem.RefAllele != refSequence)
{
if (_skipIncorrectRefEntries)
{
continue;
}
throw new UserErrorException($"The provided reference allele {saItem.RefAllele} at {saItem.Chromosome.UcscName}:{saItem.Position} is different from {refSequence} in the reference genome sequence.");
}
itemsMinHeap.Add(saItem);
// in order to allow room for left shifted variants, we hold off on removing them from the heap
WriteUptoPosition(itemsMinHeap, writeToPos - VariantUtils.MaxUpstreamLength);
}
//flushing out the remaining items in buffer
WriteUptoPosition(itemsMinHeap, int.MaxValue);
Flush(chromIndex);
Console.WriteLine($"Chromosome {currentEnsemblName} completed in {Benchmark.ToHumanReadable(benchmark.GetElapsedTime())}");
_index.Write();
return(_count);
}
public void Write(IEnumerable <ISupplementaryDataItem> saItems)
{
var itemsMinHeap = new MinHeap <ISupplementaryDataItem>(SuppDataUtilities.CompareTo);
var chromIndex = ushort.MaxValue;
var currentEnsemblName = "";
var benchmark = new Benchmark();
foreach (var saItem in saItems)
{
if (chromIndex != saItem.Chromosome.Index)
{
if (chromIndex != ushort.MaxValue)
{
//flushing out the remaining items in buffer
WriteUptoPosition(itemsMinHeap, int.MaxValue);
Console.WriteLine($"Chromosome {currentEnsemblName} completed in {Benchmark.ToHumanReadable(benchmark.GetElapsedTime())}");
benchmark.Reset();
}
chromIndex = saItem.Chromosome.Index;
currentEnsemblName = saItem.Chromosome.EnsemblName;
_refProvider.LoadChromosome(saItem.Chromosome);
}
if (saItem.RefAllele != _refProvider.Sequence.Substring(saItem.Position - 1, saItem.RefAllele.Length))
{
continue;
}
//the items come in sorted order of the pre-trimmed position.
//So when writing out, we have to make sure that we do not write past this position.
//Once a position has been seen in the stream, we can safely write all positions before that.
var writeToPos = saItem.Position;
saItem.Trim();
itemsMinHeap.Add(saItem);
WriteUptoPosition(itemsMinHeap, writeToPos);
}
//flushing out the remaining items in buffer
WriteUptoPosition(itemsMinHeap, int.MaxValue);
Console.WriteLine($"Chromosome {currentEnsemblName} completed in {Benchmark.ToHumanReadable(benchmark.GetElapsedTime())}");
_refMinorIndex.Write(_stream.Position);
}
private bool ValidateReference(string chromosome, int position, string refAllele)
{
if (_sequenceProvider == null)
{
return(true);
}
var refDictionary = _sequenceProvider.RefNameToChromosome;
if (!refDictionary.ContainsKey(chromosome))
{
return(false);
}
var chrom = refDictionary[chromosome];
_sequenceProvider.LoadChromosome(chrom);
var refSequence = _sequenceProvider.Sequence.Substring(position - 1, ReferenceWindow);
return(SupplementaryAnnotationUtilities.ValidateRefAllele(refAllele, refSequence));
}
public IEnumerable <ISimplePosition> Recompose(List <ISimplePosition> recomposablePositions, List <int> functionBlockRanges)
{
var positionSet = PositionSet.CreatePositionSet(recomposablePositions, functionBlockRanges);
var alleleSet = positionSet.AlleleSet;
var alleleIndexBlockToSampleIndex = positionSet.AlleleIndexBlockToSampleIndex;
var numSamples = positionSet.NumSamples;
_sequenceProvider.LoadChromosome(alleleSet.Chromosome);
int regionStart = alleleSet.Starts[0];
string lastRefAllele = alleleSet.VariantArrays.Last()[0];
int regionEnd = alleleSet.Starts.Last() + lastRefAllele.Length + 100; // make it long enough
if (regionEnd > _sequenceProvider.Sequence.Length)
{
regionEnd = _sequenceProvider.Sequence.Length;
}
string totalRefSequence = _sequenceProvider.Sequence.Substring(regionStart - 1, regionEnd - regionStart); // VCF positions are 1-based
var recomposedAlleleSet = new RecomposedAlleleSet(positionSet.ChrName, numSamples);
var decomposedPosVarIndex = new HashSet <(int PosIndex, int VarIndex)>();
foreach (var(alleleIndexBlock, sampleAlleles) in alleleIndexBlockToSampleIndex)
{
var(start, _, refAllele, altAllele) = GetPositionsAndRefAltAlleles(alleleIndexBlock, alleleSet, totalRefSequence, regionStart, decomposedPosVarIndex);
var variantSite = new VariantSite(start, refAllele);
if (!recomposedAlleleSet.RecomposedAlleles.TryGetValue(variantSite, out var variantInfo))
{
variantInfo = GetVariantInfo(positionSet, alleleIndexBlock);
recomposedAlleleSet.RecomposedAlleles[variantSite] = variantInfo;
}
variantInfo.AddAllele(altAllele, sampleAlleles);
}
// Set decomposed tag to positions used for recomposition
foreach (var indexTuple in decomposedPosVarIndex)
{
recomposablePositions[indexTuple.PosIndex].IsDecomposed[indexTuple.VarIndex] = true;
}
return(recomposedAlleleSet.GetRecomposedVcfRecords().Select(x => SimplePosition.GetSimplePosition(x, _sequenceProvider.RefNameToChromosome, true)));
}
public static IPosition ToPosition(ISimplePosition simplePosition, IRefMinorProvider refMinorProvider, ISequenceProvider sequenceProvider, IMitoHeteroplasmyProvider mitoHeteroplasmyProvider, VariantFactory variantFactory, bool enableDq = false)
{
if (simplePosition == null)
{
return(null);
}
sequenceProvider.LoadChromosome(simplePosition.Chromosome);
string[] vcfFields = simplePosition.VcfFields;
string[] altAlleles = vcfFields[VcfCommon.AltIndex].OptimizedSplit(',');
bool isReference = altAlleles.Length == 1 && VcfCommon.ReferenceAltAllele.Contains(altAlleles[0]);
string globalMajorAllele = isReference
? refMinorProvider?.GetGlobalMajorAllele(simplePosition.Chromosome, simplePosition.Start)
: null;
bool isRefMinor = isReference && globalMajorAllele != null;
if (isReference && !isRefMinor)
{
return(GetReferencePosition(simplePosition));
}
var infoData = VcfInfoParser.Parse(vcfFields[VcfCommon.InfoIndex]);
int end = ExtractEnd(infoData, simplePosition.Start, simplePosition.RefAllele.Length);
double?quality = vcfFields[VcfCommon.QualIndex].GetNullableValue <double>(double.TryParse);
string[] filters = vcfFields[VcfCommon.FilterIndex].OptimizedSplit(';');
ISample[] samples = vcfFields.ToSamples(variantFactory.FormatIndices, simplePosition, mitoHeteroplasmyProvider, enableDq);
IVariant[] variants = variantFactory.CreateVariants(simplePosition.Chromosome, simplePosition.Start, end,
simplePosition.RefAllele, altAlleles, infoData, simplePosition.IsDecomposed,
simplePosition.IsRecomposed, simplePosition.LinkedVids, globalMajorAllele);
return(new Position(simplePosition.Chromosome, simplePosition.Start, end, simplePosition.RefAllele,
altAlleles, quality, filters, variants, samples, infoData, vcfFields, simplePosition.IsDecomposed,
simplePosition.IsRecomposed));
}
public static IDictionary <IChromosome, List <int> > GetPositions(Stream vcfStream, GenomicRange genomicRange, ISequenceProvider sequenceProvider)
{
var benchmark = new Benchmark();
Console.Write("Scanning positions required for SA pre-loading....");
var chromPositions = new Dictionary <IChromosome, List <int> >();
var rangeChecker = new GenomicRangeChecker(genomicRange);
var refNameToChrom = sequenceProvider.RefNameToChromosome;
using (var reader = new StreamReader(vcfStream))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (!NeedProcessThisLine(refNameToChrom, line, out var splits, out IChromosome iChrom))
{
continue;
}
int position = int.Parse(splits[VcfCommon.PosIndex]);
if (rangeChecker.OutOfRange(iChrom, position))
{
break;
}
string refAllele = splits[VcfCommon.RefIndex];
string altAllele = splits[VcfCommon.AltIndex];
sequenceProvider.LoadChromosome(iChrom);
UpdateChromToPositions(chromPositions, iChrom, position, refAllele, altAllele, sequenceProvider.Sequence);
}
}
int count = SortPositionsAndGetCount(chromPositions);
Console.WriteLine($"{count} positions found in {Benchmark.ToHumanReadable(benchmark.GetElapsedTime())}");
return(chromPositions);
}
private string GetNextChromDestinations(string line)
{
//extracting current chrom info from first line provided
var currentChromName = line.Split('\t', 2)[VcfCommon.ChromIndex];
Console.Write($"Getting destinations for chromosome:{currentChromName}...");
var currentChrom = ReferenceNameUtilities.GetChromosome(_desSequenceProvider.RefNameToChromosome, currentChromName);
_desSequenceProvider.LoadChromosome(currentChrom);
do
{
var splits = line.Split('\t', VcfCommon.InfoIndex);
var chrom = splits[VcfCommon.ChromIndex];
if (chrom != currentChromName)
{
break;
}
var refAllele = splits[VcfCommon.RefIndex];
var altAlleles = splits[VcfCommon.AltIndex].Split(',');
var position = int.Parse(splits[VcfCommon.PosIndex]);
var rsIds = Utilities.GetRsids(splits[VcfCommon.IdIndex]);
if (rsIds == null)
{
continue;
}
var processedVariants = altAlleles.Select(x => VariantUtils.TrimAndLeftAlign(position, refAllele, x, _desSequenceProvider.Sequence)).ToArray();
foreach (var(start, variantRef, variantAlt) in processedVariants)
{
foreach (var rsId in rsIds)
{
if (!_destinationVariants.TryGetValue((rsId, variantRef.Length, variantAlt), out var variants))
{
variants = new List <int>();
_destinationVariants[(rsId, variantRef.Length, variantAlt)] = variants;
public int Map()
{
// write out the relocated locations of the leftover rsIds whenever possible
//reading in the leftover ids
var leftoverIds = new HashSet <(long, string)>();
Console.Write("Loading leftover ids...");
string line;
while ((line = _leftoverReader.ReadLine()) != null)
{
var splits = line.Split('#', 3);
var id = long.Parse(splits[0]);
var alt = splits[1];
leftoverIds.Add((id, alt));
}
Console.WriteLine($"{leftoverIds.Count} found.");
// stream through the dest file to find locations
var leftoversWithDest = new Dictionary <(long, string), List <GenomicLocation> >();
var currentChromName = "";
while ((line = _destReader.ReadLine()) != null)
{
if (line.OptimizedStartsWith('#'))
{
continue;
}
var splits = line.Split('\t', VcfCommon.InfoIndex);
var chromName = splits[VcfCommon.ChromIndex];
if (chromName != currentChromName)
{
currentChromName = chromName;
Console.WriteLine($"Getting destinations for chromosome:{currentChromName}...");
var currentChrom = ReferenceNameUtilities.GetChromosome(_desSequenceProvider.RefNameToChromosome,
currentChromName);
_desSequenceProvider.LoadChromosome(currentChrom);
}
var refAllele = splits[VcfCommon.RefIndex];
var altAlleles = splits[VcfCommon.AltIndex].Split(',');
var position = int.Parse(splits[VcfCommon.PosIndex]);
var rsIds = Utilities.GetRsids(splits[VcfCommon.IdIndex]);
if (rsIds == null)
{
continue;
}
var processedVariants = altAlleles.Select(x => VariantUtils.TrimAndLeftAlign(position, refAllele, x, _desSequenceProvider.Sequence)).ToArray();
foreach (var(_, _, variantAlt) in processedVariants)
{
foreach (var rsId in rsIds)
{
if (!leftoverIds.Contains((rsId, variantAlt)))
{
continue;
}
var pos = int.Parse(splits[VcfCommon.PosIndex]);
if (!leftoversWithDest.TryGetValue((rsId, variantAlt), out var locations))
{
locations = new List <GenomicLocation>();
leftoversWithDest[(rsId, variantAlt)] = locations;
private string ProcessNextChromSource(string line)
{
//extracting current chrom info from first line provided
var currentChromName = line.Split('\t', 2)[VcfCommon.ChromIndex];
var currentChrom = ReferenceNameUtilities.GetChromosome(_srcSequenceProvider.RefNameToChromosome, currentChromName);
_srcSequenceProvider.LoadChromosome(currentChrom);
var leftoverCount = 0;
do
{
var splits = line.Split('\t', VcfCommon.InfoIndex);
var chrom = splits[VcfCommon.ChromIndex];
if (chrom != currentChromName)
{
break;
}
var refAllele = splits[VcfCommon.RefIndex];
var altAlleles = splits[VcfCommon.AltIndex].Split(',');
var position = int.Parse(splits[VcfCommon.PosIndex]);
var rsIds = Utilities.GetRsids(splits[VcfCommon.IdIndex]);
if (rsIds == null)
{
continue;
}
var processedVariants = altAlleles.Select(x => VariantUtils.TrimAndLeftAlign(position, refAllele, x, _srcSequenceProvider.Sequence)).ToArray();
var foundInDest = false;
foreach (var(_, variantRef, variantAlt) in processedVariants)
{
foreach (var rsId in rsIds)
{
if (!_destinationVariants.TryGetValue((rsId, variantRef.Length, variantAlt), out var targetPositions))
{
continue;
}
targetPositions.ForEach(x => WriteRemappedEntry(chrom, x, variantRef, variantAlt, line));
//flipping the sign to indicate it has been mapped
//_destinationVariants[rsId] = (-variant.position, variant.refAllele, variant.altAlleles);
foundInDest = true;
}
}
if (foundInDest)
{
continue;
}
foreach (var(_, _, variantAlt) in processedVariants)
{
foreach (var rsId in rsIds)
{
_leftoverWriter.WriteLine(string.Join('#', rsId.ToString(), variantAlt, line));
}
}
leftoverCount++;
} while ((line = _srcReader.ReadLine()) != null);
Console.WriteLine($"Leftover count for {currentChromName}: {leftoverCount}");
//Console.WriteLine($"Number of entries discarded due to allele mismatch: {_alleleMismatchCount}");
_leftoverCount += leftoverCount;
return(line);
}