public static List <CoverageRegion> GetTargetCoverageRegion(ITargetBuilderOptions options, IProgressCallback progress, bool removeRegionWithoutSequence = true) { List <CoverageRegion> result; if (options.TargetFile.EndsWith(".xml")) { result = GetTargetCoverageRegionFromXml(options, progress); } else { result = GetTargetCoverageRegionFromBed(options, progress); } var dic = result.ToGroupDictionary(m => m.Seqname); progress.SetMessage("Filling sequence from {0}...", options.GenomeFastaFile); using (var sr = new StreamReader(options.GenomeFastaFile)) { var ff = new FastaFormat(); Sequence seq; while ((seq = ff.ReadSequence(sr)) != null) { progress.SetMessage("Processing chromosome {0} ...", seq.Reference); var seqname = seq.Name.StringAfter("chr"); List <CoverageRegion> lst; if (dic.TryGetValue(seqname, out lst)) { foreach (var l in lst) { l.Sequence = seq.SeqString.Substring((int)(l.Start - 1), (int)l.Length); if (l.Strand == '+') { l.ReverseComplementedSequence = SequenceUtils.GetReverseComplementedSequence(l.Sequence); } } } } } if (removeRegionWithoutSequence) { result.RemoveAll(l => string.IsNullOrEmpty(l.Sequence)); } progress.SetMessage("Filling sequence finished."); var namemap = new MapReader(1, 12).ReadFromFile(options.RefgeneFile); result.ForEach(m => { var gene = m.Name.StringBefore("_utr3"); m.GeneSymbol = namemap.ContainsKey(gene) ? namemap[gene] : string.Empty; }); return(result); }
public void FillSequence(Sequence seq) { //matched exon fill sequence exons.ForEach(m => m.FillSequence(seq, this.Strand)); //fill direct sequence this.DirectExpectSequence = seq.SeqString.Substring((int)this.ExpectStart, (int)(this.ExpectEnd - this.ExpectStart + 1)).ToUpper(); if (this.Strand == '-') { this.DirectExpectSequence = SequenceUtils.GetReverseComplementedSequence(this.DirectExpectSequence); } }
public void FillSequence(Sequence seq, char strand) { StringBuilder sb = new StringBuilder(); foreach (var loc in this) { sb.Append(seq.SeqString.Substring((int)loc.Start, (int)loc.Length)); } this.Sequence = sb.ToString().ToUpper(); if (strand == '-') { this.Sequence = SequenceUtils.GetReverseComplementedSequence(this.Sequence); } }
public static SeedItem GetSeed(CoverageRegion cr, int offset, int seedLength, double minCoverage) { if (cr.Sequence.Length < offset + seedLength) { return(null); } var coverages = cr.Coverages.Skip(offset).Take(seedLength).ToList(); var coverage = coverages.Average(l => l.Coverage); if (coverage < minCoverage) { return(null); } var newseq = cr.Sequence.Substring(offset, seedLength); var start = cr.Start + offset; var end = cr.Start + offset + seedLength - 1; if (cr.Strand == '+') { newseq = SequenceUtils.GetReverseComplementedSequence(newseq); } return(new SeedItem() { Seqname = cr.Seqname, Start = start, End = end, Strand = cr.Strand, Coverage = coverage, Name = cr.Name, Sequence = newseq, Source = cr, SourceOffset = offset, GeneSymbol = cr.GeneSymbol }); }
public SAMAlignedItem NextSAMAlignedItem() { string line; while ((line = _file.ReadLine()) != null) { var parts = line.Split('\t'); var qname = parts[SAMFormatConst.QNAME_INDEX]; var seq = parts[SAMFormatConst.SEQ_INDEX]; var flag = (SAMFlags)int.Parse(parts[SAMFormatConst.FLAG_INDEX]); //unmatched if (flag.HasFlag(SAMFlags.UnmappedQuery)) { continue; } //check map quality var mapq = int.Parse(parts[SAMFormatConst.MAPQ_INDEX]); if (mapq < _options.MinimumReadQuality) { continue; } var sam = new SAMAlignedItem { Qname = qname, }; bool isReversed = flag.HasFlag(SAMFlags.QueryOnReverseStrand); char strand; if (isReversed) { strand = '-'; sam.Sequence = SequenceUtils.GetReverseComplementedSequence(seq); } else { strand = '+'; sam.Sequence = seq; } var loc = new SAMAlignedLocation(sam) { Seqname = parts[SAMFormatConst.RNAME_INDEX], Start = int.Parse(parts[SAMFormatConst.POS_INDEX]), Strand = strand, Cigar = parts[SAMFormatConst.CIGAR_INDEX], MismatchPositions = _format.GetMismatchPositions(parts), NumberOfMismatch = _format.GetNumberOfMismatch(parts), Sequence = seq, Qual = parts[SAMFormatConst.QUAL_INDEX] }; loc.ParseEnd(sam.Sequence); sam.AddLocation(loc); if (_format.HasAlternativeHits) { _format.ParseAlternativeHits(parts, sam); } return(sam); } return(null); }
protected override SAMItemSlim GetAlignedSequence() { var array = new byte[4]; ReadUnCompressedData(array, 0, 4); var blockLen = Helper.GetInt32(array, 0); var alignmentBlock = new byte[blockLen]; ReadUnCompressedData(alignmentBlock, 0, blockLen); if (!Filter.Accept(alignmentBlock)) { return(null); } int value; // 8 - 12 bytes "bin<<16|mapQual<<8|read_name_len" var unsignedValue = Helper.GetUInt32(alignmentBlock, 8); // 8th bytes var queryNameLen = (int)(unsignedValue & 0x000000FF); // 32-(32+readLen) bytes var qname = Encoding.ASCII.GetString(alignmentBlock, 32, queryNameLen - 1); if (IgnoreQuery.Contains(qname)) { return(null); } var result = new SAMItemSlim() { Qname = qname }; // 12 - 16 bytes unsignedValue = Helper.GetUInt32(alignmentBlock, 12); // 14-16 bytes var flagValue = (int)(unsignedValue & 0xFFFF0000) >> 16; var flag = (SAMFlags)flagValue; // 12-14 bytes var cigarLen = (int)(unsignedValue & 0x0000FFFF); // 16-20 bytes var readLen = Helper.GetInt32(alignmentBlock, 16); // 32-(32+readLen) bytes var startIndex = 32 + queryNameLen + cigarLen * 4; var sequence = new StringBuilder(); var index = startIndex; for (; index < (startIndex + (readLen + 1) / 2) - 1; index++) { // Get first 4 bit value value = (alignmentBlock[index] & 0xF0) >> 4; sequence.Append(GetSeqChar(value, result.Qname)); // Get last 4 bit value value = alignmentBlock[index] & 0x0F; sequence.Append(GetSeqChar(value, result.Qname)); } value = (alignmentBlock[index] & 0xF0) >> 4; sequence.Append(GetSeqChar(value, result.Qname)); if (readLen % 2 == 0) { value = alignmentBlock[index] & 0x0F; sequence.Append(GetSeqChar(value, result.Qname)); } startIndex = index + 1; var qualValues = new StringBuilder(); if (alignmentBlock[startIndex] != 0xFF) { for (var i = startIndex; i < (startIndex + readLen); i++) { qualValues.Append((char)(alignmentBlock[i] + 33)); } } else { qualValues.Append(SAMParser.AsteriskAsByte); } if (flag.HasFlag(SAMFlags.QueryOnReverseStrand)) { result.Sequence = SequenceUtils.GetReverseComplementedSequence(sequence.ToString()); result.Qual = qualValues.ToString().Reverse(); } else { result.Sequence = sequence.ToString(); result.Qual = qualValues.ToString(); } return(result); }
public override IEnumerable <string> Process() { PileupCountList pc = new PileupCountList(); var format = options.GetSAMFormat(); var cm = new SmallRNACountMap(options.CountFile); var srItems = SequenceRegionUtils.GetSequenceRegions(options.CoordinateFile, "miRNA", options.BedAsGtf); srItems.ForEach(m => { m.Seqname = m.Seqname.StringAfter("chr"); }); var srmap = srItems.GroupBy(m => m.Seqname).ToDictionary(m => m.Key, m => m.ToList()); StreamWriter swScript = null; try { if (options.ExportIgvScript) { swScript = new StreamWriter(options.OutputFile + ".igv"); swScript.WriteLine("snapshotDirectory {0}", Path.GetDirectoryName(options.OutputFile).Replace('\\', '/')); } using (StreamWriter sw = new StreamWriter(options.OutputFile)) { sw.WriteLine(@"##fileformat=VCFv4.2 ##fileDate={0:yyyyMMdd} ##source={1} ##phasing=partial ##INFO=<ID=NS,Number=1,Type=Integer,Description=""Number of Samples With Data""> ##INFO=<ID=DP,Number=1,Type=Integer,Description=""Total Depth""> ##INFO=<ID=AF,Number=A,Type=Float,Description=""Allele Frequency""> ##INFO=<ID=FP,Number=1,Type=Float,Description=""Fisher Exact Test P-Value""> ##INFO=<ID=MN,Number=.,Type=String,Description=""miRNA name contains this position""> ##FILTER=<ID=FisherET,Description=""Fisher exact test Pvalue less than {2}""> ##FILTER=<ID=AltAlleFreq,Description=""Alternative allele frequency less than {3}""> ##FILTER=<ID=notMiRNA,Description=""Position not located in miRNA locus""> ##FORMAT=<ID=DP,Number=1,Type=Integer,Description=""Read Depth""> ##FORMAT=<ID=AD,Number=1,Type=Integer,Description=""Allelic Depth""> #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT {4}", DateTime.Now, "PileupCountBuilder", options.FisherPValue, options.MinimumAlternativeAlleleFrequency, Path.GetFileNameWithoutExtension(options.InputFile)); using (var sr = SAMFactory.GetReader(options.InputFile, true)) { int count = 0; string line; while ((line = sr.ReadLine()) != null) { count++; if (count % 100 == 0) { if (Progress.IsCancellationPending()) { throw new UserTerminatedException(); } } if (count % 100000 == 0) { Progress.SetMessage("{0} reads processed", count); } var parts = line.Split('\t'); var qname = parts[SAMFormatConst.QNAME_INDEX]; var seq = parts[SAMFormatConst.SEQ_INDEX]; //too short if (seq.Length < options.MinimumReadLength) { continue; } SAMFlags flag = (SAMFlags)int.Parse(parts[SAMFormatConst.FLAG_INDEX]); //unmatched if (flag.HasFlag(SAMFlags.UnmappedQuery)) { continue; } var cigar = parts[SAMFormatConst.CIGAR_INDEX]; //insertion/deletion if (cigar.Any(m => m == 'I' || m == 'D')) { continue; } var sam = new SAMAlignedItem() { Qname = qname, }; bool isReversed = flag.HasFlag(SAMFlags.QueryOnReverseStrand); char strand; if (isReversed) { strand = '-'; sam.Sequence = SequenceUtils.GetReverseComplementedSequence(seq); } else { strand = '+'; sam.Sequence = seq; } var loc = new SAMAlignedLocation(sam) { Seqname = parts[SAMFormatConst.RNAME_INDEX].StringAfter("chr"), Start = int.Parse(parts[SAMFormatConst.POS_INDEX]), Strand = strand, Cigar = parts[SAMFormatConst.CIGAR_INDEX], MismatchPositions = format.GetMismatchPositions(parts), NumberOfMismatch = format.GetNumberOfMismatch(parts), Sequence = seq }; loc.ParseEnd(sam.Sequence); sam.AddLocation(loc); if (format.HasAlternativeHits) { format.ParseAlternativeHits(parts, sam); } var finished = pc.Add(sam, cm.GetCount(sam.Qname)); if (null == finished || 0 == finished.Count) { continue; } foreach (var fin in finished) { //if (fin.Chromosome.Equals("1") && fin.Position == 5160725) //{ // Console.WriteLine(fin); //} var ft = fin.FisherExactTest(); if (ft.PValue <= options.FisherPValue) { var total = fin.Sum(m => m.Value); var minallele = total * options.MinimumAlternativeAlleleFrequency; if (ft.Sample2.Failed >= minallele) { List <GtfItem> srs; List <string> ranges = new List <string>(); if (srmap.TryGetValue(sam.Locations[0].Seqname, out srs)) { foreach (var seqr in srs) { if (seqr.Contains(fin.Position)) { ranges.Add(seqr.GetNameLocation()); } } } var alter = (from r in fin where r.Key != fin.Reference orderby r.Key select r).ToList(); var str = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\tNS={7};DP={8};AF={9};FP={10:0.##E0}{11}\tDP:AD\t{12}:{13},{14}", fin.Chromosome, fin.Position, ".", fin.Reference, (from r in alter select r.Key.ToString()).Merge(","), 0, ranges.Count == 0 ? "notMiRNA" : "PASS", 1, total, (from r in alter select string.Format("{0:0.###}", r.Value * 1.0 / total)).Merge(","), ft.PValue, ranges.Count == 0 ? "" : ";" + ranges.Merge(","), total, ft.Sample2.Succeed, (from r in alter select r.Value.ToString()).Merge(",")); sw.WriteLine(str); //Console.WriteLine(str); if (swScript != null && ranges.Count > 0) { swScript.WriteLine(@"goto {0}:{1} sort position snapshot {0}_{2}_{1}.png", fin.Chromosome, fin.Position, ranges[0].Replace('(', '_').Replace(')', '_').Replace(':', '_')); } } } } finished.Clear(); } } } } finally { if (swScript != null) { swScript.Close(); } } return(new string[] { options.OutputFile }); }
private static string GetSequence(char gstrand, SAMAlignedItem m) { return(gstrand == '+' ? m.Sequence : SequenceUtils.GetReverseComplementedSequence(m.Sequence)); }
public List <ChromosomeCountSlimItem> Build(string fileName) { if (File.Exists(options.CategoryMapFile)) { Progress.SetMessage("Reading name map file " + options.CategoryMapFile + " ..."); nameMap = new MapItemReader(0, 1).ReadFromFile(options.CategoryMapFile).ToDictionary(m => m.Key, m => m.Value.Value); } var result = new List <ChromosomeCountSlimItem>(); var queries = new Dictionary <string, SAMChromosomeItem>(); var chromosomes = new Dictionary <string, ChromosomeCountSlimItem>(); Regex chromosomeRegex = null; Func <string, bool> acceptChromosome; if (string.IsNullOrEmpty(options.ChromosomePattern)) { acceptChromosome = m => true; } else { chromosomeRegex = new Regex(options.ChromosomePattern); acceptChromosome = m => chromosomeRegex.Match(m).Success; } Progress.SetMessage("Parsing alignment file " + fileName + " ..."); using (var sr = SAMFactory.GetReader(fileName, true)) { int count = 0; int waitingcount = 0; string line; while ((line = sr.ReadLine()) != null) { if (count % 1000 == 0) { if (Progress.IsCancellationPending()) { throw new UserTerminatedException(); } } if (count % 100000 == 0 && count > 0) { Progress.SetMessage("{0} candidates from {1} reads", waitingcount, count); } count++; var parts = line.Split('\t'); SAMFlags flag = (SAMFlags)int.Parse(parts[SAMFormatConst.FLAG_INDEX]); //unmatched if (flag.HasFlag(SAMFlags.UnmappedQuery)) { continue; } var seqname = GetName(parts[SAMFormatConst.RNAME_INDEX]); if (!acceptChromosome(seqname)) { continue; } var qname = parts[SAMFormatConst.QNAME_INDEX]; SAMChromosomeItem query; if (!queries.TryGetValue(qname, out query)) { query = new SAMChromosomeItem(); query.Qname = qname; queries[qname] = query; if (options.KeepSequence) { query.Sequence = parts[SAMFormatConst.SEQ_INDEX]; if (flag.HasFlag(SAMFlags.QueryOnReverseStrand)) { query.Sequence = SequenceUtils.GetReverseComplementedSequence(query.Sequence); } } } query.Chromosomes.Add(seqname); ChromosomeCountSlimItem item; if (!chromosomes.TryGetValue(seqname, out item)) { item = new ChromosomeCountSlimItem(); item.Names.Add(seqname); chromosomes[seqname] = item; result.Add(item); } item.Queries.Add(query); waitingcount++; } Progress.SetMessage("Finally, there are {0} candidates from {1} reads", waitingcount, count); } foreach (var query in queries.Values) { query.Chromosomes = query.Chromosomes.Distinct().OrderBy(m => m).ToList(); } foreach (var sam in chromosomes.Values) { sam.Queries = sam.Queries.Distinct().OrderBy(m => m.Qname).ToList(); } if (!string.IsNullOrEmpty(options.PreferPrefix)) { foreach (var query in queries.Values) { if (query.Chromosomes.Any(l => l.StartsWith(options.PreferPrefix))) { var chroms = query.Chromosomes.Where(l => l.StartsWith(options.PreferPrefix)).ToArray(); foreach (var chrom in chroms) { chromosomes[chrom].Queries.Remove(query); query.Chromosomes.Remove(chrom); } } } result.RemoveAll(l => l.Queries.Count == 0); } return(result); }
protected override List <T> DoBuild <T>(string fileName, out List <QueryInfo> totalQueries) { var result = new List <T>(); _format = _options.GetSAMFormat(); totalQueries = new List <QueryInfo>(); using (var sr = SAMFactory.GetReader(fileName, true)) { int count = 0; int waitingcount = 0; string line; while ((line = sr.ReadLine()) != null) { count++; if (count % 1000 == 0) { if (Progress.IsCancellationPending()) { throw new UserTerminatedException(); } } var parts = line.Split('\t'); var qname = parts[SAMFormatConst.QNAME_INDEX]; var qi = new QueryInfo(qname); totalQueries.Add(qi); SAMFlags flag = (SAMFlags)int.Parse(parts[SAMFormatConst.FLAG_INDEX]); if (!_filter.AcceptFlags(flag)) { continue; } var mismatchCount = _format.GetNumberOfMismatch(parts); var seq = parts[SAMFormatConst.SEQ_INDEX]; qi.Mismatch = mismatchCount; qi.Length = seq.Length; //too many mismatchs if (!_filter.AcceptMismatch(mismatchCount)) { continue; } if (!_filter.AcceptQueryName(qname)) { continue; } if (!_filter.AcceptLength(seq.Length)) { continue; } var cigar = parts[SAMFormatConst.CIGAR_INDEX]; if (!_filter.AcceptCigar(cigar)) { continue; } var seqname = parts[SAMFormatConst.RNAME_INDEX].StringAfter("chr"); var start = int.Parse(parts[SAMFormatConst.POS_INDEX]); var end = SAMUtils.ParseEnd(start, cigar); bool isReversed = flag.HasFlag(SAMFlags.QueryOnReverseStrand); char strand; if (isReversed) { strand = '-'; } else { strand = '+'; } var sam = new T(); var loc = new SAMAlignedLocation(sam) { Seqname = seqname, Start = start, End = end, Strand = strand, }; if (!_filter.AcceptLocus(loc)) { continue; } if (isReversed) { seq = SequenceUtils.GetReverseComplementedSequence(seq); } sam.Qname = qname; sam.Sequence = seq; loc.AlignmentScore = _format.GetAlignmentScore(parts); loc.Cigar = cigar; loc.NumberOfMismatch = mismatchCount; loc.MismatchPositions = _format.GetMismatchPositions(parts); if (_format.HasAlternativeHits) { _format.ParseAlternativeHits(parts, sam); } result.Add(sam); waitingcount++; if (waitingcount % 100 == 0) { Progress.SetMessage("{0} feature reads from {1} reads", waitingcount, count); } } } return(result); }
protected override List <T> DoBuild <T>(string fileName, out List <QueryInfo> totalQueries) { var result = new List <T>(); _format = _options.GetSAMFormat(); totalQueries = new List <QueryInfo>(); using (var sr = SAMFactory.GetReader(fileName, true)) { int count = 0; int waitingcount = 0; string line; while ((line = sr.ReadLine()) != null) { if (count % 1000 == 0) { if (Progress.IsCancellationPending()) { throw new UserTerminatedException(); } } if (count % 100000 == 0 && count > 0) { Progress.SetMessage("{0} candidates from {1} reads", waitingcount, count); } count++; var qname = line.StringBefore("\t"); //Console.WriteLine("line = {0}", line); //Console.WriteLine("query = {0}", qname); var qi = new QueryInfo(qname); totalQueries.Add(qi); var parts = line.Split('\t'); SAMFlags flag = (SAMFlags)int.Parse(parts[SAMFormatConst.FLAG_INDEX]); //unmatched if (flag.HasFlag(SAMFlags.UnmappedQuery)) { continue; } //too many mismatchs var mismatchCount = _format.GetNumberOfMismatch(parts); var seq = parts[SAMFormatConst.SEQ_INDEX]; qi.Mismatch = mismatchCount; qi.Length = seq.Length; qi.NoPenaltyMutation = 0; if (_options.T2cAsNoPenaltyMutation) { } if (mismatchCount > _options.MaximumMismatch) { continue; } if (!AcceptQueryName(qname)) { continue; } //too short if (seq.Length < _options.MinimumReadLength) { continue; } //too long if (seq.Length > _options.MaximumReadLength) { continue; } var cigar = parts[SAMFormatConst.CIGAR_INDEX]; ////insertion/deletion //if (cigar.Any(m => m == 'I' || m == 'D')) //{ // continue; //} bool isReversed = flag.HasFlag(SAMFlags.QueryOnReverseStrand); char strand; if (isReversed) { strand = '-'; seq = SequenceUtils.GetReverseComplementedSequence(seq); } else { strand = '+'; } var score = _format.GetAlignmentScore(parts); var sam = new T() { Qname = qname, Sequence = seq }; var seqname = parts[SAMFormatConst.RNAME_INDEX]; var loc = new SAMAlignedLocation(sam) { Seqname = seqname, Start = int.Parse(parts[SAMFormatConst.POS_INDEX]), Strand = strand, Cigar = cigar, NumberOfMismatch = mismatchCount, AlignmentScore = score, MismatchPositions = _format.GetMismatchPositions(parts) }; loc.ParseEnd(sam.Sequence); sam.AddLocation(loc); if (_format.HasAlternativeHits) { _format.ParseAlternativeHits(parts, sam); } result.Add(sam); waitingcount++; } Progress.SetMessage("Finally, there are {0} candidates from {1} reads", waitingcount, count); } return(result); }
public override IEnumerable <string> Process() { var srItems = SequenceRegionUtils.GetSequenceRegions(options.InputFile).Where(m => options.AcceptName(m.Name)).ToList(); srItems = (from sr in srItems.GroupBy(m => m.Name) select sr.First()).ToList(); var keepChrInName = options.KeepChrInName && srItems.Any(m => m.Name.StartsWith("chr")); if (!keepChrInName) { srItems.ForEach(m => m.Seqname = m.Seqname.StringAfter("chr")); } var srMap = srItems.ToGroupDictionary(m => m.Seqname); var ff = new FastaFormat(int.MaxValue); using (StreamWriter sw = new StreamWriter(options.OutputFile)) { using (StreamReader sr = new StreamReader(options.GenomeFastaFile)) { Sequence seq; while ((seq = ff.ReadSequence(sr)) != null) { Progress.SetMessage("processing " + seq.Name + " ..."); var name = seq.Name; if (!keepChrInName) { name = name.StringAfter("chr"); } List <GtfItem> items; if (!srMap.TryGetValue(name, out items)) { if (name.Equals("M")) { name = "MT"; srMap.TryGetValue(name, out items); } else if (name.Equals("chrM")) { name = "chrMT"; srMap.TryGetValue(name, out items); } else if (name.Equals("MT")) { name = "M"; srMap.TryGetValue(name, out items); } else if (name.Equals("chrMT")) { name = "chrM"; srMap.TryGetValue(name, out items); } } if (items != null) { Progress.SetMessage(" there are {0} entries in {1} ...", items.Count, name); foreach (var item in items) { if (item.Start - 1 + item.Length >= seq.SeqString.Length) { throw new Exception(string.Format("{0} exceed chromosome {1} length {2}", item, name, seq.SeqString.Length)); } var newseq = seq.SeqString.Substring((int)item.Start - 1, (int)item.Length); if (item.Strand == '-') { newseq = SequenceUtils.GetReverseComplementedSequence(newseq); } newseq = newseq.ToUpper(); var newname = string.Format("{0} {1} {2}", item.Name, item.GetLocationWithoutStrand(), item.Strand); var entry = new Sequence(newname, newseq); ff.WriteSequence(sw, entry); } } } } } return(new string[] { options.OutputFile }); }