Exemple #1
0
        /// <summary>
        /// For each CNV calls in the truth set, compute the fraction of bases assigned correct copy number
        /// </summary>
        /// <param name="baseCounter"></param>
        private void CalculateMedianAndMeanAccuracies(BaseCounter baseCounter, Dictionary <string, List <CNInterval> > knownCN)
        {
            baseCounter.MeanAccuracy   = 0;
            baseCounter.MedianAccuracy = 0;
            var eventAccuracies = new List <double>();

            foreach (string chr in knownCN.Keys)
            {
                foreach (var interval in knownCN[chr])
                {
                    if (interval.Cn == interval.ReferenceCopyNumber)
                    {
                        continue;
                    }
                    int basecount = interval.Length - interval.BasesExcluded;
                    if (basecount <= 0)
                    {
                        continue;
                    }
                    double accuracy = interval.BasesCalledCorrectly / (double)basecount;
                    eventAccuracies.Add(accuracy);
                    baseCounter.MeanAccuracy += accuracy;
                    //Console.WriteLine("{0}\t{1:F4}", interval.End - interval.Start, accuracy);
                }
            }
            eventAccuracies.Sort();
            baseCounter.MeanAccuracy  /= Math.Max(1, eventAccuracies.Count);
            baseCounter.MedianAccuracy = double.NaN;
            if (eventAccuracies.Count > 0)
            {
                baseCounter.MedianAccuracy = eventAccuracies[eventAccuracies.Count / 2];
            }
            Console.WriteLine($"Event-level accuracy mean {baseCounter.MeanAccuracy:F4} median {baseCounter.MedianAccuracy:F4}" +
                              $" for variants sizes {baseCounter.MinSize} to {baseCounter.MaxSize}");
        }
Exemple #2
0
 private void WriteResults(string cnvCallsPath, StreamWriter outputWriter, BaseCounter baseCounter, bool includePassingOnly, MetricsCalculator metrics)
 {
     // load and append VCF header information
     _cnvChecker.HandleVcfHeaderInfo(outputWriter, new FileLocation(cnvCallsPath));
     // Report stats:
     outputWriter.WriteLine(includePassingOnly ? "Results for PASSing variants" : "Results for all variants");
     outputWriter.WriteLine("Accuracy\t{0:F4}", metrics.Accuracy);
     // SK: I felt the direction based performance metrices make more sense
     outputWriter.WriteLine("DirectionAccuracy\t{0:F4}", metrics.DirectionAccuracy);
     outputWriter.WriteLine("F-score\t{0:F4}", metrics.F1Score);
     outputWriter.WriteLine("Recall\t{0:F4}", metrics.Recall);
     outputWriter.WriteLine("DirectionRecall\t{0:F4}", metrics.DirectionRecall);
     outputWriter.WriteLine("Precision\t{0:F4}", metrics.Precision);
     outputWriter.WriteLine("DirectionPrecision\t{0:F4}", metrics.DirectionPrecision);
     outputWriter.WriteLine("GainRecall\t{0:F4}", metrics.GainRecall);
     outputWriter.WriteLine("GainDirectionRecall\t{0:F4}", metrics.GainDirectionRecall);
     outputWriter.WriteLine("GainPrecision\t{0:F4}", metrics.GainPrecision);
     outputWriter.WriteLine("GainDirectionPrecision\t{0:F4}", metrics.GainDirectionPrecision);
     outputWriter.WriteLine("LossRecall\t{0:F4}", metrics.LossRecall);
     outputWriter.WriteLine("LossDirectionRecall\t{0:F4}", metrics.LossRecall);
     outputWriter.WriteLine("LossPrecision\t{0:F4}", metrics.LossPrecision);
     outputWriter.WriteLine("LossDirectionPrecision\t{0:F4}", metrics.LossDirectionPrecision);
     outputWriter.WriteLine("MeanEventAccuracy\t{0:F4}", 100 * baseCounter.MeanAccuracy);
     outputWriter.WriteLine("MedianEventAccuracy\t{0:F4}", 100 * baseCounter.MedianAccuracy);
     outputWriter.WriteLine("VariantEventsCalled\t{0}", baseCounter.TotalVariants);
     outputWriter.WriteLine("VariantBasesCalled\t{0}", baseCounter.TotalVariantBases);
     if (baseCounter.RoiBaseCount != null && metrics.RoiBases > 0)
     {
         outputWriter.WriteLine("ROIAccuracy\t{0:F4}", metrics.ROIAccuracy);
         outputWriter.WriteLine("ROIDirectionAccuracy\t{0:F4}", metrics.ROIDirectionAccuracy);
     }
     // to separate passing and all variant results
     outputWriter.WriteLine();
 }
Exemple #3
0
        public MetricsCalculator CalculateMetrics(Dictionary <string, List <CNInterval> > knownCN, Dictionary <string, List <CnvCall> > calls,
                                                  BaseCounter baseCounter, bool optionsSkipDiploid, bool includePassingOnly, Dictionary <string, BitArray> kmerfa = null)
        {
            // string referenceBases = string.Empty;
            calls.Values.SelectMany(x => x).ForEach(call =>
            {
                if (!(call.IsAltVariant && call.Length >= baseCounter.MinSize && call.Length <= baseCounter.MaxSize))
                {
                    return;
                }
                if (includePassingOnly && !call.PassFilter)
                {
                    return;
                }
                baseCounter.TotalVariantBases += call.Length;
                baseCounter.TotalVariants++;
            });

            // skip truth interval that have >= 80% of unmapable bases
            // code is not parallel as this will be done by Regression workflow
            const double fractionUnmappableBases = 0.8;
            var          filteredknownCn         = new Dictionary <string, List <CNInterval> >();

            if (kmerfa != null)
            {
                foreach (var chromosome in knownCN.Keys)
                {
                    filteredknownCn[chromosome] = new List <CNInterval>();
                    foreach (var interval in knownCN[chromosome])
                    {
                        // always include REF intervals even if they are in unmappable regions
                        if (interval.Cn == interval.ReferenceCopyNumber)
                        {
                            filteredknownCn[chromosome].Add(interval);
                            continue;
                        }
                        var flaggedBasesCounter = 0;
                        for (var bp = interval.Start; bp < interval.End; bp++)
                        {
                            if (!kmerfa[chromosome][bp])
                            {
                                flaggedBasesCounter++;
                            }
                        }

                        if (flaggedBasesCounter / (double)interval.Length < fractionUnmappableBases)
                        {
                            filteredknownCn[chromosome].Add(interval);
                        }
                        else
                        {
                            Console.Error.WriteLine($"skipping truth interval {interval} with >= {fractionUnmappableBases} fraction of unmappable positions");
                        }
                    }
                }
            }
            else
            {
                filteredknownCn = knownCN;
            }

            foreach (CNInterval interval in filteredknownCn.Values.SelectMany(x => x))
            {
                if (!(interval.Length >= baseCounter.MinSize && interval.Length <= baseCounter.MaxSize))
                {
                    continue;
                }
                int nonOverlapBases    = interval.Length;
                int nonOverlapRoiBases = 0;
                if (!_cnvChecker.RegionsOfInterest.Empty() &&
                    _cnvChecker.RegionsOfInterest.ContainsKey(interval.Chromosome))
                {
                    foreach (CNInterval roiInterval in _cnvChecker.RegionsOfInterest[interval.Chromosome])
                    {
                        int roiOverlapStart = Math.Max(roiInterval.Start, interval.Start);
                        int roiOverlapEnd   = Math.Min(roiInterval.End, interval.End);
                        if (roiOverlapStart >= roiOverlapEnd)
                        {
                            continue;
                        }
                        int roiOverlapBases = roiOverlapEnd - roiOverlapStart;
                        nonOverlapRoiBases -= roiOverlapBases;
                    }
                }
                int    totalOverlapBases      = 0;
                int    totalRoiOverlapBases   = 0;
                int    excludeIntervalBases   = 0;
                var    totalIntervalRefPloidy = new List <(int ploidy, int length)>();
                string chromosome             = interval.Chromosome;


                if (!calls.ContainsKey(chromosome))
                {
                    chromosome = chromosome.Replace("chr", "");
                }
                if (!calls.ContainsKey(chromosome))
                {
                    chromosome = "chr" + chromosome;
                }


                IEnumerable <CnvCall> callsThisChromosome;
                if (calls.ContainsKey(chromosome))
                {
                    callsThisChromosome = calls[chromosome];
                }
                else
                {
                    Console.Error.WriteLine($"Error: no Canvas calls for chromosome {interval.Chromosome} in truth file");
                    callsThisChromosome = Enumerable.Empty <CnvCall>();
                }
                int knownCn = interval.Cn;
                if (knownCn > MaxCn)
                {
                    knownCn = MaxCn;
                }

                int thisIntervalBasesTruePositive  = 0;
                int thisIntervalBasesTrueNegative  = 0;
                int thisIntervalBasesFalsePositive = 0;
                int thisIntervalBasesFalseNegative = 0;
                int thisIntervalBasesNoCall        = interval.Length;
                int thisIntervalBasesExcluded      = 0;

                foreach (CnvCall call in callsThisChromosome)
                {
                    if (!call.RefPloidy.HasValue)
                    {
                        throw new IlluminaException($"Could not determine reference ploidy for call '{call}'. Please provide ploidy information via command line option.");
                    }
                    int refPloidy = interval.ReferenceCopyNumber ?? call.RefPloidy.Value;
                    int CN        = call.CN;
                    if (call.AltAllele == "." && optionsSkipDiploid)
                    {
                        continue;
                    }

                    if (CN > MaxCn)
                    {
                        CN = MaxCn;
                    }
                    string chr          = call.Chr;
                    int    overlapStart = Math.Max(call.Start, interval.Start);
                    int    overlapEnd   = Math.Min(call.End, interval.End);
                    if (overlapStart >= overlapEnd)
                    {
                        continue;
                    }
                    int overlapBases          = overlapEnd - overlapStart;
                    int thisCallBasesExcluded = 0;
                    // We've got an overlap interval.  Kill off some bases from this interval, if it happens
                    // to overlap with an excluded interval:
                    if (_cnvChecker.ExcludeIntervals.ContainsKey(chr))
                    {
                        foreach (CNInterval excludeInterval in _cnvChecker.ExcludeIntervals[chr])
                        {
                            int excludeOverlapStart = Math.Max(excludeInterval.Start, overlapStart);
                            int excludeOverlapEnd   = Math.Min(excludeInterval.End, overlapEnd);
                            if (excludeOverlapStart >= excludeOverlapEnd)
                            {
                                continue;
                            }
                            excludeIntervalBases  += excludeOverlapEnd - excludeOverlapStart;
                            thisCallBasesExcluded += excludeOverlapEnd - excludeOverlapStart;
                            overlapBases          -= excludeOverlapEnd - excludeOverlapStart;
                            // if majority of the region is in exclude intervals, don't consider any overlap
                            // N.B.: the denominator here looks dubious -- why compare overlap bases to the excluded interval overlap size,
                            // rather than, say, the size of the original truth interval?  Or the length of the overlap of the current CNV?
                            if (overlapBases / Math.Max(excludeOverlapEnd - excludeOverlapStart, 1) < 0.1)
                            {
                                thisCallBasesExcluded += overlapBases;
                                excludeIntervalBases  += overlapBases;
                                overlapBases           = 0;
                                break;
                            }
                        }
                    }

                    totalIntervalRefPloidy.Add((refPloidy, overlapBases));

                    if (call.PassFilter || !includePassingOnly)
                    {
                        totalOverlapBases += overlapBases;
                        baseCounter.BaseCount[knownCn, CN, refPloidy] += overlapBases;

                        if (knownCn == CN)
                        {
                            if (CN == refPloidy)
                            {
                                thisIntervalBasesTrueNegative += overlapBases;
                            }
                            else
                            {
                                thisIntervalBasesTruePositive += overlapBases;
                            }
                        }
                        else
                        {
                            if (knownCn == refPloidy)
                            {
                                thisIntervalBasesFalsePositive += overlapBases;
                            }
                            else
                            {
                                thisIntervalBasesFalseNegative += overlapBases;
                            }
                        }
                        thisIntervalBasesNoCall   -= overlapBases;
                        thisIntervalBasesNoCall   -= thisCallBasesExcluded;
                        thisIntervalBasesExcluded += thisCallBasesExcluded;
                    }

                    interval.BasesCovered += overlapBases;

                    if (knownCn == CN)
                    {
                        interval.BasesCalledCorrectly += overlapBases;
                    }
                    else
                    {
                        interval.BasesCalledIncorrectly += overlapBases;
                    }

                    if (_cnvChecker.RegionsOfInterest.Empty() ||
                        !_cnvChecker.RegionsOfInterest.ContainsKey(chr))
                    {
                        continue;
                    }

                    foreach (CNInterval roiInterval in _cnvChecker.RegionsOfInterest[chr])
                    {
                        int roiOverlapStart = Math.Max(roiInterval.Start, overlapStart);
                        int roiOverlapEnd   = Math.Min(roiInterval.End, overlapEnd);
                        if (roiOverlapStart >= roiOverlapEnd)
                        {
                            continue;
                        }
                        int roiOverlapBases = roiOverlapEnd - roiOverlapStart;
                        if (call.PassFilter || !includePassingOnly)
                        {
                            totalRoiOverlapBases += roiOverlapBases;
                            baseCounter.RoiBaseCount[knownCn, CN, refPloidy] += roiOverlapBases;
                        }
                    }
                }

                if (baseCounter.MinSize == 0 && baseCounter.MaxSize > 100000)
                {
                    Console.WriteLine($"Truth {chromosome}:{interval.Start}-{interval.End} CN={knownCn} base counts TP/TN/FP/FN/NC/EXCL {thisIntervalBasesTruePositive} {thisIntervalBasesTrueNegative} {thisIntervalBasesFalsePositive} {thisIntervalBasesFalseNegative} {thisIntervalBasesNoCall} {thisIntervalBasesExcluded}");
                }

                nonOverlapBases -= (totalOverlapBases + excludeIntervalBases);

                if (!interval.ReferenceCopyNumber.HasValue)
                {
                    if (totalIntervalRefPloidy.Empty())
                    {
                        throw new ArgumentException(
                                  $"Error: Truth variant {interval.Chromosome}:{interval.Start}-{interval.End} with no overlapping " +
                                  $"Canvas calls. Reference ploidy cannot be determined! Please provide reference ploidy via command line options");
                    }

                    interval.ReferenceCopyNumber = Convert.ToInt32(Math.Round(Utilities.WeightedMean(
                                                                                  totalIntervalRefPloidy.Select(x => (double)x.ploidy).ToList(),
                                                                                  totalIntervalRefPloidy.Select(x => (double)Math.Max(x.length, 1)).ToList())));
                }
                if (nonOverlapBases < 0)
                {
                    throw new InvalidDataException($"Truth variant {interval.Chromosome}:{interval.Start}-{interval.End} has negative non-overlap bases");
                }
                baseCounter.NoCalls[knownCn, interval.ReferenceCopyNumber.Value] += nonOverlapBases;
            }

            CalculateMedianAndMeanAccuracies(baseCounter, knownCN);
            var allIntervals = knownCN.SelectMany(kvp => kvp.Value).ToList();

            // find truth interval with highest number of false negatives (hurts recall)
            var variantIntervals = allIntervals.Where(interval => interval.Cn != interval.ReferenceCopyNumber).ToList();

            if (variantIntervals.Any())
            {
                var intervalMaxFalseNegatives = variantIntervals.MaxBy(interval => interval.BasesNotCalled + interval.BasesCalledIncorrectly);
                Console.WriteLine($"Truth interval with most false negatives (hurts recall): {intervalMaxFalseNegatives}");
            }
            // find truth interval with highest number of false positive (hurts precision)
            var refIntervals = allIntervals.Where(interval => interval.Cn == interval.ReferenceCopyNumber).ToList();

            if (refIntervals.Any())
            {
                var intervalMaxFalsePositives = refIntervals.MaxBy(interval => interval.BasesCalledIncorrectly);
                Console.WriteLine($"Truth interval with most false positives (hurts precision): {intervalMaxFalsePositives}");
            }
            return(MetricsCalculator.CalculateMetrics(baseCounter, MaxCn, 2));
        }