private List<ROCPoint> BuildROC(IList<MatchingResult> sortedResults, int negativeCount, int positiveCount)
        {
            int fp = 0;
            int tp = 0;
            var curve = new List<ROCPoint>();
            for (int i = 0; i < sortedResults.Count; i++)
            {
                MatchingResult mr = sortedResults[i];
                if (mr.Type == MatchingType.Positive)
                    tp++;
                else
                    fp++;
                    ROCPoint rocPoint = new ROCPoint(100.0 * fp / negativeCount, 100 - 100.0 * tp / positiveCount, mr.ComparissonValue);
                    if (curve.Count > 1)
                    {
                        if (curve[curve.Count - 2].x == rocPoint.x)
                            curve[curve.Count - 1] = rocPoint;
                        else
                            if (curve[curve.Count - 2].y == rocPoint.y)
                                curve[curve.Count - 1] = rocPoint;
                            else
                                curve.Add(rocPoint);
                    }
                    else
                        curve.Add(rocPoint);
            }

            curve.RemoveAt(0);
            curve.Add(new ROCPoint(100, curve[curve.Count - 1].y, curve[curve.Count - 1].matchingValue));
            curve.Add(new ROCPoint(100, 0, curve[curve.Count - 1].matchingValue));
            return curve;
        }
예제 #2
0
        private void UpdateChart()
        {
            if (InvokeRequired)
            {
                Invoke((Action)UpdateChart);
            }
            else
            {
                chart.Series.Clear();
                chart.Annotations.Clear();
                cachedRocPoints.Clear();

                int slices = 100;
                IEnumerable <int> rows;

                if (cmbSamples.SelectedItem.ToString() == TrainingSamples)
                {
                    rows = Content.ProblemData.TrainingIndices;
                }
                else if (cmbSamples.SelectedItem.ToString() == TestSamples)
                {
                    rows = Content.ProblemData.TestIndices;
                }
                else
                {
                    throw new InvalidOperationException();
                }

                double[] estimatedValues    = Content.GetEstimatedValues(rows).ToArray();
                double[] targetClassValues  = Content.ProblemData.Dataset.GetDoubleValues(Content.ProblemData.TargetVariable, rows).ToArray();
                double   minThreshold       = estimatedValues.Min();
                double   maxThreshold       = estimatedValues.Max();
                double   thresholdIncrement = (maxThreshold - minThreshold) / slices;
                minThreshold -= thresholdIncrement;
                maxThreshold += thresholdIncrement;

                List <double> classValues = Content.ProblemData.ClassValues.ToList();

                foreach (double classValue in classValues)
                {
                    List <ROCPoint> rocPoints = new List <ROCPoint>();
                    int             positives = targetClassValues.Where(c => c.IsAlmost(classValue)).Count();
                    int             negatives = targetClassValues.Length - positives;

                    for (double lowerThreshold = minThreshold; lowerThreshold < maxThreshold; lowerThreshold += thresholdIncrement)
                    {
                        for (double upperThreshold = lowerThreshold + thresholdIncrement; upperThreshold < maxThreshold; upperThreshold += thresholdIncrement)
                        {
                            //only adapt lower threshold for binary classification problems and upper class prediction
                            if (classValues.Count == 2 && classValue == classValues[1])
                            {
                                upperThreshold = double.PositiveInfinity;
                            }

                            int truePositives  = 0;
                            int falsePositives = 0;

                            for (int row = 0; row < estimatedValues.Length; row++)
                            {
                                if (lowerThreshold < estimatedValues[row] && estimatedValues[row] < upperThreshold)
                                {
                                    if (targetClassValues[row].IsAlmost(classValue))
                                    {
                                        truePositives++;
                                    }
                                    else
                                    {
                                        falsePositives++;
                                    }
                                }
                            }

                            double truePositiveRate  = ((double)truePositives) / positives;
                            double falsePositiveRate = ((double)falsePositives) / negatives;

                            ROCPoint rocPoint = new ROCPoint(truePositiveRate, falsePositiveRate, lowerThreshold, upperThreshold);
                            if (!rocPoints.Any(x => x.TruePositiveRate >= rocPoint.TruePositiveRate && x.FalsePositiveRate <= rocPoint.FalsePositiveRate))
                            {
                                rocPoints.RemoveAll(x => x.FalsePositiveRate >= rocPoint.FalsePositiveRate && x.TruePositiveRate <= rocPoint.TruePositiveRate);
                                rocPoints.Add(rocPoint);
                            }
                        }
                        //only adapt upper threshold for binary classification problems and upper class prediction
                        if (classValues.Count == 2 && classValue == classValues[0])
                        {
                            lowerThreshold = double.PositiveInfinity;
                        }
                    }

                    string className = Content.ProblemData.ClassNames.ElementAt(classValues.IndexOf(classValue));
                    cachedRocPoints[className] = rocPoints.OrderBy(x => x.FalsePositiveRate).ToList();;

                    Series series = new Series(className);
                    series.ChartType   = SeriesChartType.Line;
                    series.MarkerStyle = MarkerStyle.Diamond;
                    series.MarkerSize  = 5;
                    chart.Series.Add(series);
                    FillSeriesWithDataPoints(series, cachedRocPoints[className]);

                    double auc = CalculateAreaUnderCurve(series);
                    series.LegendToolTip = "AUC: " + auc;
                }
            }
        }
    private void UpdateChart() {
      if (InvokeRequired) Invoke((Action)UpdateChart);
      else {
        chart.Series.Clear();
        chart.Annotations.Clear();
        cachedRocPoints.Clear();

        int slices = 100;
        IEnumerable<int> rows;

        if (cmbSamples.SelectedItem.ToString() == TrainingSamples) {
          rows = Content.ProblemData.TrainingIndices;
        } else if (cmbSamples.SelectedItem.ToString() == TestSamples) {
          rows = Content.ProblemData.TestIndices;
        } else throw new InvalidOperationException();

        double[] estimatedValues = Content.GetEstimatedValues(rows).ToArray();
        double[] targetClassValues = Content.ProblemData.Dataset.GetDoubleValues(Content.ProblemData.TargetVariable, rows).ToArray();
        double minThreshold = estimatedValues.Min();
        double maxThreshold = estimatedValues.Max();
        double thresholdIncrement = (maxThreshold - minThreshold) / slices;
        minThreshold -= thresholdIncrement;
        maxThreshold += thresholdIncrement;

        List<double> classValues = Content.ProblemData.ClassValues.ToList();

        foreach (double classValue in classValues) {
          List<ROCPoint> rocPoints = new List<ROCPoint>();
          int positives = targetClassValues.Where(c => c.IsAlmost(classValue)).Count();
          int negatives = targetClassValues.Length - positives;

          for (double lowerThreshold = minThreshold; lowerThreshold < maxThreshold; lowerThreshold += thresholdIncrement) {
            for (double upperThreshold = lowerThreshold + thresholdIncrement; upperThreshold < maxThreshold; upperThreshold += thresholdIncrement) {
              //only adapt lower threshold for binary classification problems and upper class prediction              
              if (classValues.Count == 2 && classValue == classValues[1]) upperThreshold = double.PositiveInfinity;

              int truePositives = 0;
              int falsePositives = 0;

              for (int row = 0; row < estimatedValues.Length; row++) {
                if (lowerThreshold < estimatedValues[row] && estimatedValues[row] < upperThreshold) {
                  if (targetClassValues[row].IsAlmost(classValue)) truePositives++;
                  else falsePositives++;
                }
              }

              double truePositiveRate = ((double)truePositives) / positives;
              double falsePositiveRate = ((double)falsePositives) / negatives;

              ROCPoint rocPoint = new ROCPoint(truePositiveRate, falsePositiveRate, lowerThreshold, upperThreshold);
              if (!rocPoints.Any(x => x.TruePositiveRate >= rocPoint.TruePositiveRate && x.FalsePositiveRate <= rocPoint.FalsePositiveRate)) {
                rocPoints.RemoveAll(x => x.FalsePositiveRate >= rocPoint.FalsePositiveRate && x.TruePositiveRate <= rocPoint.TruePositiveRate);
                rocPoints.Add(rocPoint);
              }
            }
            //only adapt upper threshold for binary classification problems and upper class prediction              
            if (classValues.Count == 2 && classValue == classValues[0]) lowerThreshold = double.PositiveInfinity;
          }

          string className = Content.ProblemData.ClassNames.ElementAt(classValues.IndexOf(classValue));
          cachedRocPoints[className] = rocPoints.OrderBy(x => x.FalsePositiveRate).ToList(); ;

          Series series = new Series(className);
          series.ChartType = SeriesChartType.Line;
          series.MarkerStyle = MarkerStyle.Diamond;
          series.MarkerSize = 5;
          chart.Series.Add(series);
          FillSeriesWithDataPoints(series, cachedRocPoints[className]);

          double auc = CalculateAreaUnderCurve(series);
          series.LegendToolTip = "AUC: " + auc;
        }
      }
    }
        /// <summary>
        ///     Deserialize the ROC curve from the specified file name.
        /// </summary>
        /// <param name="fileName">
        ///     The file name where the ROC curve is serialized.
        /// </param>
        /// <param name="xLabel">
        ///     The label of horizontal axis in the ROC curve.
        /// </param>
        /// <param name="yLabel">
        ///     The label of vertical axis in the ROC curve.
        /// </param>
        /// <param name="matcherName">
        ///     The name of the fingerprint matcher.
        /// </param>
        /// <returns>
        ///     The points composing the ROC curve.
        /// </returns>
        public static List<ROCPoint> Deserialize(string fileName, out string xLabel, out string yLabel, out string matcherName)
        {
            var roc = new List<ROCPoint>();
            using (var sr = new StreamReader(fileName))
            {
                matcherName = sr.ReadLine();
                var line = sr.ReadLine();
                var splitStringArray = new string[1] { ";" };
                var members = line.Split(splitStringArray, StringSplitOptions.None);
                xLabel = members[0];
                yLabel = members[1];

                // Read lines from the file until the end of the file is reached.
                while ((line = sr.ReadLine()) != null)
                {
                    members = line.Split(splitStringArray, StringSplitOptions.None);
                    var rocPoint = new ROCPoint(Convert.ToDouble(members[0]), Convert.ToDouble(members[1]), Convert.ToDouble(members[2]));
                    roc.Add(rocPoint);
                }
                sr.Close();
            }
            return roc;
        }
예제 #5
0
        public void Main(string[] args)
        {
            string       path   = @"E:\Experiment_1_22_16_25\";
            List <Trial> trials = LoadData(path + "data.txt");

            List <Trial> trialsWithCorrectPassword    = trials.Where(t => t.IsPasswordCorrect).ToList();
            List <Trial> trialsUsersCorrectPassword   = trialsWithCorrectPassword.Where(t => !t.IsFromAttacker || !t.IsAGuess).ToList();
            List <Trial> trialsGuessesCorrectPassword = trialsWithCorrectPassword.Where(t => t.IsFromAttacker && t.IsAGuess).ToList();
            int          numConditions = trialsWithCorrectPassword.First().scoreForEachCondition.Length;

            //long numCorrectBenignFromAttackersIPpool = trialsUsersCorrectPassword.LongCount(t => t.IsIpInAttackersPool);
            //long numCorrectBenignFromProxy = trialsUsersCorrectPassword.LongCount(t => t.IsClientAProxyIP);
            //long numCorrectBenignBoth = trialsUsersCorrectPassword.LongCount(t => t.IsIpInAttackersPool && t.IsClientAProxyIP);
            long numCorrectGuessesFromProxy        = trialsGuessesCorrectPassword.LongCount(t => t.IsClientAProxyIP);
            long numCorrectGuessesFromBenignIpPool = trialsGuessesCorrectPassword.LongCount(t => t.IsIpInBenignPool);
            long numCorrectGuessesFromBoth         = trialsGuessesCorrectPassword.LongCount(t => t.IsIpInBenignPool && t.IsClientAProxyIP);

            for (int conditionNumber = 0; conditionNumber < numConditions; conditionNumber++)
            {
                using (StreamWriter writer = new StreamWriter(path + "PointsFor_" + conditionNumber.ToString() + ".csv"))
                {
                    writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14}",
                                     "False +",
                                     "False -",
                                     "True +",
                                     "True -",
                                     "FP Rate",
                                     "TP Rate",
                                     "Precision",
                                     "Recall",
                                     "FPwAttackerIP",
                                     "FPwProxy",
                                     "FPwBoth",
                                     "FNwBenignIP",
                                     "FNwProxy",
                                     "FNwBoth",
                                     "Threshold");

                    List <Trial> originalMalicious = new List <Trial>(trialsGuessesCorrectPassword);
                    List <Trial> originalBenign    = new List <Trial>(trialsUsersCorrectPassword);
                    originalMalicious.Sort((a, b) => - a.CompareTo(b, conditionNumber));
                    originalBenign.Sort((a, b) => - a.CompareTo(b, conditionNumber));

                    Queue <Trial> malicious = new Queue <Trial>(originalMalicious);
                    Queue <Trial> benign    = new Queue <Trial>(originalBenign);

                    int  falsePositivesWithProxy              = 0;
                    int  falsePositivesWithAttackerIp         = 0;
                    int  falsePositivesWithProxyAndAttackerIp = 0;
                    long falseNegativeWithProxy     = numCorrectGuessesFromProxy;
                    long falseNegativeBenignIp      = numCorrectGuessesFromBenignIpPool;
                    long falseNegativeBenignProxyIp = numCorrectGuessesFromBoth;
                    //int falseNegativeFromDefendersIpPool;


                    double          blockThreshold = malicious.Peek().GetScoreForCondition(conditionNumber);
                    List <ROCPoint> rocPoints      = new List <ROCPoint>
                    {
                        new ROCPoint(0, 0, originalMalicious.Count, originalBenign.Count,
                                     falsePositivesWithAttackerIp, falsePositivesWithProxy, falsePositivesWithProxyAndAttackerIp,
                                     falseNegativeBenignIp, falseNegativeWithProxy, falseNegativeBenignProxyIp,
                                     blockThreshold)
                    };
                    while (malicious.Count > 0)
                    {
                        // Remove all malicious requests above this new threshold
                        while (malicious.Count > 0 && malicious.Peek().GetScoreForCondition(conditionNumber) >= blockThreshold)
                        {
                            Trial t = malicious.Dequeue();
                            if (t.IsClientAProxyIP)
                            {
                                falseNegativeWithProxy--;
                            }
                            if (t.IsIpInBenignPool)
                            {
                                falseNegativeBenignIp--;
                            }
                            if (t.IsIpInBenignPool && t.IsClientAProxyIP)
                            {
                                falseNegativeBenignProxyIp--;
                            }
                        }

                        // Remove all benign requests above this new threshold
                        while (benign.Count > 0 && benign.Peek().GetScoreForCondition(conditionNumber) >= blockThreshold)
                        {
                            Trial t = benign.Dequeue();
                            if (t.IsIpInAttackersPool)
                            {
                                falsePositivesWithAttackerIp++;
                            }
                            if (t.IsClientAProxyIP)
                            {
                                falsePositivesWithProxy++;
                            }
                            if (t.IsIpInAttackersPool && t.IsClientAProxyIP)
                            {
                                falsePositivesWithProxyAndAttackerIp++;
                            }
                        }

                        rocPoints.Add(new ROCPoint(originalBenign.Count - benign.Count, originalMalicious.Count - malicious.Count,
                                                   malicious.Count, benign.Count,
                                                   falsePositivesWithAttackerIp, falsePositivesWithProxy, falsePositivesWithProxyAndAttackerIp,
                                                   falseNegativeBenignIp, falseNegativeWithProxy, falseNegativeBenignProxyIp,
                                                   blockThreshold));

                        // Identify next threshold
                        if (malicious.Count > 0)
                        {
                            blockThreshold = malicious.Peek().GetScoreForCondition(conditionNumber);
                        }
                    }

                    List <ROCPoint>  finalROCPoints = new List <ROCPoint>();
                    Queue <ROCPoint> rocPointQueue  = new Queue <ROCPoint>(rocPoints);
                    while (rocPointQueue.Count > 0)
                    {
                        ROCPoint point = rocPointQueue.Dequeue();
                        if (rocPointQueue.Count == 0 || rocPointQueue.Peek().FalsePositives > point.FalsePositives)
                        {
                            finalROCPoints.Add(point);
                        }
                    }

                    foreach (ROCPoint point in finalROCPoints)
                    {
                        writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14}",
                                         point.FalsePositives, point.FalseNegatives, point.TruePositives, point.TrueNegatives,
                                         point.FalsePositiveRate, point.TruePositiveRate, point.Precision, point.Recall,
                                         point.FalsePositivesWithAttackerIp, point.FalsePositivesWithProxy, point.FalsePositivesWithProxyAndAttackerIp,
                                         point.FalseNegativesWithBenignIp, point.FalseNegativesWithProxy, point.FalseNegativesWithBenignProxyIp,
                                         point.BlockingThreshold);
                    }
                    writer.Flush();
                }
            }
        }