private List<ROCPoint> BuildROC(IList<MatchingResult> sortedResults, int negativeCount, int positiveCount)
{
int fp = 0;
int tp = 0;
var curve = new List<ROCPoint>();
for (int i = 0; i < sortedResults.Count; i++)
{
MatchingResult mr = sortedResults[i];
if (mr.Type == MatchingType.Positive)
tp++;
else
fp++;
ROCPoint rocPoint = new ROCPoint(100.0 * fp / negativeCount, 100 - 100.0 * tp / positiveCount, mr.ComparissonValue);
if (curve.Count > 1)
{
if (curve[curve.Count - 2].x == rocPoint.x)
curve[curve.Count - 1] = rocPoint;
else
if (curve[curve.Count - 2].y == rocPoint.y)
curve[curve.Count - 1] = rocPoint;
else
curve.Add(rocPoint);
}
else
curve.Add(rocPoint);
}
curve.RemoveAt(0);
curve.Add(new ROCPoint(100, curve[curve.Count - 1].y, curve[curve.Count - 1].matchingValue));
curve.Add(new ROCPoint(100, 0, curve[curve.Count - 1].matchingValue));
return curve;
}
private void UpdateChart()
{
if (InvokeRequired)
{
Invoke((Action)UpdateChart);
}
else
{
chart.Series.Clear();
chart.Annotations.Clear();
cachedRocPoints.Clear();
int slices = 100;
IEnumerable <int> rows;
if (cmbSamples.SelectedItem.ToString() == TrainingSamples)
{
rows = Content.ProblemData.TrainingIndices;
}
else if (cmbSamples.SelectedItem.ToString() == TestSamples)
{
rows = Content.ProblemData.TestIndices;
}
else
{
throw new InvalidOperationException();
}
double[] estimatedValues = Content.GetEstimatedValues(rows).ToArray();
double[] targetClassValues = Content.ProblemData.Dataset.GetDoubleValues(Content.ProblemData.TargetVariable, rows).ToArray();
double minThreshold = estimatedValues.Min();
double maxThreshold = estimatedValues.Max();
double thresholdIncrement = (maxThreshold - minThreshold) / slices;
minThreshold -= thresholdIncrement;
maxThreshold += thresholdIncrement;
List <double> classValues = Content.ProblemData.ClassValues.ToList();
foreach (double classValue in classValues)
{
List <ROCPoint> rocPoints = new List <ROCPoint>();
int positives = targetClassValues.Where(c => c.IsAlmost(classValue)).Count();
int negatives = targetClassValues.Length - positives;
for (double lowerThreshold = minThreshold; lowerThreshold < maxThreshold; lowerThreshold += thresholdIncrement)
{
for (double upperThreshold = lowerThreshold + thresholdIncrement; upperThreshold < maxThreshold; upperThreshold += thresholdIncrement)
{
//only adapt lower threshold for binary classification problems and upper class prediction
if (classValues.Count == 2 && classValue == classValues[1])
{
upperThreshold = double.PositiveInfinity;
}
int truePositives = 0;
int falsePositives = 0;
for (int row = 0; row < estimatedValues.Length; row++)
{
if (lowerThreshold < estimatedValues[row] && estimatedValues[row] < upperThreshold)
{
if (targetClassValues[row].IsAlmost(classValue))
{
truePositives++;
}
else
{
falsePositives++;
}
}
}
double truePositiveRate = ((double)truePositives) / positives;
double falsePositiveRate = ((double)falsePositives) / negatives;
ROCPoint rocPoint = new ROCPoint(truePositiveRate, falsePositiveRate, lowerThreshold, upperThreshold);
if (!rocPoints.Any(x => x.TruePositiveRate >= rocPoint.TruePositiveRate && x.FalsePositiveRate <= rocPoint.FalsePositiveRate))
{
rocPoints.RemoveAll(x => x.FalsePositiveRate >= rocPoint.FalsePositiveRate && x.TruePositiveRate <= rocPoint.TruePositiveRate);
rocPoints.Add(rocPoint);
}
}
//only adapt upper threshold for binary classification problems and upper class prediction
if (classValues.Count == 2 && classValue == classValues[0])
{
lowerThreshold = double.PositiveInfinity;
}
}
string className = Content.ProblemData.ClassNames.ElementAt(classValues.IndexOf(classValue));
cachedRocPoints[className] = rocPoints.OrderBy(x => x.FalsePositiveRate).ToList();;
Series series = new Series(className);
series.ChartType = SeriesChartType.Line;
series.MarkerStyle = MarkerStyle.Diamond;
series.MarkerSize = 5;
chart.Series.Add(series);
FillSeriesWithDataPoints(series, cachedRocPoints[className]);
double auc = CalculateAreaUnderCurve(series);
series.LegendToolTip = "AUC: " + auc;
}
}
}
private void UpdateChart() {
if (InvokeRequired) Invoke((Action)UpdateChart);
else {
chart.Series.Clear();
chart.Annotations.Clear();
cachedRocPoints.Clear();
int slices = 100;
IEnumerable<int> rows;
if (cmbSamples.SelectedItem.ToString() == TrainingSamples) {
rows = Content.ProblemData.TrainingIndices;
} else if (cmbSamples.SelectedItem.ToString() == TestSamples) {
rows = Content.ProblemData.TestIndices;
} else throw new InvalidOperationException();
double[] estimatedValues = Content.GetEstimatedValues(rows).ToArray();
double[] targetClassValues = Content.ProblemData.Dataset.GetDoubleValues(Content.ProblemData.TargetVariable, rows).ToArray();
double minThreshold = estimatedValues.Min();
double maxThreshold = estimatedValues.Max();
double thresholdIncrement = (maxThreshold - minThreshold) / slices;
minThreshold -= thresholdIncrement;
maxThreshold += thresholdIncrement;
List<double> classValues = Content.ProblemData.ClassValues.ToList();
foreach (double classValue in classValues) {
List<ROCPoint> rocPoints = new List<ROCPoint>();
int positives = targetClassValues.Where(c => c.IsAlmost(classValue)).Count();
int negatives = targetClassValues.Length - positives;
for (double lowerThreshold = minThreshold; lowerThreshold < maxThreshold; lowerThreshold += thresholdIncrement) {
for (double upperThreshold = lowerThreshold + thresholdIncrement; upperThreshold < maxThreshold; upperThreshold += thresholdIncrement) {
//only adapt lower threshold for binary classification problems and upper class prediction
if (classValues.Count == 2 && classValue == classValues[1]) upperThreshold = double.PositiveInfinity;
int truePositives = 0;
int falsePositives = 0;
for (int row = 0; row < estimatedValues.Length; row++) {
if (lowerThreshold < estimatedValues[row] && estimatedValues[row] < upperThreshold) {
if (targetClassValues[row].IsAlmost(classValue)) truePositives++;
else falsePositives++;
}
}
double truePositiveRate = ((double)truePositives) / positives;
double falsePositiveRate = ((double)falsePositives) / negatives;
ROCPoint rocPoint = new ROCPoint(truePositiveRate, falsePositiveRate, lowerThreshold, upperThreshold);
if (!rocPoints.Any(x => x.TruePositiveRate >= rocPoint.TruePositiveRate && x.FalsePositiveRate <= rocPoint.FalsePositiveRate)) {
rocPoints.RemoveAll(x => x.FalsePositiveRate >= rocPoint.FalsePositiveRate && x.TruePositiveRate <= rocPoint.TruePositiveRate);
rocPoints.Add(rocPoint);
}
}
//only adapt upper threshold for binary classification problems and upper class prediction
if (classValues.Count == 2 && classValue == classValues[0]) lowerThreshold = double.PositiveInfinity;
}
string className = Content.ProblemData.ClassNames.ElementAt(classValues.IndexOf(classValue));
cachedRocPoints[className] = rocPoints.OrderBy(x => x.FalsePositiveRate).ToList(); ;
Series series = new Series(className);
series.ChartType = SeriesChartType.Line;
series.MarkerStyle = MarkerStyle.Diamond;
series.MarkerSize = 5;
chart.Series.Add(series);
FillSeriesWithDataPoints(series, cachedRocPoints[className]);
double auc = CalculateAreaUnderCurve(series);
series.LegendToolTip = "AUC: " + auc;
}
}
}
/// <summary>
/// Deserialize the ROC curve from the specified file name.
/// </summary>
/// <param name="fileName">
/// The file name where the ROC curve is serialized.
/// </param>
/// <param name="xLabel">
/// The label of horizontal axis in the ROC curve.
/// </param>
/// <param name="yLabel">
/// The label of vertical axis in the ROC curve.
/// </param>
/// <param name="matcherName">
/// The name of the fingerprint matcher.
/// </param>
/// <returns>
/// The points composing the ROC curve.
/// </returns>
public static List<ROCPoint> Deserialize(string fileName, out string xLabel, out string yLabel, out string matcherName)
{
var roc = new List<ROCPoint>();
using (var sr = new StreamReader(fileName))
{
matcherName = sr.ReadLine();
var line = sr.ReadLine();
var splitStringArray = new string[1] { ";" };
var members = line.Split(splitStringArray, StringSplitOptions.None);
xLabel = members[0];
yLabel = members[1];
// Read lines from the file until the end of the file is reached.
while ((line = sr.ReadLine()) != null)
{
members = line.Split(splitStringArray, StringSplitOptions.None);
var rocPoint = new ROCPoint(Convert.ToDouble(members[0]), Convert.ToDouble(members[1]), Convert.ToDouble(members[2]));
roc.Add(rocPoint);
}
sr.Close();
}
return roc;
}
public void Main(string[] args)
{
string path = @"E:\Experiment_1_22_16_25\";
List <Trial> trials = LoadData(path + "data.txt");
List <Trial> trialsWithCorrectPassword = trials.Where(t => t.IsPasswordCorrect).ToList();
List <Trial> trialsUsersCorrectPassword = trialsWithCorrectPassword.Where(t => !t.IsFromAttacker || !t.IsAGuess).ToList();
List <Trial> trialsGuessesCorrectPassword = trialsWithCorrectPassword.Where(t => t.IsFromAttacker && t.IsAGuess).ToList();
int numConditions = trialsWithCorrectPassword.First().scoreForEachCondition.Length;
//long numCorrectBenignFromAttackersIPpool = trialsUsersCorrectPassword.LongCount(t => t.IsIpInAttackersPool);
//long numCorrectBenignFromProxy = trialsUsersCorrectPassword.LongCount(t => t.IsClientAProxyIP);
//long numCorrectBenignBoth = trialsUsersCorrectPassword.LongCount(t => t.IsIpInAttackersPool && t.IsClientAProxyIP);
long numCorrectGuessesFromProxy = trialsGuessesCorrectPassword.LongCount(t => t.IsClientAProxyIP);
long numCorrectGuessesFromBenignIpPool = trialsGuessesCorrectPassword.LongCount(t => t.IsIpInBenignPool);
long numCorrectGuessesFromBoth = trialsGuessesCorrectPassword.LongCount(t => t.IsIpInBenignPool && t.IsClientAProxyIP);
for (int conditionNumber = 0; conditionNumber < numConditions; conditionNumber++)
{
using (StreamWriter writer = new StreamWriter(path + "PointsFor_" + conditionNumber.ToString() + ".csv"))
{
writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14}",
"False +",
"False -",
"True +",
"True -",
"FP Rate",
"TP Rate",
"Precision",
"Recall",
"FPwAttackerIP",
"FPwProxy",
"FPwBoth",
"FNwBenignIP",
"FNwProxy",
"FNwBoth",
"Threshold");
List <Trial> originalMalicious = new List <Trial>(trialsGuessesCorrectPassword);
List <Trial> originalBenign = new List <Trial>(trialsUsersCorrectPassword);
originalMalicious.Sort((a, b) => - a.CompareTo(b, conditionNumber));
originalBenign.Sort((a, b) => - a.CompareTo(b, conditionNumber));
Queue <Trial> malicious = new Queue <Trial>(originalMalicious);
Queue <Trial> benign = new Queue <Trial>(originalBenign);
int falsePositivesWithProxy = 0;
int falsePositivesWithAttackerIp = 0;
int falsePositivesWithProxyAndAttackerIp = 0;
long falseNegativeWithProxy = numCorrectGuessesFromProxy;
long falseNegativeBenignIp = numCorrectGuessesFromBenignIpPool;
long falseNegativeBenignProxyIp = numCorrectGuessesFromBoth;
//int falseNegativeFromDefendersIpPool;
double blockThreshold = malicious.Peek().GetScoreForCondition(conditionNumber);
List <ROCPoint> rocPoints = new List <ROCPoint>
{
new ROCPoint(0, 0, originalMalicious.Count, originalBenign.Count,
falsePositivesWithAttackerIp, falsePositivesWithProxy, falsePositivesWithProxyAndAttackerIp,
falseNegativeBenignIp, falseNegativeWithProxy, falseNegativeBenignProxyIp,
blockThreshold)
};
while (malicious.Count > 0)
{
// Remove all malicious requests above this new threshold
while (malicious.Count > 0 && malicious.Peek().GetScoreForCondition(conditionNumber) >= blockThreshold)
{
Trial t = malicious.Dequeue();
if (t.IsClientAProxyIP)
{
falseNegativeWithProxy--;
}
if (t.IsIpInBenignPool)
{
falseNegativeBenignIp--;
}
if (t.IsIpInBenignPool && t.IsClientAProxyIP)
{
falseNegativeBenignProxyIp--;
}
}
// Remove all benign requests above this new threshold
while (benign.Count > 0 && benign.Peek().GetScoreForCondition(conditionNumber) >= blockThreshold)
{
Trial t = benign.Dequeue();
if (t.IsIpInAttackersPool)
{
falsePositivesWithAttackerIp++;
}
if (t.IsClientAProxyIP)
{
falsePositivesWithProxy++;
}
if (t.IsIpInAttackersPool && t.IsClientAProxyIP)
{
falsePositivesWithProxyAndAttackerIp++;
}
}
rocPoints.Add(new ROCPoint(originalBenign.Count - benign.Count, originalMalicious.Count - malicious.Count,
malicious.Count, benign.Count,
falsePositivesWithAttackerIp, falsePositivesWithProxy, falsePositivesWithProxyAndAttackerIp,
falseNegativeBenignIp, falseNegativeWithProxy, falseNegativeBenignProxyIp,
blockThreshold));
// Identify next threshold
if (malicious.Count > 0)
{
blockThreshold = malicious.Peek().GetScoreForCondition(conditionNumber);
}
}
List <ROCPoint> finalROCPoints = new List <ROCPoint>();
Queue <ROCPoint> rocPointQueue = new Queue <ROCPoint>(rocPoints);
while (rocPointQueue.Count > 0)
{
ROCPoint point = rocPointQueue.Dequeue();
if (rocPointQueue.Count == 0 || rocPointQueue.Peek().FalsePositives > point.FalsePositives)
{
finalROCPoints.Add(point);
}
}
foreach (ROCPoint point in finalROCPoints)
{
writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14}",
point.FalsePositives, point.FalseNegatives, point.TruePositives, point.TrueNegatives,
point.FalsePositiveRate, point.TruePositiveRate, point.Precision, point.Recall,
point.FalsePositivesWithAttackerIp, point.FalsePositivesWithProxy, point.FalsePositivesWithProxyAndAttackerIp,
point.FalseNegativesWithBenignIp, point.FalseNegativesWithProxy, point.FalseNegativesWithBenignProxyIp,
point.BlockingThreshold);
}
writer.Flush();
}
}
}