public static void Main() // string[] args { string runDirectoryPath = @"F:\OneDrive\StopGuessingData\Run_5000000_9_15_14_17"; DirectoryInfo runDir = new DirectoryInfo(runDirectoryPath); foreach (DirectoryInfo testDir in runDir.EnumerateDirectories()) { string path = testDir.FullName; Console.Out.WriteLine("Experiment: {0}", path); if (!File.Exists(path + @"\LegitimateAttemptsWithValidPasswords.txt") || !File.Exists(path + @"\AttackAttemptsWithValidPasswords.txt")) { continue; } Trial[] trialsUsersCorrectPassword = LoadData(path + @"\LegitimateAttemptsWithValidPasswords.txt"); Trial[] trialsGuessesCorrectPassword = LoadData(path + @"\AttackAttemptsWithValidPasswords.txt"); int benignWithCookie = trialsUsersCorrectPassword.Count(r => r.DeviceCookieHadPriorSuccessfulLoginForThisAccount); //Trial[] trialsWithCorrectPassword = trials.Where(t => t.IsPasswordCorrect).ToArray(); //Trial[] trialsUsersCorrectPassword = trials.Where(t => t.IsPasswordCorrect && !(t.IsFromAttacker && t.IsAGuess)).ToArray(); //Trial[] trialsGuessesCorrectPassword = trials.Where(t => t.IsPasswordCorrect && (t.IsFromAttacker && t.IsAGuess)).ToArray(); //int numConditions = 15; //long numCorrectBenignFromAttackersIPpool = trialsUsersCorrectPassword.LongCount(t => t.IsIpInAttackersPool); //long numCorrectBenignFromProxy = trialsUsersCorrectPassword.LongCount(t => t.IsClientAProxyIP); //long numCorrectBenignBoth = trialsUsersCorrectPassword.LongCount(t => t.IsIpInAttackersPool && t.IsClientAProxyIP); long numCorrectGuessesFromProxy = trialsGuessesCorrectPassword.LongCount(t => t.IsClientAProxyIP); long numCorrectGuessesFromBenignIpPool = trialsGuessesCorrectPassword.LongCount(t => t.IsIpInBenignPool); long numCorrectGuessesFromBoth = trialsGuessesCorrectPassword.LongCount(t => t.IsIpInBenignPool); // && t.IsClientAProxyIP List <Task> tasks = new List <Task>(); ICondition[] conditions = new ICondition[] { new StopGuessingCondition("AllOn"), new StopGuessingCondition("NoTypoDetection") { beta_typo = 1 }, new StopGuessingCondition("NoRepeatCorrection") { repeat = 1 }, new StopGuessingCondition("PhiIgnoresFrequency") { phi_frequent = 1 }, new StopGuessingCondition("FixedThreshold") { T = 1 }, new StopGuessingCondition("NoCookies") { cookies_off = true }, new StopGuessingCondition("NoAlpha") { alpha = 1 }, new StopGuessingCondition("Control") { alpha = 1, beta_typo = 1, beta_notypo = 1, phi_frequent = 1, phi_infrequent = 1, T = 1, repeat = 1, gamma = 0 }, new StopGuessingCondition("ControlNoRepeats") { alpha = 1, beta_typo = 1, beta_notypo = 1, phi_frequent = 1, phi_infrequent = 1, T = 1, repeat = 0, gamma = 0 }, new PasswordFrequencyOnlyCondition(), new PasswordFrequencyOnlyLadderBinaryCondition(), new AccountLoginFailuresOnlyCondition() }; foreach (ICondition condition in conditions) { Console.Out.WriteLine("Starting work on Condition: {0}", condition.Name); List <ROCPoint> rocPoints = new List <ROCPoint>(); { Queue <Trial> malicious = new Queue <Trial>( trialsGuessesCorrectPassword.AsParallel() .OrderByDescending((x) => condition.GetScore(x))); Console.Out.WriteLine("Sort 1 completed for Condition {0}", condition.Name); Queue <Trial> benign = new Queue <Trial>( trialsUsersCorrectPassword.AsParallel() .OrderByDescending((x) => condition.GetScore(x))); Console.Out.WriteLine("Sort 2 completed for Condition {0}", condition.Name); int originalMaliciousCount = malicious.Count; int originalBenignCount = benign.Count; int falsePositives = 0; int falsePositivesWithProxy = 0; int falsePositivesWithAttackerIp = 0; int falsePositivesWithProxyAndAttackerIp = 0; long falseNegativeWithProxy = numCorrectGuessesFromProxy; long falseNegativeBenignIp = numCorrectGuessesFromBenignIpPool; long falseNegativeBenignProxyIp = numCorrectGuessesFromBoth; HashSet <int> falsePositiveUsers = new HashSet <int>(); HashSet <uint> falsePositiveIPs = new HashSet <uint>(); //int falseNegativeFromDefendersIpPool; double blockThreshold = malicious.Count == 0 ? 0 : condition.GetScore(malicious.Peek()); rocPoints.Add( new ROCPoint(0, 0, 0, 0, originalMaliciousCount, originalBenignCount, falsePositivesWithAttackerIp, falsePositivesWithProxy, falsePositivesWithProxyAndAttackerIp, falseNegativeBenignIp, falseNegativeWithProxy, falseNegativeBenignProxyIp, blockThreshold)); while (malicious.Count > 0) { // Remove all malicious requests above this new threshold while (malicious.Count > 0 && condition.GetScore(malicious.Peek()) >= blockThreshold) { Trial t = malicious.Dequeue(); if (t.IsClientAProxyIP) { falseNegativeWithProxy--; } if (t.IsIpInBenignPool) { falseNegativeBenignIp--; } if (t.IsIpInBenignPool && t.IsClientAProxyIP) { falseNegativeBenignProxyIp--; } } // Remove all benign requests above this new threshold while (benign.Count > 0 && condition.GetScore(benign.Peek()) >= blockThreshold) { Trial t = benign.Dequeue(); falsePositives++; falsePositiveUsers.Add(t.UserID); falsePositiveIPs.Add(t.ClientIP); if (t.IsIpInAttackersPool) { falsePositivesWithAttackerIp++; } if (t.IsClientAProxyIP) { falsePositivesWithProxy++; } if (t.IsIpInAttackersPool && t.IsClientAProxyIP) { falsePositivesWithProxyAndAttackerIp++; } } rocPoints.Add(new ROCPoint( falsePositives, //originalBenign.Count - benign.Count, falsePositiveUsers.Count(), falsePositiveIPs.Count(), //falseNegatives, // originalMaliciousCount - malicious.Count, malicious.Count, benign.Count, falsePositivesWithAttackerIp, falsePositivesWithProxy, falsePositivesWithProxyAndAttackerIp, falseNegativeBenignIp, falseNegativeWithProxy, falseNegativeBenignProxyIp, blockThreshold)); // Identify next threshold if (malicious.Count > 0) { blockThreshold = condition.GetScore(malicious.Peek()); } } } Console.Out.WriteLine("ROC Points identified for Condition {0}", condition.Name); List <ROCPoint> finalROCPoints = new List <ROCPoint>(); Queue <ROCPoint> rocPointQueue = new Queue <ROCPoint>(rocPoints); while (rocPointQueue.Count > 0) { ROCPoint point = rocPointQueue.Dequeue(); if (rocPointQueue.Count == 0 || rocPointQueue.Peek().FalsePositives > point.FalsePositives) { finalROCPoints.Add(point); } } using ( StreamWriter writer = new StreamWriter(path + @"\PointsFor_" + condition.Name + ".csv") ) { writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16}", "False +", "False + users", "False + IPs", "False -", "True +", "True -", "FP Rate", "TP Rate", "Precision", "Recall", "FPwAttackerIP", "FPwProxy", "FPwBoth", "FNwBenignIP", "FNwProxy", "FNwBoth", "Threshold"); foreach (ROCPoint point in finalROCPoints) { writer.WriteLine( "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16}", point.FalsePositives, point.FalsePositiveUniqueUsers, point.FalsePositiveUniqueIPs, point.FalseNegatives, point.TruePositives, point.TrueNegatives, point.FalsePositiveRate, point.TruePositiveRate, point.Precision, point.Recall, point.FalsePositivesWithAttackerIp, point.FalsePositivesWithProxy, point.FalsePositivesWithProxyAndAttackerIp, point.FalseNegativesWithBenignIp, point.FalseNegativesWithProxy, point.FalseNegativesWithBenignProxyIp, point.BlockingThreshold); } writer.Flush(); } Console.Out.WriteLine("Finished with Condition {0}", condition.Name); } } }
public static void Main() // string[] args { string path = @"E:\Size_7_Strategy_BreadthFirst_Remove_0_Proxies_0_Overlap_0_Run_2_5_15_30\"; Trial[] trials = LoadDataParallel(path + "data.txt"); //Trial[] trialsWithCorrectPassword = trials.Where(t => t.IsPasswordCorrect).ToArray(); Trial[] trialsUsersCorrectPassword = trials.Where(t => t.IsPasswordCorrect && !(t.IsFromAttacker && t.IsAGuess)).ToArray(); Trial[] trialsGuessesCorrectPassword = trials.Where(t => t.IsPasswordCorrect && (t.IsFromAttacker && t.IsAGuess)).ToArray(); int numConditions = 15; //long numCorrectBenignFromAttackersIPpool = trialsUsersCorrectPassword.LongCount(t => t.IsIpInAttackersPool); //long numCorrectBenignFromProxy = trialsUsersCorrectPassword.LongCount(t => t.IsClientAProxyIP); //long numCorrectBenignBoth = trialsUsersCorrectPassword.LongCount(t => t.IsIpInAttackersPool && t.IsClientAProxyIP); long numCorrectGuessesFromProxy = trialsGuessesCorrectPassword.LongCount(t => t.IsClientAProxyIP); long numCorrectGuessesFromBenignIpPool = trialsGuessesCorrectPassword.LongCount(t => t.IsIpInBenignPool); long numCorrectGuessesFromBoth = trialsGuessesCorrectPassword.LongCount(t => t.IsIpInBenignPool && t.IsClientAProxyIP); List <Task> tasks = new List <Task>(); for (int c = 0; c < numConditions; c++) { //List<Trial> originalMalicious = new List<Trial>(trialsGuessesCorrectPassword); //List<Trial> originalBenign = new List<Trial>(trialsUsersCorrectPassword); //originalMalicious.Sort((a, b) => -a.CompareTo(b, conditionNumber)); //originalBenign.Sort((a, b) => -a.CompareTo(b, conditionNumber)); //Queue<Trial> malicious = new Queue<Trial>(originalMalicious); //Queue<Trial> benign = new Queue<Trial>(originalBenign); if (c > 0 && c < 7) { continue; } int conditionNumber = c; tasks.Add(Task.Run(() => { Console.Out.WriteLine("Starting work on Condition: {0}", conditionNumber); List <ROCPoint> rocPoints = new List <ROCPoint>(); { Queue <Trial> malicious = new Queue <Trial>( trialsGuessesCorrectPassword.AsParallel().OrderByDescending((x) => x.GetScoreForCondition(conditionNumber))); Console.Out.WriteLine("Sort 1 completed for Condition {0}", conditionNumber); Queue <Trial> benign = new Queue <Trial>( trialsUsersCorrectPassword.AsParallel().OrderByDescending((x) => x.GetScoreForCondition(conditionNumber))); Console.Out.WriteLine("Sort 2 completed for Condition {0}", conditionNumber); int originalMaliciousCount = malicious.Count; int originalBenignCount = benign.Count; int falsePositives = 0; int falsePositivesWithProxy = 0; int falsePositivesWithAttackerIp = 0; int falsePositivesWithProxyAndAttackerIp = 0; long falseNegativeWithProxy = numCorrectGuessesFromProxy; long falseNegativeBenignIp = numCorrectGuessesFromBenignIpPool; long falseNegativeBenignProxyIp = numCorrectGuessesFromBoth; HashSet <int> falsePositiveUsers = new HashSet <int>(); HashSet <uint> falsePositiveIPs = new HashSet <uint>(); //int falseNegativeFromDefendersIpPool; double blockThreshold = malicious.Peek().GetScoreForCondition(conditionNumber); rocPoints.Add( new ROCPoint(0, 0, 0, 0, originalMaliciousCount, originalBenignCount, falsePositivesWithAttackerIp, falsePositivesWithProxy, falsePositivesWithProxyAndAttackerIp, falseNegativeBenignIp, falseNegativeWithProxy, falseNegativeBenignProxyIp, blockThreshold)); while (malicious.Count > 0) { // Remove all malicious requests above this new threshold while (malicious.Count > 0 && malicious.Peek().GetScoreForCondition(conditionNumber) >= blockThreshold) { Trial t = malicious.Dequeue(); if (t.IsClientAProxyIP) { falseNegativeWithProxy--; } if (t.IsIpInBenignPool) { falseNegativeBenignIp--; } if (t.IsIpInBenignPool && t.IsClientAProxyIP) { falseNegativeBenignProxyIp--; } } // Remove all benign requests above this new threshold while (benign.Count > 0 && benign.Peek().GetScoreForCondition(conditionNumber) >= blockThreshold) { Trial t = benign.Dequeue(); falsePositives++; falsePositiveUsers.Add(t.UserID); falsePositiveIPs.Add(t.ClientIP); if (t.IsIpInAttackersPool) { falsePositivesWithAttackerIp++; } if (t.IsClientAProxyIP) { falsePositivesWithProxy++; } if (t.IsIpInAttackersPool && t.IsClientAProxyIP) { falsePositivesWithProxyAndAttackerIp++; } } rocPoints.Add(new ROCPoint( falsePositives, //originalBenign.Count - benign.Count, falsePositiveUsers.Count(), falsePositiveIPs.Count(), //falseNegatives, // originalMaliciousCount - malicious.Count, malicious.Count, benign.Count, falsePositivesWithAttackerIp, falsePositivesWithProxy, falsePositivesWithProxyAndAttackerIp, falseNegativeBenignIp, falseNegativeWithProxy, falseNegativeBenignProxyIp, blockThreshold)); // Identify next threshold if (malicious.Count > 0) { blockThreshold = malicious.Peek().GetScoreForCondition(conditionNumber); } } } Console.Out.WriteLine("ROC Points identified for Condition {0}", conditionNumber); List <ROCPoint> finalROCPoints = new List <ROCPoint>(); Queue <ROCPoint> rocPointQueue = new Queue <ROCPoint>(rocPoints); while (rocPointQueue.Count > 0) { ROCPoint point = rocPointQueue.Dequeue(); if (rocPointQueue.Count == 0 || rocPointQueue.Peek().FalsePositives > point.FalsePositives) { finalROCPoints.Add(point); } } using ( StreamWriter writer = new StreamWriter(path + "PointsFor_" + conditionNumber.ToString() + ".csv") ) { writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16}", "False +", "False + users", "False + IPs", "False -", "True +", "True -", "FP Rate", "TP Rate", "Precision", "Recall", "FPwAttackerIP", "FPwProxy", "FPwBoth", "FNwBenignIP", "FNwProxy", "FNwBoth", "Threshold"); foreach (ROCPoint point in finalROCPoints) { writer.WriteLine( "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16}", point.FalsePositives, point.FalsePositiveUniqueUsers, point.FalsePositiveUniqueIPs, point.FalseNegatives, point.TruePositives, point.TrueNegatives, point.FalsePositiveRate, point.TruePositiveRate, point.Precision, point.Recall, point.FalsePositivesWithAttackerIp, point.FalsePositivesWithProxy, point.FalsePositivesWithProxyAndAttackerIp, point.FalseNegativesWithBenignIp, point.FalseNegativesWithProxy, point.FalseNegativesWithBenignProxyIp, point.BlockingThreshold); } writer.Flush(); } Console.Out.WriteLine("Finished with Condition {0}", conditionNumber); })); } Task.WaitAll(tasks.ToArray()); }