/// <summary> /// Perform n-fold cross validation /// </summary> /// <param name="cvFolder">folder that contains the folds</param> /// <param name="silent">whether or not to be silent on the screen</param> /// <param name="learner">The svm learner with the relevant parameters</param> /// <returns>results</returns> public Results PerformCrossValidation(String cvFolder, bool silent, SVMLearn learner) { String[] foldDirectories = Directory.GetDirectories(cvFolder); double[] precisions = new double[foldDirectories.Length]; double[] recalls = new double[foldDirectories.Length]; double[] accuracies = new double[foldDirectories.Length]; for (int counter = 0; counter < foldDirectories.Length; counter++) { String concatenatedFile = Path.Combine(foldDirectories[counter], "combined" + counter + ".data"); using (StreamWriter writer = new StreamWriter(concatenatedFile, false)) { for (int innerCounter = 0; innerCounter < foldDirectories.Length; innerCounter++) { if (innerCounter != counter) { using (StreamReader reader = new StreamReader(Path.Combine(foldDirectories[innerCounter], innerCounter + ".data"))) { while (!reader.EndOfStream) { writer.WriteLine(reader.ReadLine()); } } } } } //concatenated file created. now train and test. delete concatenated file this.TrainFold(foldDirectories, counter, concatenatedFile, silent, learner); File.Delete(concatenatedFile); SVMClassify classifier = this.TestFold(foldDirectories, counter); precisions[counter] = classifier.Precision; recalls[counter] = classifier.Recall; accuracies[counter] = classifier.Accuracy; } double accuracy = accuracies.Average(), precision = precisions.Average(), recall = recalls.Average(); Console.WriteLine("Accuracy: {0}, Precision: {1}, Recall: {2}", accuracy, precision, recall); return(new Results { accuracy = accuracy, precision = precision, recall = recall }); }
static void Main(string[] args) { //ENTER FULL PATH OF EXECUTABLES. SVMLearn learner = new SVMLearn(); learner.mode = Mode.Classification; learner.kernelType = Kernel.Linear ; //learner.ParamC = 2; //learner.ParamD = 2; //learner.LeaveOneOutCrossValidation = true; //learner.ParamG = 5; //learner.TrainingErrorAndMarginTradeoff = 10; learner.Cost = 0.55; //high cost model learner.RemoveInconsistentTrainingExamples = true; Utilities u = new Utilities(); SVMLearn[] learners = new SVMLearn[10]; Dictionary<double, Results> cvResults = new Dictionary<double, Results>(10); String outputFolder = @""; u.ConstructNFolds(@"", 10, outputFolder, r => r.StartsWith("-")); for (int counter = 0; counter < 10; counter++) { learners[counter] = new SVMLearn(); learners[counter].mode = Mode.Classification; learners[counter].kernelType = Kernel.Linear; learners[counter].Cost = 0.35 + counter * 0.20; learners[counter].RemoveInconsistentTrainingExamples = true; cvResults.Add(learners[counter].Cost, u.PerformCrossValidation(outputFolder, false, learners[counter])); } StreamWriter learningRate = new StreamWriter(Path.Combine(outputFolder, "learning.txt")); foreach (KeyValuePair<double, Results> entry in cvResults) { Console.WriteLine("Cost: {0}: Accuracy {1}, Precision {2}, Recall {3}", entry.Key, entry.Value.accuracy, entry.Value.precision, entry.Value.recall); learningRate.WriteLine(entry.Key + ", " + entry.Value.accuracy + ", " + entry.Value.precision + ", " + entry.Value.recall); } Console.ReadLine(); }
static void Main(string[] args) { //ENTER FULL PATH OF EXECUTABLES. SVMLearn learner = new SVMLearn(); learner.mode = Mode.Classification; learner.kernelType = Kernel.Linear ; //learner.ParamC = 2; //learner.ParamD = 2; //learner.LeaveOneOutCrossValidation = true; //learner.ParamG = 5; //learner.TrainingErrorAndMarginTradeoff = 10; learner.Cost = 0.55; //high cost model learner.RemoveInconsistentTrainingExamples = true; //String keyword = "kv"; //String logFileName = keyword + learner.kernelType.ToString() + learner.ParamC + learner.ParamD + ".log.txt"; //learner.ExecuteLearner("svm_learn.exe", // @"E:\kv\My Dropbox\code\SocialGraphLinking\" + keyword + @"\entitylinking-" + // keyword + ".data", // @"E:\kv\My Dropbox\code\SocialGraphLinking\" + keyword + "\\" + keyword + ".model", // @"E:\kv\My Dropbox\code\SocialGraphLinking\" + keyword + "\\" + logFileName, false); //Utilities utility = new Utilities(); //utility.ConstructTrainingAndTestSets // (@"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\entitylinking-kv-akshaya-varish.data", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\kv-akshaya-varish-training.arff", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\kv-akshaya-varish-test.data.arff", 0.7, true, // line => line.StartsWith("@"), line => line.Split(',')[5].Trim().Equals("0")); //learner.ExecuteLearner("svm_learn.exe", @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\kv-akshaya-varish-reduced-training.data", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\kv-akshaya-varish-reduced.model", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\kv-akshaya-varish-reduced-training.log.txt", false); //SVMClassify classifier = new SVMClassify(); //classifier.ExecuteClassifier("svm_classify.exe", @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\entitylinking-kv-akshaya-varish.data", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\combined-linear-highcost.model", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\whatever.output", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\whatever-log.txt", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\whatever-incorrect.txt", // false); //classifier.FindIncorrectlyClassifiedInstances(@"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\kv-akshaya-varish-reduced-test.data", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\kv-akshaya-varish-reduced.output", // @"E:\kv\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\kv-akshaya-varish-reduced-incorrect.txt"); //IncorrectFileParser.ParseIncorrectFile(@"D:\kv\official\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\combined-linear-incorrect.txt"); //Utilities.ConstructTrainingAndTestSets(@"D:\kv\official\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\entitylinking-kv-akshaya-varish.data", // @"D:\kv\official\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\training1.data", // @"D:\kv\official\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\test1.data", 0.7, false); //int pos, neg; //Utilities.GetNumberOfTrainingExamples(@"D:\kv\official\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\training.data", // out pos, out neg); //Console.WriteLine("positive: {0}, negative:{1}", pos, neg); Utilities u = new Utilities(); u.EntryPoint(@"D:\kv\official\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\combined.data", 10, @"D:\kv\official\My Dropbox\code\SocialGraphLinking\kv-akshaya-varish\10foldcv", line => line.StartsWith("-")); //Random random = new Random(); //for (int counter = 0; counter < 20; counter++) //{ // Console.WriteLine(random.Next(5)); // Console.ReadKey(); //} //Console.ReadLine(); }
/// <summary> /// Private method that trains a model on the current fold /// </summary> /// <param name="foldDirectories">Directory in which all the folds are placed</param> /// <param name="fold">the current fold under computation</param> /// <param name="concatenatedFile">training file that was created for this fold</param> /// <param name="silent">whether or not to be silent on the screen</param> /// <param name="learner">The svm learner with the relevant parameters</param> private void TrainFold(String[] foldDirectories, int fold, String concatenatedFile, bool silent, SVMLearn learner) { learner.ExecuteLearner("svm_learn.exe", concatenatedFile, Path.Combine(foldDirectories[fold], fold + ".model"), null, Path.Combine(foldDirectories[fold], fold + ".log.txt"), silent); }
/// <summary> /// Perform n-fold cross validation /// </summary> /// <param name="cvFolder">folder that contains the folds</param> /// <param name="silent">whether or not to be silent on the screen</param> /// <param name="learner">The svm learner with the relevant parameters</param> /// <returns>results</returns> public Results PerformCrossValidation(String cvFolder, bool silent, SVMLearn learner) { String[] foldDirectories = Directory.GetDirectories(cvFolder); double[] precisions = new double[foldDirectories.Length]; double[] recalls = new double[foldDirectories.Length]; double[] accuracies = new double[foldDirectories.Length]; for (int counter = 0; counter < foldDirectories.Length; counter++) { String concatenatedFile = Path.Combine(foldDirectories[counter], "combined" + counter + ".data"); using(StreamWriter writer = new StreamWriter(concatenatedFile, false)) { for (int innerCounter = 0; innerCounter < foldDirectories.Length; innerCounter++) { if (innerCounter != counter) { using (StreamReader reader = new StreamReader(Path.Combine(foldDirectories[innerCounter], innerCounter + ".data"))) { while (!reader.EndOfStream) { writer.WriteLine(reader.ReadLine()); } } } } } //concatenated file created. now train and test. delete concatenated file this.TrainFold(foldDirectories, counter, concatenatedFile, silent, learner); File.Delete(concatenatedFile); SVMClassify classifier = this.TestFold(foldDirectories, counter); precisions[counter] = classifier.Precision; recalls[counter] = classifier.Recall; accuracies[counter] = classifier.Accuracy; } double accuracy = accuracies.Average(), precision = precisions.Average(), recall = recalls.Average(); Console.WriteLine("Accuracy: {0}, Precision: {1}, Recall: {2}", accuracy, precision, recall); return new Results { accuracy = accuracy, precision = precision, recall = recall }; }
/// <summary> /// Private method that trains a model on the current fold /// </summary> /// <param name="foldDirectories">Directory in which all the folds are placed</param> /// <param name="fold">the current fold under computation</param> /// <param name="concatenatedFile">training file that was created for this fold</param> /// <param name="silent">whether or not to be silent on the screen</param> /// <param name="learner">The svm learner with the relevant parameters</param> private void TrainFold(String[] foldDirectories, int fold, String concatenatedFile, bool silent, SVMLearn learner) { learner.ExecuteLearner("svm_learn.exe", concatenatedFile, Path.Combine(foldDirectories[fold], fold + ".model"), Path.Combine(foldDirectories[fold], fold + ".log.txt"), silent); }