private void PlotLearningCurve(Testing.CurvePoint[] learningCurve, out GraphForm graphFormS, out GraphForm graphFormFn, out GraphForm graphFormFp, double[] simpleAlgorithm, bool excludeLearnedFromTestingSet) { // Create 3 new graphs GraphForm.GraphPlot graphSuccess = new GraphForm.GraphPlot("TrainingSet", "Success"); GraphForm.GraphPlot graphFN = new GraphForm.GraphPlot("TrainingSet", "False Negative"); GraphForm.GraphPlot graphFP = new GraphForm.GraphPlot("TrainingSet", "False Positive"); graphSuccess.setUsedForLearningCurve(true); graphFN.setUsedForLearningCurve(true); graphFP.setUsedForLearningCurve(true); // add point (0,0) graphSuccess.addPoint(0, 0, "", Color.Black); graphFN.addPoint(0, 0, "", Color.Black); graphFP.addPoint(0, 0, "", Color.Black); foreach (Testing.CurvePoint p in learningCurve) { graphSuccess.addPoint(p.getTrainingSetPercentage(), p.getAvgPercentageCorrect(), Math.Round(p.getStdPercentageCorrect(), 2).ToString(), Color.Black); graphFN.addPoint(p.getTrainingSetPercentage(), p.getAvgPercentageFalseNegative(), Math.Round(p.getStdPercentageFalseNegative(), 2).ToString(), Color.Black); graphFP.addPoint(p.getTrainingSetPercentage(), p.getAvgPercentageFalsePositive(), Math.Round(p.getStdPercentageFalsePositive(), 2).ToString(), Color.Black); } // add point (100,100) or (100,0) graphSuccess.addPoint(100, 100, "", Color.Black); graphFN.addPoint(100, 0, "", Color.Black); graphFP.addPoint(100, 0, "", Color.Black); //add comparison line for statistical simple algorithms string partially = (excludeLearnedFromTestingSet) ? "" : "partially_statistical"; graphSuccess.addComparisonLines(simpleAlgorithm[0], "statistical", partially); graphFN.addComparisonLines(simpleAlgorithm[1], "statistical", partially); graphFP.addComparisonLines(simpleAlgorithm[2], "statistical", partially); // Draw graphFormS = new GraphForm(graphSuccess); graphFormFn = new GraphForm(graphFN); graphFormFp = new GraphForm(graphFP); graphFormS.Show(); graphFormFn.Show(); graphFormFp.Show(); }
//private static string[] GetfilesFromDirectory(string folder) //{ // List<string> files = Directory.GetFiles(folder).ToList(); // List<String> removeList = new List<string>(); // for (int i = 0; i < files.Count; i++) // { // if(files[i].EndsWith("ini")) // removeList.Add(files[i]); // } // foreach (string path in removeList) // { // files.Remove(path); // } // return files.ToArray(); //} private static int HandleIdenticalVectorWithDifferentChoise(ref List<string> aFiles, ref List<string> tFiles, ref List<string> fFiles, VectorRepository rep, Testing.HandleIdenticalMethod identical) { if (identical == Testing.HandleIdenticalMethod.Ignore) return -1; ImageVector tVector; ImageVector fVector; string tVectorC; string fVectorC; bool same; int count = 0; for (int t = 0; t < tFiles.Count; t++) for (int f = 0; f < fFiles.Count; f++) { if (tFiles[t] == "" || fFiles[f] == "") continue; tVector = null; fVector = null; same = true; // Load Images as vectors tVector = rep.getVectorByPath(tFiles[t]); fVector = rep.getVectorByPath(fFiles[f]); if (tVector == null || fVector == null) throw (new Exception("Unexpected Error: a picture was not found in repository")); // Classify tVectorC = Classifier.ClassifyVector(tVector); fVectorC = Classifier.ClassifyVector(fVector); // Check similarity for (int c=0; c < ImageVector.NUMBER_OF_PARAMETERS; c++) if (tVectorC[c] != fVectorC[c]) { same = false; break; } // Handle identical vectors (Mark as empty for removel) if (same) { count++; switch (identical) { case Testing.HandleIdenticalMethod.Remove: aFiles.Remove(tFiles[t]); aFiles.Remove(fFiles[f]); tFiles[t] = ""; fFiles[f] = ""; break; } } } // Remove marked while (tFiles.Remove("")); while (fFiles.Remove("")); return count; }
/// <summary> /// creates learning curve after learning+deciding about given precentage of pictures according to percent[] /// </summary> /// <param name="folderAll">folder of all images</param> /// <param name="folderTrue">folder of true images</param> /// <param name="learn">data learning object for learning</param> /// <param name="decide">decision making object for deciding</param> /// <param name="percent">percentages to use for learning (percent.length is number of points in learning curve)</param> /// <returns>true if succeeded creating the curve</returns> public static CurvePoint[] LearningCurve(string folderAll, string folderTrue, DataLearning learning, DecisionMaking deciding, int[] percent, out double[] simpleAlgo, Testing.HandleIdenticalMethod identical, bool excludeTrainingSet) { _progress = 0; int itrCount=0; simpleAlgo = new double[3]; //randomLearningCurve = null; const int ITERATIONS_PER_POINT = 5; // Make All, True & False files lists List<string> allFiles, falseFiles, trueFiles; try { allFiles = LoadImages(folderAll).ToList(); trueFiles = LoadImages(folderTrue).ToList(); falseFiles = SubstractListsByFileName(allFiles, trueFiles); trueFiles = SubstractListsByFileName(allFiles, falseFiles); // In order for the path to be via 'allFiles' folder } catch (Exception exc) { MessageBox.Show(exc.Message); return null; } // Handle duplicates ProgressString = "Checking for identicals with different choise"; HandleIdenticalVectorWithDifferentChoise(ref allFiles, ref trueFiles, ref falseFiles, learning.Repository, identical); //create curve of percent.length number of points ProgressString = "Calculating Curves"; CurvePoint[] learningCurve = new CurvePoint[percent.Length]; //for main graph //randomLearningCurve = new CurvePoint[percent.Length]; //for random algorithm graph for (int k = 0; k < percent.Length; k++) { learningCurve[k] = new CurvePoint(percent[k]); // randomLearningCurve[k] = new CurvePoint(percent[k]); for (int itr = 0; itr < ITERATIONS_PER_POINT; itr++) { //get wanted amount of randome images List<string> chosenFiles; do chosenFiles = getRandomImages(allFiles, percent[k]); while (SubstractListsByFileName(chosenFiles, trueFiles).Count == 0 || SubstractListsByFileName(chosenFiles, falseFiles).Count == 0); // incase only true or only false was chosen //get new lists of true and false files (Used for learning in current curve point) List<string> subsetTrueFiles = SubstractListsByFileName(chosenFiles, falseFiles); List<string> subsetFalseFiles = SubstractListsByFileName(chosenFiles, trueFiles); //------------------------------------- //learn from subsets learning.LearnForTesting(chosenFiles, subsetTrueFiles); //------------------------------------- //decide for all files double[] results; deciding.DecideForTesting(allFiles, learning.Repository, out results); //------------------------------------- //make true and false lists according to results List<string> resultsTrue = new List<string>(); List<string> resultsFalse = new List<string>(); for (int i = 0; i < results.Length; i++) if (results[i]>0) //*******here deside for testing*******/ resultsTrue.Add(allFiles[i]); else resultsFalse.Add(allFiles[i]); //------------------------------------- //calculate success precentage CurvePoint.SingleCurvePointCalc currentIteration = new CurvePoint.SingleCurvePointCalc(trueFiles, falseFiles, subsetTrueFiles, subsetFalseFiles, resultsTrue, resultsFalse, excludeTrainingSet); learningCurve[k].AddSingleCurvePointCalc(currentIteration); //------------------------------------- //get simple algorithms calculations double simpleFalseNegative, simpleFalsePositive, simpleSuccess; Testing.SimpleAlgorithm.CalcStatisticalAlgorithm(allFiles.Count, trueFiles.Count, out simpleSuccess, out simpleFalseNegative, out simpleFalsePositive); simpleAlgo[0] = simpleSuccess*100; simpleAlgo[1] = simpleFalseNegative * 100; simpleAlgo[2] = simpleFalsePositive * 100; // Update progress indicators itrCount++; _progress = (int)((itrCount * 100)/(ITERATIONS_PER_POINT * percent.Length)); } } ProgressString = "Done Calculating Curves"; return learningCurve; }
/* Writing log file for given learning curve object */ public static void WriteLearningCurveToLog(Testing.CurvePoint[] lc, string filename) { int itrN; StreamWriter file = new StreamWriter(filename); file.WriteLine("<?xml version=\"1.0\"?>"); file.WriteLine("<LearningCurve>"); for (int p = 0; p < lc.Length; p++) { file.WriteLine("\t<CurvePoint>"); file.WriteLine("\t\t<TrainingSetPercentage> " + lc[p].getTrainingSetPercentage().ToString() + " </TrainingSetPercentage>"); file.WriteLine("\t\t<AveragePercentageCorrect> " + lc[p].getAvgPercentageCorrect().ToString() + " </AveragePercentageCorrect>"); file.WriteLine("\t\t<AveragePercentageFalseNegative> " + lc[p].getAvgPercentageFalseNegative().ToString() + " </AveragePercentageFalseNegative>"); file.WriteLine("\t\t<AveragePercentageFalsePositive> " + lc[p].getAvgPercentageFalsePositive().ToString() + " </AveragePercentageFalsePositive>"); file.WriteLine("\t\t<StandardDeviationCorrect> " + lc[p].getStdPercentageCorrect().ToString() + " </StandardDeviationCorrect>"); file.WriteLine("\t\t<StandardDeviationFalseNegative> " + lc[p].getStdPercentageFalseNegative().ToString() + " </StandardDeviationFalseNegative>"); file.WriteLine("\t\t<StandardDeviationFalsePositive> " + lc[p].getStdPercentageFalsePositive().ToString() + " </StandardDeviationFalsePositive>"); file.WriteLine("\t\t<Iterations>"); itrN = 0; foreach (Testing.CurvePoint.SingleCurvePointCalc itr in lc[p].getPointIterations()) { List<string> at, af, tt, tf, rt, rf; itr.getWholeFileLists(out at, out af); itr.getTrainingSetFileLists(out tt, out tf); itr.getResultsFileLists(out rt, out rf); file.WriteLine("\t\t\t<Iteration " + (++itrN).ToString() + ">"); file.WriteLine("\t\t\t\t<PercentageCorrect> " + itr.getPercentageCorrect().ToString() + " </AveragePercentageCorrect>"); file.WriteLine("\t\t\t\t<PercentageFalseNegative> " + itr.getPercentageFalseNegative().ToString() + " </AveragePercentageFalseNegative>"); file.WriteLine("\t\t\t\t<PercentageFalsePositive> " + itr.getPercentageFalsePositive().ToString() + " </AveragePercentageFalsePositive>"); file.WriteLine("\t\t\t\t<ActualLists>"); file.WriteLine("\t\t\t\t\t<True>"); foreach (string str in at) file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>"); file.WriteLine("\t\t\t\t\t</True>"); file.WriteLine("\t\t\t\t\t<False>"); foreach (string str in af) file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>"); file.WriteLine("\t\t\t\t\t</False>"); file.WriteLine("\t\t\t\t</ActualLists>"); file.WriteLine("\t\t\t\t<TrainitSet>"); file.WriteLine("\t\t\t\t\t<True>"); foreach (string str in tt) file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>"); file.WriteLine("\t\t\t\t\t</True>"); file.WriteLine("\t\t\t\t\t<False>"); foreach (string str in tf) file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>"); file.WriteLine("\t\t\t\t\t</False>"); file.WriteLine("\t\t\t\t</TrainitSet>"); file.WriteLine("\t\t\t\t<Results>"); file.WriteLine("\t\t\t\t\t<True>"); foreach (string str in rt) file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>"); file.WriteLine("\t\t\t\t\t</True>"); file.WriteLine("\t\t\t\t\t<False>"); foreach (string str in rf) file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>"); file.WriteLine("\t\t\t\t\t</False>"); file.WriteLine("\t\t\t\t</Results>"); file.WriteLine("\t\t\t</Iteration>"); } file.WriteLine("\t\t</Iterations>"); file.WriteLine("\t</CurvePoint>"); } file.WriteLine("</LearningCurve>"); file.Close(); }