private void PlotLearningCurve(Testing.CurvePoint[] learningCurve, out GraphForm graphFormS, out GraphForm graphFormFn, out GraphForm graphFormFp, double[] simpleAlgorithm, bool excludeLearnedFromTestingSet)
{
// Create 3 new graphs
GraphForm.GraphPlot graphSuccess = new GraphForm.GraphPlot("TrainingSet", "Success");
GraphForm.GraphPlot graphFN = new GraphForm.GraphPlot("TrainingSet", "False Negative");
GraphForm.GraphPlot graphFP = new GraphForm.GraphPlot("TrainingSet", "False Positive");
graphSuccess.setUsedForLearningCurve(true);
graphFN.setUsedForLearningCurve(true);
graphFP.setUsedForLearningCurve(true);
// add point (0,0)
graphSuccess.addPoint(0, 0, "", Color.Black);
graphFN.addPoint(0, 0, "", Color.Black);
graphFP.addPoint(0, 0, "", Color.Black);
foreach (Testing.CurvePoint p in learningCurve)
{
graphSuccess.addPoint(p.getTrainingSetPercentage(), p.getAvgPercentageCorrect(), Math.Round(p.getStdPercentageCorrect(), 2).ToString(), Color.Black);
graphFN.addPoint(p.getTrainingSetPercentage(), p.getAvgPercentageFalseNegative(), Math.Round(p.getStdPercentageFalseNegative(), 2).ToString(), Color.Black);
graphFP.addPoint(p.getTrainingSetPercentage(), p.getAvgPercentageFalsePositive(), Math.Round(p.getStdPercentageFalsePositive(), 2).ToString(), Color.Black);
}
// add point (100,100) or (100,0)
graphSuccess.addPoint(100, 100, "", Color.Black);
graphFN.addPoint(100, 0, "", Color.Black);
graphFP.addPoint(100, 0, "", Color.Black);
//add comparison line for statistical simple algorithms
string partially = (excludeLearnedFromTestingSet) ? "" : "partially_statistical";
graphSuccess.addComparisonLines(simpleAlgorithm[0], "statistical", partially);
graphFN.addComparisonLines(simpleAlgorithm[1], "statistical", partially);
graphFP.addComparisonLines(simpleAlgorithm[2], "statistical", partially);
// Draw
graphFormS = new GraphForm(graphSuccess);
graphFormFn = new GraphForm(graphFN);
graphFormFp = new GraphForm(graphFP);
graphFormS.Show();
graphFormFn.Show();
graphFormFp.Show();
}
//private static string[] GetfilesFromDirectory(string folder)
//{
// List<string> files = Directory.GetFiles(folder).ToList();
// List<String> removeList = new List<string>();
// for (int i = 0; i < files.Count; i++)
// {
// if(files[i].EndsWith("ini"))
// removeList.Add(files[i]);
// }
// foreach (string path in removeList)
// {
// files.Remove(path);
// }
// return files.ToArray();
//}
private static int HandleIdenticalVectorWithDifferentChoise(ref List<string> aFiles, ref List<string> tFiles, ref List<string> fFiles, VectorRepository rep, Testing.HandleIdenticalMethod identical)
{
if (identical == Testing.HandleIdenticalMethod.Ignore)
return -1;
ImageVector tVector;
ImageVector fVector;
string tVectorC;
string fVectorC;
bool same;
int count = 0;
for (int t = 0; t < tFiles.Count; t++)
for (int f = 0; f < fFiles.Count; f++)
{
if (tFiles[t] == "" || fFiles[f] == "")
continue;
tVector = null;
fVector = null;
same = true;
// Load Images as vectors
tVector = rep.getVectorByPath(tFiles[t]);
fVector = rep.getVectorByPath(fFiles[f]);
if (tVector == null || fVector == null)
throw (new Exception("Unexpected Error: a picture was not found in repository"));
// Classify
tVectorC = Classifier.ClassifyVector(tVector);
fVectorC = Classifier.ClassifyVector(fVector);
// Check similarity
for (int c=0; c < ImageVector.NUMBER_OF_PARAMETERS; c++)
if (tVectorC[c] != fVectorC[c])
{
same = false;
break;
}
// Handle identical vectors (Mark as empty for removel)
if (same)
{
count++;
switch (identical)
{
case Testing.HandleIdenticalMethod.Remove:
aFiles.Remove(tFiles[t]);
aFiles.Remove(fFiles[f]);
tFiles[t] = "";
fFiles[f] = "";
break;
}
}
}
// Remove marked
while (tFiles.Remove(""));
while (fFiles.Remove(""));
return count;
}
/// <summary>
/// creates learning curve after learning+deciding about given precentage of pictures according to percent[]
/// </summary>
/// <param name="folderAll">folder of all images</param>
/// <param name="folderTrue">folder of true images</param>
/// <param name="learn">data learning object for learning</param>
/// <param name="decide">decision making object for deciding</param>
/// <param name="percent">percentages to use for learning (percent.length is number of points in learning curve)</param>
/// <returns>true if succeeded creating the curve</returns>
public static CurvePoint[] LearningCurve(string folderAll, string folderTrue, DataLearning learning, DecisionMaking deciding, int[] percent, out double[] simpleAlgo,
Testing.HandleIdenticalMethod identical, bool excludeTrainingSet)
{
_progress = 0;
int itrCount=0;
simpleAlgo = new double[3];
//randomLearningCurve = null;
const int ITERATIONS_PER_POINT = 5;
// Make All, True & False files lists
List<string> allFiles, falseFiles, trueFiles;
try
{
allFiles = LoadImages(folderAll).ToList();
trueFiles = LoadImages(folderTrue).ToList();
falseFiles = SubstractListsByFileName(allFiles, trueFiles);
trueFiles = SubstractListsByFileName(allFiles, falseFiles); // In order for the path to be via 'allFiles' folder
}
catch (Exception exc)
{
MessageBox.Show(exc.Message);
return null;
}
// Handle duplicates
ProgressString = "Checking for identicals with different choise";
HandleIdenticalVectorWithDifferentChoise(ref allFiles, ref trueFiles, ref falseFiles, learning.Repository, identical);
//create curve of percent.length number of points
ProgressString = "Calculating Curves";
CurvePoint[] learningCurve = new CurvePoint[percent.Length]; //for main graph
//randomLearningCurve = new CurvePoint[percent.Length]; //for random algorithm graph
for (int k = 0; k < percent.Length; k++)
{
learningCurve[k] = new CurvePoint(percent[k]);
// randomLearningCurve[k] = new CurvePoint(percent[k]);
for (int itr = 0; itr < ITERATIONS_PER_POINT; itr++)
{
//get wanted amount of randome images
List<string> chosenFiles;
do chosenFiles = getRandomImages(allFiles, percent[k]);
while (SubstractListsByFileName(chosenFiles, trueFiles).Count == 0 || SubstractListsByFileName(chosenFiles, falseFiles).Count == 0); // incase only true or only false was chosen
//get new lists of true and false files (Used for learning in current curve point)
List<string> subsetTrueFiles = SubstractListsByFileName(chosenFiles, falseFiles);
List<string> subsetFalseFiles = SubstractListsByFileName(chosenFiles, trueFiles);
//-------------------------------------
//learn from subsets
learning.LearnForTesting(chosenFiles, subsetTrueFiles);
//-------------------------------------
//decide for all files
double[] results;
deciding.DecideForTesting(allFiles, learning.Repository, out results);
//-------------------------------------
//make true and false lists according to results
List<string> resultsTrue = new List<string>();
List<string> resultsFalse = new List<string>();
for (int i = 0; i < results.Length; i++)
if (results[i]>0) //*******here deside for testing*******/
resultsTrue.Add(allFiles[i]);
else
resultsFalse.Add(allFiles[i]);
//-------------------------------------
//calculate success precentage
CurvePoint.SingleCurvePointCalc currentIteration = new CurvePoint.SingleCurvePointCalc(trueFiles, falseFiles, subsetTrueFiles, subsetFalseFiles, resultsTrue, resultsFalse, excludeTrainingSet);
learningCurve[k].AddSingleCurvePointCalc(currentIteration);
//-------------------------------------
//get simple algorithms calculations
double simpleFalseNegative,
simpleFalsePositive,
simpleSuccess;
Testing.SimpleAlgorithm.CalcStatisticalAlgorithm(allFiles.Count, trueFiles.Count, out simpleSuccess, out simpleFalseNegative, out simpleFalsePositive);
simpleAlgo[0] = simpleSuccess*100;
simpleAlgo[1] = simpleFalseNegative * 100;
simpleAlgo[2] = simpleFalsePositive * 100;
// Update progress indicators
itrCount++;
_progress = (int)((itrCount * 100)/(ITERATIONS_PER_POINT * percent.Length));
}
}
ProgressString = "Done Calculating Curves";
return learningCurve;
}
/* Writing log file for given learning curve object */
public static void WriteLearningCurveToLog(Testing.CurvePoint[] lc, string filename)
{
int itrN;
StreamWriter file = new StreamWriter(filename);
file.WriteLine("<?xml version=\"1.0\"?>");
file.WriteLine("<LearningCurve>");
for (int p = 0; p < lc.Length; p++)
{
file.WriteLine("\t<CurvePoint>");
file.WriteLine("\t\t<TrainingSetPercentage> " + lc[p].getTrainingSetPercentage().ToString() + " </TrainingSetPercentage>");
file.WriteLine("\t\t<AveragePercentageCorrect> " + lc[p].getAvgPercentageCorrect().ToString() + " </AveragePercentageCorrect>");
file.WriteLine("\t\t<AveragePercentageFalseNegative> " + lc[p].getAvgPercentageFalseNegative().ToString() + " </AveragePercentageFalseNegative>");
file.WriteLine("\t\t<AveragePercentageFalsePositive> " + lc[p].getAvgPercentageFalsePositive().ToString() + " </AveragePercentageFalsePositive>");
file.WriteLine("\t\t<StandardDeviationCorrect> " + lc[p].getStdPercentageCorrect().ToString() + " </StandardDeviationCorrect>");
file.WriteLine("\t\t<StandardDeviationFalseNegative> " + lc[p].getStdPercentageFalseNegative().ToString() + " </StandardDeviationFalseNegative>");
file.WriteLine("\t\t<StandardDeviationFalsePositive> " + lc[p].getStdPercentageFalsePositive().ToString() + " </StandardDeviationFalsePositive>");
file.WriteLine("\t\t<Iterations>");
itrN = 0;
foreach (Testing.CurvePoint.SingleCurvePointCalc itr in lc[p].getPointIterations())
{
List<string> at, af, tt, tf, rt, rf;
itr.getWholeFileLists(out at, out af);
itr.getTrainingSetFileLists(out tt, out tf);
itr.getResultsFileLists(out rt, out rf);
file.WriteLine("\t\t\t<Iteration " + (++itrN).ToString() + ">");
file.WriteLine("\t\t\t\t<PercentageCorrect> " + itr.getPercentageCorrect().ToString() + " </AveragePercentageCorrect>");
file.WriteLine("\t\t\t\t<PercentageFalseNegative> " + itr.getPercentageFalseNegative().ToString() + " </AveragePercentageFalseNegative>");
file.WriteLine("\t\t\t\t<PercentageFalsePositive> " + itr.getPercentageFalsePositive().ToString() + " </AveragePercentageFalsePositive>");
file.WriteLine("\t\t\t\t<ActualLists>");
file.WriteLine("\t\t\t\t\t<True>");
foreach (string str in at)
file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>");
file.WriteLine("\t\t\t\t\t</True>");
file.WriteLine("\t\t\t\t\t<False>");
foreach (string str in af)
file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>");
file.WriteLine("\t\t\t\t\t</False>");
file.WriteLine("\t\t\t\t</ActualLists>");
file.WriteLine("\t\t\t\t<TrainitSet>");
file.WriteLine("\t\t\t\t\t<True>");
foreach (string str in tt)
file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>");
file.WriteLine("\t\t\t\t\t</True>");
file.WriteLine("\t\t\t\t\t<False>");
foreach (string str in tf)
file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>");
file.WriteLine("\t\t\t\t\t</False>");
file.WriteLine("\t\t\t\t</TrainitSet>");
file.WriteLine("\t\t\t\t<Results>");
file.WriteLine("\t\t\t\t\t<True>");
foreach (string str in rt)
file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>");
file.WriteLine("\t\t\t\t\t</True>");
file.WriteLine("\t\t\t\t\t<False>");
foreach (string str in rf)
file.WriteLine("\t\t\t\t\t\t<File>" + str + "</File>");
file.WriteLine("\t\t\t\t\t</False>");
file.WriteLine("\t\t\t\t</Results>");
file.WriteLine("\t\t\t</Iteration>");
}
file.WriteLine("\t\t</Iterations>");
file.WriteLine("\t</CurvePoint>");
}
file.WriteLine("</LearningCurve>");
file.Close();
}
