static void TestNN(MultiLayerNetwork nw, List <GradeDigest> testDigests, int run)
{
List <Tuple <bool, double> > results = new List <Tuple <bool, double> >();
foreach (var gd in testDigests)
{
double[] output = new double[4];
nw.NetOUT(NNUtils.ToNetworkInput(GradeDigest.UnpackBits(gd.data)), out output);
int ans = inputCodes[NNUtils.Answer(output)];
double certainity = NNUtils.AnswerConfidence(output);
results.Add(new Tuple <bool, double>(ans == gd.grade, certainity));
}
double confidenceThreshold = results.Select(t => t.Item2).OrderBy(x => x).ElementAt((int)Math.Floor(results.Count * 0.95));
confidenceThreshold = 0.0001;
int testSuccess = 0;
int testFailure = 0;
int sureTestSuccess = 0;
int sureTestFailure = 0;
int unsure = 0;
foreach (var res in results)
{
if (res.Item1)
{
testSuccess++;
}
else
{
testFailure++;
}
if (res.Item2 > confidenceThreshold)
{
unsure++;
}
else if (res.Item1)
{
sureTestSuccess++;
}
else
{
sureTestFailure++;
}
}
Func <int, double> perc = x => ((double)x / testDigests.Count * 100);
Console.WriteLine("Test results (r/w%): {0:F2}/{1:F2}", perc(testSuccess), perc(testFailure));
Console.WriteLine("Test results (r/u/w%): {0:F2}/{1:F2}/{2:F2} (confidence threshold = {3})",
perc(sureTestSuccess), perc(unsure), perc(sureTestFailure), confidenceThreshold);
nw.SaveNW(String.Format("e:/Pronko/prj/Grader/ocr-data/grade-recognition_{0}_{1:F2}_{2:F2}.nn",
run, perc(testSuccess), perc(sureTestFailure)));
}
static void Main(string[] args)
{
MultiLayerNetwork nw = new MultiLayerNetwork(GradeDigest.dataSize, new int[] { 100, 4 });
List <GradeDigest> trainDigests = GradeDigestSet.staticInstance.GetDigestList();
List <GradeDigest> testDigests = GradeDigestSet.Read("e:/Pronko/prj/Grader/ocr-data/test-data/grade-digests.db").GetDigestList();
for (int trainingRun = 1; trainingRun <= 100; trainingRun++)
{
Util.Timed(String.Format("training run #{0}", trainingRun), () => {
int c = 0;
foreach (var gd in trainDigests.Shuffle())
{
double[] desiredOutput = new double[4];
desiredOutput[inputCodes.IndexOf(gd.grade)] = 1;
nw.LearnNW(NNUtils.ToNetworkInput(GradeDigest.UnpackBits(gd.data)), desiredOutput, 0.1);
c++;
}
});
TestNN(nw, testDigests, trainingRun);
}
}
static void Main(string[] args)
{
List <GradeDigest> testDigests = GradeFS.LoadDigests(TestOcrData);
var gradePairs = new List <Tuple <GradeDigest, RecognitionResult> >();
Util.Timed("compare with database", () => {
int c = 0;
foreach (var gd in testDigests)
{
if (c % 100 == 0)
{
Console.WriteLine("Processed {0}/{1} digests...", c, testDigests.Count);
}
gradePairs.Add(new Tuple <GradeDigest, RecognitionResult>(gd, GradeOCR.Program.RecognizeGrade(gd)));
c++;
}
});
// output results
Func <byte, int> rights = g => gradePairs.Count(gp =>
gp.Item1.grade == gp.Item2.Grade &&
gp.Item1.grade == g);
Func <byte, int> wrongs = g => gradePairs.Count(gp =>
gp.Item1.grade != gp.Item2.Grade &&
gp.Item1.grade == g);
Func <int, double> percentTotal = c => (double)c / gradePairs.Count * 100;
Func <int, int, double> percent = (c, g) => {
int tot = gradePairs.Count(gp => gp.Item1.grade == g);
if (tot > 0)
{
return((double)c / tot * 100);
}
else
{
return(0);
}
};
Console.WriteLine("RESULTS:\n");
int rightTotal = gradePairs.Count(gp => gp.Item1.grade == gp.Item2.Grade);
int wrongTotal = gradePairs.Count(gp => gp.Item1.grade != gp.Item2.Grade);
Console.WriteLine("Total (r/w): {0}/{1}, {2:F1}/{3:F1} %",
rightTotal, wrongTotal,
percentTotal(rightTotal), percentTotal(wrongTotal));
for (byte g = 2; g <= 5; g++)
{
Console.WriteLine("{0}: {1}/{2}, {3:F1}/{4:F1} %", g,
rights(g), wrongs(g), percent(rights(g), g), percent(wrongs(g), g));
}
Console.WriteLine();
Func <byte, int> sureRights = g => gradePairs.Count(gp =>
gp.Item1.grade == gp.Item2.Grade &&
gp.Item1.grade == g &&
gp.Item2.Confident);
Func <byte, int> sureWrongs = g => gradePairs.Count(gp =>
gp.Item1.grade != gp.Item2.Grade &&
gp.Item1.grade == g &&
gp.Item2.Confident);
Func <byte, int> unsure = g => gradePairs.Count(gp =>
gp.Item1.grade == g &&
!gp.Item2.Confident);
int sureRightTotal = gradePairs.Count(gp =>
gp.Item1.grade == gp.Item2.Grade &&
gp.Item2.Confident);
int sureWrongTotal = gradePairs.Count(gp =>
gp.Item1.grade != gp.Item2.Grade &&
gp.Item2.Confident);
int unsureTotal = gradePairs.Count(gp => !gp.Item2.Confident);
Console.WriteLine("Total (r/w/u): {0}/{1}/{2}, {3:F1}/{4:F1}/{5:F1} %",
sureRightTotal, sureWrongTotal, unsureTotal,
percentTotal(sureRightTotal), percentTotal(sureWrongTotal), percentTotal(unsureTotal));
for (byte g = 2; g <= 5; g++)
{
Console.WriteLine("{0}: {1}/{2}/{3}, {4:F1}/{5:F1}/{6:F1} %", g,
sureRights(g), sureWrongs(g), unsure(g),
percent(sureRights(g), g), percent(sureWrongs(g), g), percent(unsure(g), g));
}
Console.WriteLine();
Console.WriteLine("Recognition failures:");
gradePairs.Where(gp => gp.Item1.grade != gp.Item2.Grade).ToList().ForEach(gp => {
GradeDigest failedDigest = testDigests.Find(gd => gd == gp.Item1);
Console.WriteLine("file: " + failedDigest.fileName + ", confident: " + gp.Item2.Confident);
});
}