static void TestNN(MultiLayerNetwork nw, List <GradeDigest> testDigests, int run) { List <Tuple <bool, double> > results = new List <Tuple <bool, double> >(); foreach (var gd in testDigests) { double[] output = new double[4]; nw.NetOUT(NNUtils.ToNetworkInput(GradeDigest.UnpackBits(gd.data)), out output); int ans = inputCodes[NNUtils.Answer(output)]; double certainity = NNUtils.AnswerConfidence(output); results.Add(new Tuple <bool, double>(ans == gd.grade, certainity)); } double confidenceThreshold = results.Select(t => t.Item2).OrderBy(x => x).ElementAt((int)Math.Floor(results.Count * 0.95)); confidenceThreshold = 0.0001; int testSuccess = 0; int testFailure = 0; int sureTestSuccess = 0; int sureTestFailure = 0; int unsure = 0; foreach (var res in results) { if (res.Item1) { testSuccess++; } else { testFailure++; } if (res.Item2 > confidenceThreshold) { unsure++; } else if (res.Item1) { sureTestSuccess++; } else { sureTestFailure++; } } Func <int, double> perc = x => ((double)x / testDigests.Count * 100); Console.WriteLine("Test results (r/w%): {0:F2}/{1:F2}", perc(testSuccess), perc(testFailure)); Console.WriteLine("Test results (r/u/w%): {0:F2}/{1:F2}/{2:F2} (confidence threshold = {3})", perc(sureTestSuccess), perc(unsure), perc(sureTestFailure), confidenceThreshold); nw.SaveNW(String.Format("e:/Pronko/prj/Grader/ocr-data/grade-recognition_{0}_{1:F2}_{2:F2}.nn", run, perc(testSuccess), perc(sureTestFailure))); }
static void Main(string[] args) { MultiLayerNetwork nw = new MultiLayerNetwork(GradeDigest.dataSize, new int[] { 100, 4 }); List <GradeDigest> trainDigests = GradeDigestSet.staticInstance.GetDigestList(); List <GradeDigest> testDigests = GradeDigestSet.Read("e:/Pronko/prj/Grader/ocr-data/test-data/grade-digests.db").GetDigestList(); for (int trainingRun = 1; trainingRun <= 100; trainingRun++) { Util.Timed(String.Format("training run #{0}", trainingRun), () => { int c = 0; foreach (var gd in trainDigests.Shuffle()) { double[] desiredOutput = new double[4]; desiredOutput[inputCodes.IndexOf(gd.grade)] = 1; nw.LearnNW(NNUtils.ToNetworkInput(GradeDigest.UnpackBits(gd.data)), desiredOutput, 0.1); c++; } }); TestNN(nw, testDigests, trainingRun); } }
static void Main(string[] args) { List <GradeDigest> testDigests = GradeFS.LoadDigests(TestOcrData); var gradePairs = new List <Tuple <GradeDigest, RecognitionResult> >(); Util.Timed("compare with database", () => { int c = 0; foreach (var gd in testDigests) { if (c % 100 == 0) { Console.WriteLine("Processed {0}/{1} digests...", c, testDigests.Count); } gradePairs.Add(new Tuple <GradeDigest, RecognitionResult>(gd, GradeOCR.Program.RecognizeGrade(gd))); c++; } }); // output results Func <byte, int> rights = g => gradePairs.Count(gp => gp.Item1.grade == gp.Item2.Grade && gp.Item1.grade == g); Func <byte, int> wrongs = g => gradePairs.Count(gp => gp.Item1.grade != gp.Item2.Grade && gp.Item1.grade == g); Func <int, double> percentTotal = c => (double)c / gradePairs.Count * 100; Func <int, int, double> percent = (c, g) => { int tot = gradePairs.Count(gp => gp.Item1.grade == g); if (tot > 0) { return((double)c / tot * 100); } else { return(0); } }; Console.WriteLine("RESULTS:\n"); int rightTotal = gradePairs.Count(gp => gp.Item1.grade == gp.Item2.Grade); int wrongTotal = gradePairs.Count(gp => gp.Item1.grade != gp.Item2.Grade); Console.WriteLine("Total (r/w): {0}/{1}, {2:F1}/{3:F1} %", rightTotal, wrongTotal, percentTotal(rightTotal), percentTotal(wrongTotal)); for (byte g = 2; g <= 5; g++) { Console.WriteLine("{0}: {1}/{2}, {3:F1}/{4:F1} %", g, rights(g), wrongs(g), percent(rights(g), g), percent(wrongs(g), g)); } Console.WriteLine(); Func <byte, int> sureRights = g => gradePairs.Count(gp => gp.Item1.grade == gp.Item2.Grade && gp.Item1.grade == g && gp.Item2.Confident); Func <byte, int> sureWrongs = g => gradePairs.Count(gp => gp.Item1.grade != gp.Item2.Grade && gp.Item1.grade == g && gp.Item2.Confident); Func <byte, int> unsure = g => gradePairs.Count(gp => gp.Item1.grade == g && !gp.Item2.Confident); int sureRightTotal = gradePairs.Count(gp => gp.Item1.grade == gp.Item2.Grade && gp.Item2.Confident); int sureWrongTotal = gradePairs.Count(gp => gp.Item1.grade != gp.Item2.Grade && gp.Item2.Confident); int unsureTotal = gradePairs.Count(gp => !gp.Item2.Confident); Console.WriteLine("Total (r/w/u): {0}/{1}/{2}, {3:F1}/{4:F1}/{5:F1} %", sureRightTotal, sureWrongTotal, unsureTotal, percentTotal(sureRightTotal), percentTotal(sureWrongTotal), percentTotal(unsureTotal)); for (byte g = 2; g <= 5; g++) { Console.WriteLine("{0}: {1}/{2}/{3}, {4:F1}/{5:F1}/{6:F1} %", g, sureRights(g), sureWrongs(g), unsure(g), percent(sureRights(g), g), percent(sureWrongs(g), g), percent(unsure(g), g)); } Console.WriteLine(); Console.WriteLine("Recognition failures:"); gradePairs.Where(gp => gp.Item1.grade != gp.Item2.Grade).ToList().ForEach(gp => { GradeDigest failedDigest = testDigests.Find(gd => gd == gp.Item1); Console.WriteLine("file: " + failedDigest.fileName + ", confident: " + gp.Item2.Confident); }); }