private static double scoreBitmap(WordsearchRotation rotation, Classifier classifier)
        {
            //Extract each charcater in this wordsearch, then run them through the classifier and sum the liklihoods of 
            //  the most probable class to determine an overall score for the image
            Bitmap[,] chars = null;

            //If using number of rows & cols for a fixed row/col width/height
            if(rotation.Segmentation == null)
            {
                //Use standardised width & height for characters (do this by first resizing the image)
                int wordsearchWidth = Constants.CHAR_WITH_WHITESPACE_WIDTH * rotation.Cols;
                int wordsearchHeight = Constants.CHAR_WITH_WHITESPACE_HEIGHT * rotation.Rows;

                ResizeBicubic resize = new ResizeBicubic(wordsearchWidth, wordsearchHeight);
                Bitmap resizedImg = resize.Apply(rotation.Bitmap);

                //Split the bitmap up into a 2D array of bitmaps
                chars = SplitImage.Grid(resizedImg, rotation.Rows, rotation.Cols);

                //If the image got resized, dispose of the resized copy
                if(resizedImg != rotation.Bitmap)
                {
                    resizedImg.Dispose();
                }
            }
            else //Otherwise we have a Segmentation object to use
            {
                chars = SplitImage.Segment(rotation.Bitmap, rotation.Segmentation);
            }

            double score = 0;
            foreach(Bitmap charImg in chars)
            {
                //Remove all of the whitespace etc... returning an image that can be used for classification
                Bitmap extractedCharImg = CharImgExtractor.Extract(charImg);

                //Classify this bitmap
                double[] charResult = classifier.Classify(extractedCharImg);

                //Get the largest probability from the classifier output and add it to the overall score
                double largest = charResult[0];
                for(int i = 1; i < charResult.Length; i++)
                {
                    if(charResult[i] > largest)
                    {
                        largest = charResult[i];
                    }
                }

                score += largest;

                //Clean up
                extractedCharImg.Dispose();
                charImg.Dispose();
            }

            return score;
        }
        internal static double Evaluate(List<WordsearchImage> wordsearchImages, Classifier classifier)
        {
            DefaultLog.Info("Evaluating Wordsearch Image Rotation Correction . . .");

            int numCorrect = 0;
            int numTotal = 0;

            //Test the algorithm on each Wordsearch Image
            foreach(WordsearchImage wordsearchImage in wordsearchImages)
            {
                //Register an interest in the Bitmap of the wordsearch Image
                wordsearchImage.RegisterInterestInBitmap();

                //Test the system on each posisble rotation of the wordsearch image
                int[] angles = new int[] { 0, 90, 180, 270 };

                Parallel.ForEach(angles, angle =>
                {
                    WordsearchRotation rotation = new WordsearchRotation(wordsearchImage.Bitmap.DeepCopy(), (int)wordsearchImage.Rows, (int)wordsearchImage.Cols);
                    rotation.Rotate(angle);

                    //Rotate the image for the wordsearch back to the correct orientation so that we know the correct answer
                    WordsearchRotation correctRotation = rotation.DeepCopy();
                    correctRotation.Rotate((360 - angle) % 360); //Angles must be EXACTLY the same as the ones used in the correction in order to yield the same result (i.e. 0, 90, 180, 270)

                    WordsearchRotation proposedRotation = WordsearchRotationCorrection.CorrectOrientation(rotation, classifier);

                    //Keep track of the number correct & total number
                    if (proposedRotation.Bitmap.DataEquals(correctRotation.Bitmap))
                    {
                        numCorrect++;
                    }
                    numTotal++;

                    //Clean up
                    rotation.Bitmap.Dispose();
                    correctRotation.Bitmap.Dispose();
                    proposedRotation.Bitmap.Dispose();
                });

                //Clean up
                wordsearchImage.DeregisterInterestInBitmap();
            }

            DefaultLog.Info("Returned {0}/{1} Wordsearch Rotations Correctly", numCorrect, numTotal);

            DefaultLog.Info("Wordsearch Image Rotation Evaluation Completed");

            return (double)numCorrect / numTotal;
        }
 //Constructors
 public AlgorithmCombination(SegmentationAlgorithm detectionSegmentationAlgorithm,
     bool detectionSegmentationRemoveSmallRowsAndCols, SegmentationAlgorithm segmentationAlgorithm, 
     bool segmentationRemoveSmallRowsAndCols, EvaluateFullSystem.SegmentationMethod segmentationMethod,
     Classifier probabilisticRotationCorrectionClassifier, Classifier classifier,
     Solver wordsearchSolver)
 {
     this.DetectionSegmentationAlgorithm = detectionSegmentationAlgorithm;
     this.DetectionSegmentationRemoveSmallRowsAndCols = detectionSegmentationRemoveSmallRowsAndCols;
     this.SegmentationAlgorithm = segmentationAlgorithm;
     this.SegmentationRemoveSmallRowsAndCols = segmentationRemoveSmallRowsAndCols;
     this.SegmentationMethod = segmentationMethod;
     this.ProbabilisticRotationCorrectionClassifier = probabilisticRotationCorrectionClassifier;
     this.Classifier = classifier;
     this.WordsearchSolver = wordsearchSolver;
 }
        public static WordsearchRotation CorrectOrientation(WordsearchRotation wordsearchRotation, Classifier classifier) //Classifier MUST be probablistic
        {
            //Rotate the Bitmap through the 4 possible rotations
            WordsearchRotation[] rotations = new WordsearchRotation[4];

            for(int i = 0; i < rotations.Length; i++)
            {
                int angle = i * 90;

                rotations[i] = wordsearchRotation.DeepCopy();
                rotations[i].Rotate(angle);
            }

            //Calculate how likely wach rotation is to be correct
            double bestScore = double.MinValue;
            int bestIdx = -1;
            for(int i = 0; i < rotations.Length; i++)
            {
                double score = scoreBitmap(rotations[i], classifier);

                if(score > bestScore)
                {
                    bestScore = score;
                    bestIdx = i;
                }
            }

            //Dispose of the Wordsearch Rotations that weren't the best
            for(int i = 0; i < rotations.Length; i++)
            {
                if(i != bestIdx)
                {
                    rotations[i].Bitmap.Dispose();
                }
            }

            return rotations[bestIdx];
        }
        internal static WordsearchSolutionEvaluator EvaluateWordsearchBitmap(Bitmap wordsearchBitmap, string[] wordsToFind,
            Dictionary<string, List<WordPosition>> correctSolutions, SegmentationAlgorithm segmentationAlgorithm, 
            bool segmentationRemoveSmallRowsAndCols, SegmentationMethod segmentationMethod,
            Classifier probabilisticRotationCorrectionClassifier, Classifier classifier, Solver wordsearchSolver)
        {
            /*
             * Wordsearch Segmentation
             */
            Segmentation segmentation = segmentationAlgorithm.Segment(wordsearchBitmap);

            //Remove erroneously small rows and columns from the segmentation if that option is specified
            if(segmentationRemoveSmallRowsAndCols)
            {
                segmentation = segmentation.RemoveSmallRowsAndCols();
            }

            /*
             * Wordsearch Rotation Correction
             */
            WordsearchRotation originalRotation;

            //If we're using fixed row & col width
            if (segmentationMethod == SegmentationMethod.FixedWidth)
            {
                originalRotation = new WordsearchRotation(wordsearchBitmap, segmentation.NumRows, segmentation.NumCols);
            }
            else //Otherwise we're using varied row/col width segmentation, use the Segmentation object
            {
                originalRotation = new WordsearchRotation(wordsearchBitmap, segmentation);
            }

            WordsearchRotation rotatedWordsearch = WordsearchRotationCorrection.CorrectOrientation(originalRotation, probabilisticRotationCorrectionClassifier);

            Bitmap rotatedImage = rotatedWordsearch.Bitmap;

            //If the wordsearch has been rotated
            if (rotatedImage != wordsearchBitmap)
            {
                //Update the segmentation

                //If the wordsearch rotation won't have been passed a segmentation
                if (segmentationMethod == SegmentationMethod.FixedWidth)
                {
                    //Make a new fixed width segmentation from the WordsearchRotation
                    segmentation = new Segmentation(rotatedWordsearch.Rows, rotatedWordsearch.Cols,
                        rotatedImage.Width, rotatedImage.Height);
                }
                else
                {
                    //Use the rotated segmentation 
                    segmentation = rotatedWordsearch.Segmentation;
                }
            }

            /*
             * Classification
             */

            //Split image up into individual characters
            Bitmap[,] rawCharImgs = null;

            //If we're using fixed row & col width
            if (segmentationMethod == SegmentationMethod.FixedWidth)
            {
                ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH * segmentation.NumCols,
                    Constants.CHAR_WITH_WHITESPACE_HEIGHT * segmentation.NumRows);
                Bitmap resizedImage = resize.Apply(rotatedImage);

                rawCharImgs = SplitImage.Grid(resizedImage, segmentation.NumRows, segmentation.NumCols);

                //Resized image no longer required
                resizedImage.Dispose();
            }
            else //Otherwise we're using varied row/col width segmentation
            {
                rawCharImgs = SplitImage.Segment(rotatedImage, segmentation);

                //If the Segmentation Method is to resize the raw char imgs, resize them
                if (segmentationMethod == SegmentationMethod.VariedWidthWithResize)
                {
                    ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH, Constants.CHAR_WITH_WHITESPACE_HEIGHT);

                    for (int i = 0; i < rawCharImgs.GetLength(0); i++)
                    {
                        for (int j = 0; j < rawCharImgs.GetLength(1); j++)
                        {
                            //Only do the resize if it isn't already that size
                            if (rawCharImgs[i, j].Width != Constants.CHAR_WITH_WHITESPACE_WIDTH
                                || rawCharImgs[i, j].Height != Constants.CHAR_WITH_WHITESPACE_HEIGHT)
                            {
                                Bitmap orig = rawCharImgs[i, j];

                                rawCharImgs[i, j] = resize.Apply(orig);

                                //Remove the now unnecessary original/not resized image
                                orig.Dispose();
                            }
                        }
                    }
                }
            }

            //Full sized rotated image no longer required
            rotatedImage.Dispose();

            //Get the part of the image that actually contains the character (without any whitespace)
            Bitmap[,] charImgs = CharImgExtractor.ExtractAll(rawCharImgs);

            //Raw char img's are no longer required
            rawCharImgs.ToSingleDimension().DisposeAll();

            //Perform the classification on all of the images (returns probabilities for each possible class)
            double[][][] classifierOutput = classifier.Classify(charImgs);

            //Actual images of the characters are no longer required
            charImgs.ToSingleDimension().DisposeAll();

            /*
             * Solve Wordsearch
             */
            Solution solution = wordsearchSolver.Solve(classifierOutput, wordsToFind);

            /*
             * Evaluate the Proposed Solution
             */
            WordsearchSolutionEvaluator evaluator = new WordsearchSolutionEvaluator(solution, correctSolutions);

            return evaluator;
        }
        private static double Evaluate(List<Image> images, SegmentationAlgorithm detectionSegmentationAlgorithm, 
            bool detectionSegmentationRemoveSmallRowsAndCols, SegmentationAlgorithm segmentationAlgorithm, 
            bool segmentationRemoveSmallRowsAndCols, SegmentationMethod segmentationMethod, 
            Classifier probabilisticRotationCorrectionClassifier, Classifier classifier, Solver wordsearchSolver)
        {
            DefaultLog.Info("Evaluating Full System . . .");

            int numCorrect = 0;
            List<WordsearchSolutionEvaluator> evaluators = new List<WordsearchSolutionEvaluator>();

            foreach(Image image in images)
            {
                //Register an interest in the Bitmap of the image
                image.RegisterInterestInBitmap();

                /*
                 * Wordsearch Detection
                 */
                Tuple<List<IntPoint>, Bitmap> wordsearchImageTuple = DetectionAlgorithm.ExtractBestWordsearch(image.Bitmap, detectionSegmentationAlgorithm, detectionSegmentationRemoveSmallRowsAndCols);
                
                //Original wordsearch image is no longer required
                image.DeregisterInterestInBitmap();

                //If the system failed to find anything remotely resembling a wordsearch, fail now
                if(wordsearchImageTuple == null)
                {
                    continue;
                }

                //Get the words to look for later from this image & the correct solutions
                string[] wordsToFind = null; //Requires default, but won't even get used
                Dictionary<string, List<WordPosition>> correctSolutions = null;
                //If the image contains more than one wordsearch, we need to work out which one has been found
                if(image.WordsearchImages.Length > 1)
                {
                    List<IntPoint> coordinates = wordsearchImageTuple.Item1;
                    bool found = false;

                    //Select the wordsearch found using the algorithm for checking if the returned wordsearch is correct in EvaluateWordsearchDetection
                    foreach(WordsearchImage wordsearchImage in image.WordsearchImages)
                    {
                        //If it's this wordsearch
                        if(EvaluateWordsearchDetection.IsWordsearch(coordinates, wordsearchImage))
                        {
                            wordsToFind = wordsearchImage.Wordsearch.Words;
                            correctSolutions = wordsearchImage.Wordsearch.Solutions;
                            found = true;
                            break;
                        }
                    }

                    //If this isn't one of the wordsearches in the image, then fail now 
                    if(!found)
                    {
                        //Clean up
                        wordsearchImageTuple.Item2.Dispose();

                        continue;
                    }
                }
                else //Otherwise just use the one wordsearch that's in the image
                {
                    wordsToFind = image.WordsearchImages[0].Wordsearch.Words;
                    correctSolutions = image.WordsearchImages[0].Wordsearch.Solutions;
                }
                
                Bitmap extractedImage = wordsearchImageTuple.Item2;

                /*
                 * Image Segmentation onwards happen in EvaluateWordsearchBitmap
                 */
                WordsearchSolutionEvaluator evaluator = EvaluateWordsearchBitmap(extractedImage, wordsToFind, correctSolutions,
                    segmentationAlgorithm, segmentationRemoveSmallRowsAndCols, segmentationMethod, 
                    probabilisticRotationCorrectionClassifier, classifier, wordsearchSolver);

                //Clean up
                extractedImage.Dispose();

                //Log Evaluation
                evaluators.Add(evaluator);

                DefaultLog.Info(evaluator.ToString());

                if(evaluator.Correct)
                {
                    numCorrect++;
                }
            }

            DefaultLog.Info("System found all words correctly for {0} / {1} Images correctly", numCorrect, images.Count);

            //Calculate some extra statistics
            int numWordsearchesNoWordsFound = 0;
            int numDidntReachEvaluation = images.Count - evaluators.Count;
            double fMeasureSum = 0;
            int numValidFMeasures = 0;

            foreach (WordsearchSolutionEvaluator evaluator in evaluators)
            {
                //If no words were found correctly
                if(evaluator.TruePositive == 0)
                {
                    numWordsearchesNoWordsFound++;
                }

                //If there was a valid F-Measure
                if(!double.IsNaN(evaluator.FMeasure))
                {
                    fMeasureSum += evaluator.FMeasure;
                    numValidFMeasures++;
                }
            }

            DefaultLog.Info("In {0} wordsearches no words were found correctly at all", numWordsearchesNoWordsFound);
            DefaultLog.Info("{0} wordsearch images got discarded before reaching the evaluation stage", numDidntReachEvaluation);
            DefaultLog.Info("Average F-Measure (when not NaN): {0}", fMeasureSum / numValidFMeasures);

            DefaultLog.Info("Full System Evaluation Completed");

            return (double)numCorrect / images.Count;
        }