private static double scoreBitmap(WordsearchRotation rotation, Classifier classifier) { //Extract each charcater in this wordsearch, then run them through the classifier and sum the liklihoods of // the most probable class to determine an overall score for the image Bitmap[,] chars = null; //If using number of rows & cols for a fixed row/col width/height if(rotation.Segmentation == null) { //Use standardised width & height for characters (do this by first resizing the image) int wordsearchWidth = Constants.CHAR_WITH_WHITESPACE_WIDTH * rotation.Cols; int wordsearchHeight = Constants.CHAR_WITH_WHITESPACE_HEIGHT * rotation.Rows; ResizeBicubic resize = new ResizeBicubic(wordsearchWidth, wordsearchHeight); Bitmap resizedImg = resize.Apply(rotation.Bitmap); //Split the bitmap up into a 2D array of bitmaps chars = SplitImage.Grid(resizedImg, rotation.Rows, rotation.Cols); //If the image got resized, dispose of the resized copy if(resizedImg != rotation.Bitmap) { resizedImg.Dispose(); } } else //Otherwise we have a Segmentation object to use { chars = SplitImage.Segment(rotation.Bitmap, rotation.Segmentation); } double score = 0; foreach(Bitmap charImg in chars) { //Remove all of the whitespace etc... returning an image that can be used for classification Bitmap extractedCharImg = CharImgExtractor.Extract(charImg); //Classify this bitmap double[] charResult = classifier.Classify(extractedCharImg); //Get the largest probability from the classifier output and add it to the overall score double largest = charResult[0]; for(int i = 1; i < charResult.Length; i++) { if(charResult[i] > largest) { largest = charResult[i]; } } score += largest; //Clean up extractedCharImg.Dispose(); charImg.Dispose(); } return score; }
internal static double Evaluate(List<WordsearchImage> wordsearchImages, Classifier classifier) { DefaultLog.Info("Evaluating Wordsearch Image Rotation Correction . . ."); int numCorrect = 0; int numTotal = 0; //Test the algorithm on each Wordsearch Image foreach(WordsearchImage wordsearchImage in wordsearchImages) { //Register an interest in the Bitmap of the wordsearch Image wordsearchImage.RegisterInterestInBitmap(); //Test the system on each posisble rotation of the wordsearch image int[] angles = new int[] { 0, 90, 180, 270 }; Parallel.ForEach(angles, angle => { WordsearchRotation rotation = new WordsearchRotation(wordsearchImage.Bitmap.DeepCopy(), (int)wordsearchImage.Rows, (int)wordsearchImage.Cols); rotation.Rotate(angle); //Rotate the image for the wordsearch back to the correct orientation so that we know the correct answer WordsearchRotation correctRotation = rotation.DeepCopy(); correctRotation.Rotate((360 - angle) % 360); //Angles must be EXACTLY the same as the ones used in the correction in order to yield the same result (i.e. 0, 90, 180, 270) WordsearchRotation proposedRotation = WordsearchRotationCorrection.CorrectOrientation(rotation, classifier); //Keep track of the number correct & total number if (proposedRotation.Bitmap.DataEquals(correctRotation.Bitmap)) { numCorrect++; } numTotal++; //Clean up rotation.Bitmap.Dispose(); correctRotation.Bitmap.Dispose(); proposedRotation.Bitmap.Dispose(); }); //Clean up wordsearchImage.DeregisterInterestInBitmap(); } DefaultLog.Info("Returned {0}/{1} Wordsearch Rotations Correctly", numCorrect, numTotal); DefaultLog.Info("Wordsearch Image Rotation Evaluation Completed"); return (double)numCorrect / numTotal; }
public static WordsearchRotation CorrectOrientation(WordsearchRotation wordsearchRotation, Classifier classifier) //Classifier MUST be probablistic { //Rotate the Bitmap through the 4 possible rotations WordsearchRotation[] rotations = new WordsearchRotation[4]; for(int i = 0; i < rotations.Length; i++) { int angle = i * 90; rotations[i] = wordsearchRotation.DeepCopy(); rotations[i].Rotate(angle); } //Calculate how likely wach rotation is to be correct double bestScore = double.MinValue; int bestIdx = -1; for(int i = 0; i < rotations.Length; i++) { double score = scoreBitmap(rotations[i], classifier); if(score > bestScore) { bestScore = score; bestIdx = i; } } //Dispose of the Wordsearch Rotations that weren't the best for(int i = 0; i < rotations.Length; i++) { if(i != bestIdx) { rotations[i].Bitmap.Dispose(); } } return rotations[bestIdx]; }
internal static WordsearchSolutionEvaluator EvaluateWordsearchBitmap(Bitmap wordsearchBitmap, string[] wordsToFind, Dictionary<string, List<WordPosition>> correctSolutions, SegmentationAlgorithm segmentationAlgorithm, bool segmentationRemoveSmallRowsAndCols, SegmentationMethod segmentationMethod, Classifier probabilisticRotationCorrectionClassifier, Classifier classifier, Solver wordsearchSolver) { /* * Wordsearch Segmentation */ Segmentation segmentation = segmentationAlgorithm.Segment(wordsearchBitmap); //Remove erroneously small rows and columns from the segmentation if that option is specified if(segmentationRemoveSmallRowsAndCols) { segmentation = segmentation.RemoveSmallRowsAndCols(); } /* * Wordsearch Rotation Correction */ WordsearchRotation originalRotation; //If we're using fixed row & col width if (segmentationMethod == SegmentationMethod.FixedWidth) { originalRotation = new WordsearchRotation(wordsearchBitmap, segmentation.NumRows, segmentation.NumCols); } else //Otherwise we're using varied row/col width segmentation, use the Segmentation object { originalRotation = new WordsearchRotation(wordsearchBitmap, segmentation); } WordsearchRotation rotatedWordsearch = WordsearchRotationCorrection.CorrectOrientation(originalRotation, probabilisticRotationCorrectionClassifier); Bitmap rotatedImage = rotatedWordsearch.Bitmap; //If the wordsearch has been rotated if (rotatedImage != wordsearchBitmap) { //Update the segmentation //If the wordsearch rotation won't have been passed a segmentation if (segmentationMethod == SegmentationMethod.FixedWidth) { //Make a new fixed width segmentation from the WordsearchRotation segmentation = new Segmentation(rotatedWordsearch.Rows, rotatedWordsearch.Cols, rotatedImage.Width, rotatedImage.Height); } else { //Use the rotated segmentation segmentation = rotatedWordsearch.Segmentation; } } /* * Classification */ //Split image up into individual characters Bitmap[,] rawCharImgs = null; //If we're using fixed row & col width if (segmentationMethod == SegmentationMethod.FixedWidth) { ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH * segmentation.NumCols, Constants.CHAR_WITH_WHITESPACE_HEIGHT * segmentation.NumRows); Bitmap resizedImage = resize.Apply(rotatedImage); rawCharImgs = SplitImage.Grid(resizedImage, segmentation.NumRows, segmentation.NumCols); //Resized image no longer required resizedImage.Dispose(); } else //Otherwise we're using varied row/col width segmentation { rawCharImgs = SplitImage.Segment(rotatedImage, segmentation); //If the Segmentation Method is to resize the raw char imgs, resize them if (segmentationMethod == SegmentationMethod.VariedWidthWithResize) { ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH, Constants.CHAR_WITH_WHITESPACE_HEIGHT); for (int i = 0; i < rawCharImgs.GetLength(0); i++) { for (int j = 0; j < rawCharImgs.GetLength(1); j++) { //Only do the resize if it isn't already that size if (rawCharImgs[i, j].Width != Constants.CHAR_WITH_WHITESPACE_WIDTH || rawCharImgs[i, j].Height != Constants.CHAR_WITH_WHITESPACE_HEIGHT) { Bitmap orig = rawCharImgs[i, j]; rawCharImgs[i, j] = resize.Apply(orig); //Remove the now unnecessary original/not resized image orig.Dispose(); } } } } } //Full sized rotated image no longer required rotatedImage.Dispose(); //Get the part of the image that actually contains the character (without any whitespace) Bitmap[,] charImgs = CharImgExtractor.ExtractAll(rawCharImgs); //Raw char img's are no longer required rawCharImgs.ToSingleDimension().DisposeAll(); //Perform the classification on all of the images (returns probabilities for each possible class) double[][][] classifierOutput = classifier.Classify(charImgs); //Actual images of the characters are no longer required charImgs.ToSingleDimension().DisposeAll(); /* * Solve Wordsearch */ Solution solution = wordsearchSolver.Solve(classifierOutput, wordsToFind); /* * Evaluate the Proposed Solution */ WordsearchSolutionEvaluator evaluator = new WordsearchSolutionEvaluator(solution, correctSolutions); return evaluator; }
//Helper method to do all of the processing to solve a wordsearch private void doProcessing() { /* * Get all of the selected Algorithms to be used for the processing */ log("Loading Selected Algorithms . . ."); //Wordsearch Detection Segmentation Algorithm SegmentationAlgorithm wordsearchDetectionSegmentationAlgorithm = getSelectedSegmentationAlgorithm(wordsearchDetectionSegmentationToolStripMenuItem); //Wordsearch Segmentation Algorithm SegmentationAlgorithm wordsearchSegmentationAlgorithm = getSelectedSegmentationAlgorithm(wordsearchSegmentationToolStripMenuItem); //Segmentation Post Processor SegmentationPostProcessing segmentationPostProcessor = getSelectedSegmentationPostProcessor(); //Rotation Correction Classifier object (contains both feature extractor & classifier) Classifier rotationCorrectionClassifier = getSelectedClassifier(rotationCorrectionFeatureExtractionToolStripMenuItem, rotationCorrectionClassificationToolStripMenuItem); //Character Image Extraction: Do we normalise the input (with whitespace) image dimensions before finding the biggest blob? // Note that this is ignored if there is equal segmentation spacing (where resizing will always be used as this is how // the classifier was trained) bool charImageExtractionWithWhitespaceNormaliseDimensions = characterImageExtractionInputDimensionsNormalisedToolStripMenuItem.Checked; //Classifier object used for actual classification (contains both feature extractor & classifier) Classifier classifier = getSelectedClassifier(featureExtractionToolStripMenuItem, classificationToolStripMenuItem); //Wordsearch Solver Algorithm Solver wordsearchSolver = getSelectedWordsearchSolver(); log("Selected Algorithms Loaded Successfully!"); /* * Start Processing */ //Start the timer for all processing setProcessingStageState(ProcessingStage.All, CheckState.Indeterminate); //Get the input image Bitmap img = currentBitmap; //Get the words to find string[] wordsToFind = getWordsToFind(); /* * Wordsearch Detection */ //Show that we're starting Wordsearch Detection setProcessingStageState(ProcessingStage.WordsearchDetection, CheckState.Indeterminate); //TODO: Get all candidates & their scores and show these in the image log //Get the candidate most likely to be a Wordsearch Tuple<List<IntPoint>, Bitmap> wordsearchImageTuple = DetectionAlgorithm.ExtractBestWordsearch( img, wordsearchDetectionSegmentationAlgorithm); //If the system failed to find anything remotely resembling a wordsearch, fail now if(wordsearchImageTuple == null) { throw new ImageProcessingException("Wordsearch Detection could not find anything . . ."); } Bitmap wordsearchImage = wordsearchImageTuple.Item2; log(wordsearchImage, "Extracted Wordsearch Image"); //Mark Wordsearch Detection as completed setProcessingStageState(ProcessingStage.WordsearchDetection, CheckState.Checked); /* * Wordsearch Segmentation */ //Show that we're starting Wordsearch Segmentation setProcessingStageState(ProcessingStage.WordsearchSegmentation, CheckState.Indeterminate); Segmentation segmentation = wordsearchSegmentationAlgorithm.Segment(wordsearchImage); //Log the Segmentation (visually) log(DrawGrid.Segmentation(wordsearchImage, segmentation, DRAW_SEGMENTATION_COLOUR), "Segmentation"); //Mark Wordsearch Segmentation as completed setProcessingStageState(ProcessingStage.WordsearchSegmentation, CheckState.Checked); /* * Wordsearch Segmentation Post-Processing */ //Show that we're starting Segmentation Post-Processing setProcessingStageState(ProcessingStage.SegmentationPostProcessing, CheckState.Indeterminate); switch(segmentationPostProcessor) { case SegmentationPostProcessing.RemoveSmallRowsAndCols: segmentation = segmentation.RemoveSmallRowsAndCols(); break; } //If any post-processing was done, log the segmentation (visually) if(segmentationPostProcessor != SegmentationPostProcessing.None) { log(DrawGrid.Segmentation(wordsearchImage, segmentation, DRAW_SEGMENTATION_COLOUR), "Post-Processed Segmentation"); } //Mark Segmentation Post-Processing as complete setProcessingStageState(ProcessingStage.SegmentationPostProcessing, CheckState.Checked); /* * Wordsearch Rotation Correction */ //Show that we're starting Rotation Correction setProcessingStageState(ProcessingStage.RotationCorrection, CheckState.Indeterminate); WordsearchRotation originalRotation; //If the rows & cols in the Segmentation are all equally spaced apart, optimise by using the number of rows & cols if(segmentation.IsEquallySpaced) { originalRotation = new WordsearchRotation(wordsearchImage, segmentation.NumRows, segmentation.NumCols); } else //Otherwise the Segmentation has varied sized row/col width { originalRotation = new WordsearchRotation(wordsearchImage, segmentation); } WordsearchRotation rotatedWordsearch = WordsearchRotationCorrection.CorrectOrientation( originalRotation, rotationCorrectionClassifier); Bitmap rotatedImage = rotatedWordsearch.Bitmap; //If the wordsearch has been rotated if(rotatedImage != wordsearchImage) { //Update the segmentation //If the wordsearch rotation won't have passed a segmentation if(segmentation.IsEquallySpaced) { //Make a new fixed width segmentation from the wordsearchRotation segmentation = new Segmentation(rotatedWordsearch.Rows, rotatedWordsearch.Cols, rotatedImage.Width, rotatedImage.Height); } else //Otherwise the WordsearchRotation will have been working with a Segmentation { //Use the rotated Segmentation object segmentation = rotatedWordsearch.Segmentation; } } //Log the rotated image log(rotatedImage, "Rotated Wordsearch"); log(DrawGrid.Segmentation(rotatedImage, segmentation, DRAW_SEGMENTATION_COLOUR), "Rotated Segmentation"); //Mark Rotation Correction as completed setProcessingStageState(ProcessingStage.RotationCorrection, CheckState.Checked); /* * Character Image Extraction */ //Show that we're starting Character Image Extraction setProcessingStageState(ProcessingStage.CharacterImageExtraction, CheckState.Indeterminate); //Split the image up using the Segmentation Bitmap[,] rawCharImgs = null; //If we're using equally spaced Segmentation if(segmentation.IsEquallySpaced) { //Resize the image first, so that the characters returned when you split the image will already be the correct size ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH * segmentation.NumCols, Constants.CHAR_WITH_WHITESPACE_HEIGHT * segmentation.NumRows); Bitmap resizedImage = resize.Apply(rotatedImage); //Split the image using a standard grid based on the number of rows & cols // (which is correct because of the equally spaced Segmentation) rawCharImgs = SplitImage.Grid(resizedImage, segmentation.NumRows, segmentation.NumCols); //Resized image no longer required resizedImage.Dispose(); } else //Otherwise there is varied spacing between characters { rawCharImgs = SplitImage.Segment(rotatedImage, segmentation); //If we're resizing the raw character images (with whitespace) to consistent dimensions if(charImageExtractionWithWhitespaceNormaliseDimensions) { //Resize the raw char images so that they're all the same dimensions (gives results that are more consistent with how the // classifier was trained: with equally spaced segmentation) ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH, Constants.CHAR_WITH_WHITESPACE_HEIGHT); for (int i = 0; i < rawCharImgs.GetLength(0); i++) { for (int j = 0; j < rawCharImgs.GetLength(1); j++) { //Only do the resize if it isn't already that size if (rawCharImgs[i, j].Width != Constants.CHAR_WITH_WHITESPACE_WIDTH || rawCharImgs[i, j].Height != Constants.CHAR_WITH_WHITESPACE_HEIGHT) { Bitmap orig = rawCharImgs[i, j]; rawCharImgs[i, j] = resize.Apply(orig); //Remove the now unnecessary original/not resized image orig.Dispose(); } } } } } //Log the raw character images (if we've resized them) if(segmentation.IsEquallySpaced || charImageExtractionWithWhitespaceNormaliseDimensions) { log(CombineImages.Grid(rawCharImgs), "Raw Character Images (all chars set to equal width & height)"); } //Get the part of the image that actually contains the character (without any whitespace) Bitmap[,] charImgs = CharImgExtractor.ExtractAll(rawCharImgs); //Raw char img's are no longer required rawCharImgs.ToSingleDimension().DisposeAll(); //Log the extracted character images log(CombineImages.Grid(charImgs), "Extracted Character Images"); //Mark Character Image Extraction as completed setProcessingStageState(ProcessingStage.CharacterImageExtraction, CheckState.Checked); /* * Feature Extraction & Classification */ //Show that we're starting Feature Extraction & Classification setProcessingStageState(ProcessingStage.FeatureExtractionAndClassification, CheckState.Indeterminate); double[][][] charProbabilities = classifier.Classify(charImgs); //Actual images of the characters are no longer required charImgs.ToSingleDimension().DisposeAll(); //Log the wordsearch as classified char[,] classifiedChars = NeuralNetworkHelpers.GetMostLikelyChars(charProbabilities); log("Wordsearch as classified (character that was given the highest probability):"); for (int i = 0; i < classifiedChars.GetLength(1); i++) //Rows { StringBuilder builder = new StringBuilder(); //Cols for(int j = 0; j < classifiedChars.GetLength(0); j++) { builder.Append(classifiedChars[j, i]); } log(builder.ToString()); } //Mark Feature Extraction & Classification as completed setProcessingStageState(ProcessingStage.FeatureExtractionAndClassification, CheckState.Checked); /* * Solve Wordsearch */ //Show that we're starting to Solve the Wordsearch setProcessingStageState(ProcessingStage.WordsearchSolver, CheckState.Indeterminate); Solution solution = wordsearchSolver.Solve(charProbabilities, wordsToFind); //Log the solution visually Bitmap bitmapSolution = DrawSolution.Solution(rotatedImage, segmentation, solution, DRAW_WORDS_COLOUR); log(bitmapSolution, "Solution"); log(DrawGrid.Segmentation(bitmapSolution, segmentation, DRAW_SEGMENTATION_COLOUR), "Solution + Segmentation"); //Log an Image for the solution to each word so that you can see where it thinks each word is foreach(KeyValuePair<string, WordPosition> kvp in solution) { string word = kvp.Key; WordPosition position = kvp.Value; //Draw this WordPosition onto the rotated image log(DrawSolution.WordPosition(rotatedImage, segmentation, position, DRAW_WORDS_COLOUR), String.Format("Solution for Word: {0}", word)); } //Mark the wordsearch as having been solved setProcessingStageState(ProcessingStage.WordsearchSolver, CheckState.Checked); //Log the time taken to complete all of the processing setProcessingStageState(ProcessingStage.All, CheckState.Checked); }