private static double scoreBitmap(WordsearchRotation rotation, Classifier classifier)
{
//Extract each charcater in this wordsearch, then run them through the classifier and sum the liklihoods of
// the most probable class to determine an overall score for the image
Bitmap[,] chars = null;
//If using number of rows & cols for a fixed row/col width/height
if(rotation.Segmentation == null)
{
//Use standardised width & height for characters (do this by first resizing the image)
int wordsearchWidth = Constants.CHAR_WITH_WHITESPACE_WIDTH * rotation.Cols;
int wordsearchHeight = Constants.CHAR_WITH_WHITESPACE_HEIGHT * rotation.Rows;
ResizeBicubic resize = new ResizeBicubic(wordsearchWidth, wordsearchHeight);
Bitmap resizedImg = resize.Apply(rotation.Bitmap);
//Split the bitmap up into a 2D array of bitmaps
chars = SplitImage.Grid(resizedImg, rotation.Rows, rotation.Cols);
//If the image got resized, dispose of the resized copy
if(resizedImg != rotation.Bitmap)
{
resizedImg.Dispose();
}
}
else //Otherwise we have a Segmentation object to use
{
chars = SplitImage.Segment(rotation.Bitmap, rotation.Segmentation);
}
double score = 0;
foreach(Bitmap charImg in chars)
{
//Remove all of the whitespace etc... returning an image that can be used for classification
Bitmap extractedCharImg = CharImgExtractor.Extract(charImg);
//Classify this bitmap
double[] charResult = classifier.Classify(extractedCharImg);
//Get the largest probability from the classifier output and add it to the overall score
double largest = charResult[0];
for(int i = 1; i < charResult.Length; i++)
{
if(charResult[i] > largest)
{
largest = charResult[i];
}
}
score += largest;
//Clean up
extractedCharImg.Dispose();
charImg.Dispose();
}
return score;
}
internal static double Evaluate(List<WordsearchImage> wordsearchImages, Classifier classifier)
{
DefaultLog.Info("Evaluating Wordsearch Image Rotation Correction . . .");
int numCorrect = 0;
int numTotal = 0;
//Test the algorithm on each Wordsearch Image
foreach(WordsearchImage wordsearchImage in wordsearchImages)
{
//Register an interest in the Bitmap of the wordsearch Image
wordsearchImage.RegisterInterestInBitmap();
//Test the system on each posisble rotation of the wordsearch image
int[] angles = new int[] { 0, 90, 180, 270 };
Parallel.ForEach(angles, angle =>
{
WordsearchRotation rotation = new WordsearchRotation(wordsearchImage.Bitmap.DeepCopy(), (int)wordsearchImage.Rows, (int)wordsearchImage.Cols);
rotation.Rotate(angle);
//Rotate the image for the wordsearch back to the correct orientation so that we know the correct answer
WordsearchRotation correctRotation = rotation.DeepCopy();
correctRotation.Rotate((360 - angle) % 360); //Angles must be EXACTLY the same as the ones used in the correction in order to yield the same result (i.e. 0, 90, 180, 270)
WordsearchRotation proposedRotation = WordsearchRotationCorrection.CorrectOrientation(rotation, classifier);
//Keep track of the number correct & total number
if (proposedRotation.Bitmap.DataEquals(correctRotation.Bitmap))
{
numCorrect++;
}
numTotal++;
//Clean up
rotation.Bitmap.Dispose();
correctRotation.Bitmap.Dispose();
proposedRotation.Bitmap.Dispose();
});
//Clean up
wordsearchImage.DeregisterInterestInBitmap();
}
DefaultLog.Info("Returned {0}/{1} Wordsearch Rotations Correctly", numCorrect, numTotal);
DefaultLog.Info("Wordsearch Image Rotation Evaluation Completed");
return (double)numCorrect / numTotal;
}
public static WordsearchRotation CorrectOrientation(WordsearchRotation wordsearchRotation, Classifier classifier) //Classifier MUST be probablistic
{
//Rotate the Bitmap through the 4 possible rotations
WordsearchRotation[] rotations = new WordsearchRotation[4];
for(int i = 0; i < rotations.Length; i++)
{
int angle = i * 90;
rotations[i] = wordsearchRotation.DeepCopy();
rotations[i].Rotate(angle);
}
//Calculate how likely wach rotation is to be correct
double bestScore = double.MinValue;
int bestIdx = -1;
for(int i = 0; i < rotations.Length; i++)
{
double score = scoreBitmap(rotations[i], classifier);
if(score > bestScore)
{
bestScore = score;
bestIdx = i;
}
}
//Dispose of the Wordsearch Rotations that weren't the best
for(int i = 0; i < rotations.Length; i++)
{
if(i != bestIdx)
{
rotations[i].Bitmap.Dispose();
}
}
return rotations[bestIdx];
}
internal static WordsearchSolutionEvaluator EvaluateWordsearchBitmap(Bitmap wordsearchBitmap, string[] wordsToFind,
Dictionary<string, List<WordPosition>> correctSolutions, SegmentationAlgorithm segmentationAlgorithm,
bool segmentationRemoveSmallRowsAndCols, SegmentationMethod segmentationMethod,
Classifier probabilisticRotationCorrectionClassifier, Classifier classifier, Solver wordsearchSolver)
{
/*
* Wordsearch Segmentation
*/
Segmentation segmentation = segmentationAlgorithm.Segment(wordsearchBitmap);
//Remove erroneously small rows and columns from the segmentation if that option is specified
if(segmentationRemoveSmallRowsAndCols)
{
segmentation = segmentation.RemoveSmallRowsAndCols();
}
/*
* Wordsearch Rotation Correction
*/
WordsearchRotation originalRotation;
//If we're using fixed row & col width
if (segmentationMethod == SegmentationMethod.FixedWidth)
{
originalRotation = new WordsearchRotation(wordsearchBitmap, segmentation.NumRows, segmentation.NumCols);
}
else //Otherwise we're using varied row/col width segmentation, use the Segmentation object
{
originalRotation = new WordsearchRotation(wordsearchBitmap, segmentation);
}
WordsearchRotation rotatedWordsearch = WordsearchRotationCorrection.CorrectOrientation(originalRotation, probabilisticRotationCorrectionClassifier);
Bitmap rotatedImage = rotatedWordsearch.Bitmap;
//If the wordsearch has been rotated
if (rotatedImage != wordsearchBitmap)
{
//Update the segmentation
//If the wordsearch rotation won't have been passed a segmentation
if (segmentationMethod == SegmentationMethod.FixedWidth)
{
//Make a new fixed width segmentation from the WordsearchRotation
segmentation = new Segmentation(rotatedWordsearch.Rows, rotatedWordsearch.Cols,
rotatedImage.Width, rotatedImage.Height);
}
else
{
//Use the rotated segmentation
segmentation = rotatedWordsearch.Segmentation;
}
}
/*
* Classification
*/
//Split image up into individual characters
Bitmap[,] rawCharImgs = null;
//If we're using fixed row & col width
if (segmentationMethod == SegmentationMethod.FixedWidth)
{
ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH * segmentation.NumCols,
Constants.CHAR_WITH_WHITESPACE_HEIGHT * segmentation.NumRows);
Bitmap resizedImage = resize.Apply(rotatedImage);
rawCharImgs = SplitImage.Grid(resizedImage, segmentation.NumRows, segmentation.NumCols);
//Resized image no longer required
resizedImage.Dispose();
}
else //Otherwise we're using varied row/col width segmentation
{
rawCharImgs = SplitImage.Segment(rotatedImage, segmentation);
//If the Segmentation Method is to resize the raw char imgs, resize them
if (segmentationMethod == SegmentationMethod.VariedWidthWithResize)
{
ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH, Constants.CHAR_WITH_WHITESPACE_HEIGHT);
for (int i = 0; i < rawCharImgs.GetLength(0); i++)
{
for (int j = 0; j < rawCharImgs.GetLength(1); j++)
{
//Only do the resize if it isn't already that size
if (rawCharImgs[i, j].Width != Constants.CHAR_WITH_WHITESPACE_WIDTH
|| rawCharImgs[i, j].Height != Constants.CHAR_WITH_WHITESPACE_HEIGHT)
{
Bitmap orig = rawCharImgs[i, j];
rawCharImgs[i, j] = resize.Apply(orig);
//Remove the now unnecessary original/not resized image
orig.Dispose();
}
}
}
}
}
//Full sized rotated image no longer required
rotatedImage.Dispose();
//Get the part of the image that actually contains the character (without any whitespace)
Bitmap[,] charImgs = CharImgExtractor.ExtractAll(rawCharImgs);
//Raw char img's are no longer required
rawCharImgs.ToSingleDimension().DisposeAll();
//Perform the classification on all of the images (returns probabilities for each possible class)
double[][][] classifierOutput = classifier.Classify(charImgs);
//Actual images of the characters are no longer required
charImgs.ToSingleDimension().DisposeAll();
/*
* Solve Wordsearch
*/
Solution solution = wordsearchSolver.Solve(classifierOutput, wordsToFind);
/*
* Evaluate the Proposed Solution
*/
WordsearchSolutionEvaluator evaluator = new WordsearchSolutionEvaluator(solution, correctSolutions);
return evaluator;
}
//Helper method to do all of the processing to solve a wordsearch
private void doProcessing()
{
/*
* Get all of the selected Algorithms to be used for the processing
*/
log("Loading Selected Algorithms . . .");
//Wordsearch Detection Segmentation Algorithm
SegmentationAlgorithm wordsearchDetectionSegmentationAlgorithm =
getSelectedSegmentationAlgorithm(wordsearchDetectionSegmentationToolStripMenuItem);
//Wordsearch Segmentation Algorithm
SegmentationAlgorithm wordsearchSegmentationAlgorithm =
getSelectedSegmentationAlgorithm(wordsearchSegmentationToolStripMenuItem);
//Segmentation Post Processor
SegmentationPostProcessing segmentationPostProcessor = getSelectedSegmentationPostProcessor();
//Rotation Correction Classifier object (contains both feature extractor & classifier)
Classifier rotationCorrectionClassifier = getSelectedClassifier(rotationCorrectionFeatureExtractionToolStripMenuItem,
rotationCorrectionClassificationToolStripMenuItem);
//Character Image Extraction: Do we normalise the input (with whitespace) image dimensions before finding the biggest blob?
// Note that this is ignored if there is equal segmentation spacing (where resizing will always be used as this is how
// the classifier was trained)
bool charImageExtractionWithWhitespaceNormaliseDimensions = characterImageExtractionInputDimensionsNormalisedToolStripMenuItem.Checked;
//Classifier object used for actual classification (contains both feature extractor & classifier)
Classifier classifier = getSelectedClassifier(featureExtractionToolStripMenuItem, classificationToolStripMenuItem);
//Wordsearch Solver Algorithm
Solver wordsearchSolver = getSelectedWordsearchSolver();
log("Selected Algorithms Loaded Successfully!");
/*
* Start Processing
*/
//Start the timer for all processing
setProcessingStageState(ProcessingStage.All, CheckState.Indeterminate);
//Get the input image
Bitmap img = currentBitmap;
//Get the words to find
string[] wordsToFind = getWordsToFind();
/*
* Wordsearch Detection
*/
//Show that we're starting Wordsearch Detection
setProcessingStageState(ProcessingStage.WordsearchDetection, CheckState.Indeterminate);
//TODO: Get all candidates & their scores and show these in the image log
//Get the candidate most likely to be a Wordsearch
Tuple<List<IntPoint>, Bitmap> wordsearchImageTuple = DetectionAlgorithm.ExtractBestWordsearch(
img, wordsearchDetectionSegmentationAlgorithm);
//If the system failed to find anything remotely resembling a wordsearch, fail now
if(wordsearchImageTuple == null)
{
throw new ImageProcessingException("Wordsearch Detection could not find anything . . .");
}
Bitmap wordsearchImage = wordsearchImageTuple.Item2;
log(wordsearchImage, "Extracted Wordsearch Image");
//Mark Wordsearch Detection as completed
setProcessingStageState(ProcessingStage.WordsearchDetection, CheckState.Checked);
/*
* Wordsearch Segmentation
*/
//Show that we're starting Wordsearch Segmentation
setProcessingStageState(ProcessingStage.WordsearchSegmentation, CheckState.Indeterminate);
Segmentation segmentation = wordsearchSegmentationAlgorithm.Segment(wordsearchImage);
//Log the Segmentation (visually)
log(DrawGrid.Segmentation(wordsearchImage, segmentation, DRAW_SEGMENTATION_COLOUR), "Segmentation");
//Mark Wordsearch Segmentation as completed
setProcessingStageState(ProcessingStage.WordsearchSegmentation, CheckState.Checked);
/*
* Wordsearch Segmentation Post-Processing
*/
//Show that we're starting Segmentation Post-Processing
setProcessingStageState(ProcessingStage.SegmentationPostProcessing, CheckState.Indeterminate);
switch(segmentationPostProcessor)
{
case SegmentationPostProcessing.RemoveSmallRowsAndCols:
segmentation = segmentation.RemoveSmallRowsAndCols();
break;
}
//If any post-processing was done, log the segmentation (visually)
if(segmentationPostProcessor != SegmentationPostProcessing.None)
{
log(DrawGrid.Segmentation(wordsearchImage, segmentation, DRAW_SEGMENTATION_COLOUR), "Post-Processed Segmentation");
}
//Mark Segmentation Post-Processing as complete
setProcessingStageState(ProcessingStage.SegmentationPostProcessing, CheckState.Checked);
/*
* Wordsearch Rotation Correction
*/
//Show that we're starting Rotation Correction
setProcessingStageState(ProcessingStage.RotationCorrection, CheckState.Indeterminate);
WordsearchRotation originalRotation;
//If the rows & cols in the Segmentation are all equally spaced apart, optimise by using the number of rows & cols
if(segmentation.IsEquallySpaced)
{
originalRotation = new WordsearchRotation(wordsearchImage, segmentation.NumRows, segmentation.NumCols);
}
else //Otherwise the Segmentation has varied sized row/col width
{
originalRotation = new WordsearchRotation(wordsearchImage, segmentation);
}
WordsearchRotation rotatedWordsearch = WordsearchRotationCorrection.CorrectOrientation(
originalRotation, rotationCorrectionClassifier);
Bitmap rotatedImage = rotatedWordsearch.Bitmap;
//If the wordsearch has been rotated
if(rotatedImage != wordsearchImage)
{
//Update the segmentation
//If the wordsearch rotation won't have passed a segmentation
if(segmentation.IsEquallySpaced)
{
//Make a new fixed width segmentation from the wordsearchRotation
segmentation = new Segmentation(rotatedWordsearch.Rows, rotatedWordsearch.Cols,
rotatedImage.Width, rotatedImage.Height);
}
else //Otherwise the WordsearchRotation will have been working with a Segmentation
{
//Use the rotated Segmentation object
segmentation = rotatedWordsearch.Segmentation;
}
}
//Log the rotated image
log(rotatedImage, "Rotated Wordsearch");
log(DrawGrid.Segmentation(rotatedImage, segmentation, DRAW_SEGMENTATION_COLOUR), "Rotated Segmentation");
//Mark Rotation Correction as completed
setProcessingStageState(ProcessingStage.RotationCorrection, CheckState.Checked);
/*
* Character Image Extraction
*/
//Show that we're starting Character Image Extraction
setProcessingStageState(ProcessingStage.CharacterImageExtraction, CheckState.Indeterminate);
//Split the image up using the Segmentation
Bitmap[,] rawCharImgs = null;
//If we're using equally spaced Segmentation
if(segmentation.IsEquallySpaced)
{
//Resize the image first, so that the characters returned when you split the image will already be the correct size
ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH * segmentation.NumCols,
Constants.CHAR_WITH_WHITESPACE_HEIGHT * segmentation.NumRows);
Bitmap resizedImage = resize.Apply(rotatedImage);
//Split the image using a standard grid based on the number of rows & cols
// (which is correct because of the equally spaced Segmentation)
rawCharImgs = SplitImage.Grid(resizedImage, segmentation.NumRows, segmentation.NumCols);
//Resized image no longer required
resizedImage.Dispose();
}
else //Otherwise there is varied spacing between characters
{
rawCharImgs = SplitImage.Segment(rotatedImage, segmentation);
//If we're resizing the raw character images (with whitespace) to consistent dimensions
if(charImageExtractionWithWhitespaceNormaliseDimensions)
{
//Resize the raw char images so that they're all the same dimensions (gives results that are more consistent with how the
// classifier was trained: with equally spaced segmentation)
ResizeBicubic resize = new ResizeBicubic(Constants.CHAR_WITH_WHITESPACE_WIDTH, Constants.CHAR_WITH_WHITESPACE_HEIGHT);
for (int i = 0; i < rawCharImgs.GetLength(0); i++)
{
for (int j = 0; j < rawCharImgs.GetLength(1); j++)
{
//Only do the resize if it isn't already that size
if (rawCharImgs[i, j].Width != Constants.CHAR_WITH_WHITESPACE_WIDTH
|| rawCharImgs[i, j].Height != Constants.CHAR_WITH_WHITESPACE_HEIGHT)
{
Bitmap orig = rawCharImgs[i, j];
rawCharImgs[i, j] = resize.Apply(orig);
//Remove the now unnecessary original/not resized image
orig.Dispose();
}
}
}
}
}
//Log the raw character images (if we've resized them)
if(segmentation.IsEquallySpaced || charImageExtractionWithWhitespaceNormaliseDimensions)
{
log(CombineImages.Grid(rawCharImgs), "Raw Character Images (all chars set to equal width & height)");
}
//Get the part of the image that actually contains the character (without any whitespace)
Bitmap[,] charImgs = CharImgExtractor.ExtractAll(rawCharImgs);
//Raw char img's are no longer required
rawCharImgs.ToSingleDimension().DisposeAll();
//Log the extracted character images
log(CombineImages.Grid(charImgs), "Extracted Character Images");
//Mark Character Image Extraction as completed
setProcessingStageState(ProcessingStage.CharacterImageExtraction, CheckState.Checked);
/*
* Feature Extraction & Classification
*/
//Show that we're starting Feature Extraction & Classification
setProcessingStageState(ProcessingStage.FeatureExtractionAndClassification, CheckState.Indeterminate);
double[][][] charProbabilities = classifier.Classify(charImgs);
//Actual images of the characters are no longer required
charImgs.ToSingleDimension().DisposeAll();
//Log the wordsearch as classified
char[,] classifiedChars = NeuralNetworkHelpers.GetMostLikelyChars(charProbabilities);
log("Wordsearch as classified (character that was given the highest probability):");
for (int i = 0; i < classifiedChars.GetLength(1); i++) //Rows
{
StringBuilder builder = new StringBuilder();
//Cols
for(int j = 0; j < classifiedChars.GetLength(0); j++)
{
builder.Append(classifiedChars[j, i]);
}
log(builder.ToString());
}
//Mark Feature Extraction & Classification as completed
setProcessingStageState(ProcessingStage.FeatureExtractionAndClassification, CheckState.Checked);
/*
* Solve Wordsearch
*/
//Show that we're starting to Solve the Wordsearch
setProcessingStageState(ProcessingStage.WordsearchSolver, CheckState.Indeterminate);
Solution solution = wordsearchSolver.Solve(charProbabilities, wordsToFind);
//Log the solution visually
Bitmap bitmapSolution = DrawSolution.Solution(rotatedImage, segmentation, solution, DRAW_WORDS_COLOUR);
log(bitmapSolution, "Solution");
log(DrawGrid.Segmentation(bitmapSolution, segmentation, DRAW_SEGMENTATION_COLOUR), "Solution + Segmentation");
//Log an Image for the solution to each word so that you can see where it thinks each word is
foreach(KeyValuePair<string, WordPosition> kvp in solution)
{
string word = kvp.Key;
WordPosition position = kvp.Value;
//Draw this WordPosition onto the rotated image
log(DrawSolution.WordPosition(rotatedImage, segmentation, position, DRAW_WORDS_COLOUR),
String.Format("Solution for Word: {0}", word));
}
//Mark the wordsearch as having been solved
setProcessingStageState(ProcessingStage.WordsearchSolver, CheckState.Checked);
//Log the time taken to complete all of the processing
setProcessingStageState(ProcessingStage.All, CheckState.Checked);
}
