Esempio n. 1
0
        internal static Dictionary<string, AlgorithmCombination> GetAlgorithmsToEvaluate()
        {
            SegmentationAlgorithm detectionSegmentationAlgorithm = new SegmentByMeanDarkPixels();
            SegmentationAlgorithm segmentationAlgorithm = new SegmentByBlobRecognition();

            //Use same classifier & feature extraction for both
            FeatureExtractionPCA featureExtraction = (FeatureExtractionPCA)TrainableFeatureExtractionAlgorithm.Load(
                Program.FEATURE_EXTRACTORS_PATH + Program.PCA_ALL_FEATURES_FILE_NAME + Program.FEATURE_EXTRACTORS_FILE_EXTENSION);

            Classifier classifier = new AForgeActivationNeuralNetClassifier(featureExtraction, PCA_ALL_FEATURES_NEURAL_NETWORK_CLASSIFIER_PATH);

            Solver wordsearchSolver = new SolverNonProbabilistic();
            Solver probabilisticWordsearchSolver = new SolverProbabilistic();

            Dictionary<string, AlgorithmCombination> algorithmsToEvaluate = new Dictionary<string, AlgorithmCombination>()
            {
                //Standard System
                { 
                    "Detection Segmentation: MeanDarkPixels, Segmentation: BlobRecognition, Segmentation Method: Fixed Width, Rotation Correction Classifier: Neural net with PCA (All Features), Classifier: Neural net with PCA (All Features), Wordsearch Solver: Non-Probabilistic",
                    new AlgorithmCombination(detectionSegmentationAlgorithm, false, segmentationAlgorithm, false, SegmentationMethod.FixedWidth, classifier, classifier, wordsearchSolver) 
                },

                //Varied width segmentation, Don't resize characters to constants size after segmentation
                {
                    "Detection Segmentation: MeanDarkPixels, Segmentation: BlobRecognition, Segmentation Method: Varied Width (No Resize), Rotation Correction Classifier: Neural net with PCA (All Features), Classifier: Neural net with PCA (All Features), Wordsearch Solver: Non-Probabilistic",
                    new AlgorithmCombination(detectionSegmentationAlgorithm, false, segmentationAlgorithm, false, SegmentationMethod.VariedWidthNoResize, classifier, classifier, wordsearchSolver)
                },

                //Varied width segmentation, resize characters to constant size after segmentation
                {
                    "Detection Segmentation: MeanDarkPixels, Segmentation: BlobRecognition, Segmentation Method: Varied Width (With Resize), Rotation Correction Classifier: Neural net with PCA (All Features), Classifier: Neural net with PCA (All Features), Wordsearch Solver: Non-Probabilistic",
                    new AlgorithmCombination(detectionSegmentationAlgorithm, false, segmentationAlgorithm, false, SegmentationMethod.VariedWidthWithResize, classifier, classifier, wordsearchSolver)
                },

                //Varied width segmentation (no resize) & probablistic solver
                {
                    "Detection Segmentation: MeanDarkPixels, Segmentation: BlobRecognition, Segmentation Method: Varied Width (No Resize), Rotation Correction Classifier: Neural net with PCA (All Features), Classifier: Neural net with PCA (All Features), Wordsearch Solver: Probabilistic",
                    new AlgorithmCombination(detectionSegmentationAlgorithm, false, segmentationAlgorithm, false, SegmentationMethod.VariedWidthNoResize, classifier, classifier, probabilisticWordsearchSolver)
                },

                //Varied width segmentation (no resize) & Probabilistic solver that prevents character discrepancies (when
                //  a position is used as one character in on one word, and another character in another word)
                {
                    "Detection Segmentation: MeanDarkPixels, Segmentation: BlobRecognition, Segmentation Method: Varied Width (No Resize), Rotation Correction Classifier: Neural net with PCA (All Features), Classifier: Neural net with PCA (All Features), Wordsearch Solver: Probabilistic Prevent Character Discrepancies",
                    new AlgorithmCombination(detectionSegmentationAlgorithm, false, segmentationAlgorithm, false, SegmentationMethod.VariedWidthNoResize, classifier, classifier, new SolverProbabilisticPreventCharacterDiscrepancies())
                },

                //Remove small rows and cols after segmentation, Varied width segmentation (no resize)
                //  & Probabilistic solver that prevents character discrepancies (when a position is used 
                //  as one character in on one word, and another character in another word)
                {
                    "Detection Segmentation: MeanDarkPixels, Segmentation: BlobRecognition (RemoveSmallRowsAndCols), Segmentation Method: Varied Width (No Resize), Rotation Correction Classifier: Neural net with PCA (All Features), Classifier: Neural net with PCA (All Features), Wordsearch Solver: Probabilistic Prevent Character Discrepancies",
                    new AlgorithmCombination(detectionSegmentationAlgorithm, false, segmentationAlgorithm, true, SegmentationMethod.VariedWidthNoResize, classifier, classifier, new SolverProbabilisticPreventCharacterDiscrepancies())
                }
            };

            return algorithmsToEvaluate;
        }
Esempio n. 2
0
        static void Main(string[] args)
        {
            //Initialise logging to the console &  a file
            Log consoleLog = new ConsoleLog(LOG_LEVEL);
            Log fileLog;
            bool dirCreated = false;
            if (!Directory.Exists(LOGS_DIR_PATH))
            {
                dirCreated = true;
                System.IO.Directory.CreateDirectory(LOGS_DIR_PATH);
            }

            int logAttempt = 0;
            while(true)
            {
                string logName = String.Format("{0}/Quantitative_Evaluation_DefaultLog.{1}.{2:000}.log", LOGS_DIR_PATH, DateTime.Now.ToString("yyyy-MM-dd"), logAttempt);
                if (!System.IO.File.Exists(logName))
                {
                    fileLog = new FileLog(logName, LOG_LEVEL);
                    DefaultLog.Info("Log Initialised");
                    break;
                }
                logAttempt++;
            }
            DefaultLog.Log = new CompoundLog(consoleLog, fileLog);
            if (dirCreated)
            {
                DefaultLog.Warn("Logs Directory was not found, creating . . .");
            }

            //If the directories for storing Classifiers don't exist, make them now
            if (!Directory.Exists(CLASSIFIERS_PATH))
            {
                DefaultLog.Info("Classifiers Directory didn't exist, creating . . .");
                Directory.CreateDirectory(CLASSIFIERS_PATH);
            }
            if (!Directory.Exists(NEURAL_NETWORKS_PATH))
            {
                DefaultLog.Info("Neural Networks Path didn't exist, creating . . .");
                Directory.CreateDirectory(NEURAL_NETWORKS_PATH);
            }

            //Load the Wordsearch Database
            DefaultLog.Info("Loading Wordsearch Database . . .");
            ImageMarkupDatabase.LoadDatabase();
            DefaultLog.Info("Wordsearch Database Loaded");

            //TODO: Change to some pre-determined split rather than splitting at runtime
            //Split the Wordsearch Images into 3 groups: training, cross-validation & evaluation
            DefaultLog.Info("Splitting Wordsearch Image data into Training, Cross-Validation & Evaluation data sets");
            List<WordsearchImage> wordsearchImages = ImageMarkupDatabase.GetWordsearchImages();

            List<WordsearchImage> trainingWordsearchImages = new List<WordsearchImage>();
            List<WordsearchImage> crossValidationWordsearchImages = new List<WordsearchImage>();
            List<WordsearchImage> evaluationWordsearchImages = new List<WordsearchImage>();

            //Split the images from 2014.02.20 into the three groups & add all of the 2014.05.18 images to the evaluation data set
            int oldImagesNum = 0;
            foreach(WordsearchImage wordsearchImage in wordsearchImages)
            {
                string imgPath = wordsearchImage.FromImage.Path;

                //If this image is in the old image directory (the one being split amongst the 3 data sets)
                if(imgPath.Substring(0, OLD_IMAGES_PATH.Length) == OLD_IMAGES_PATH)
                {
                    //Determine which data set to put the image in
                    if (oldImagesNum % 3 == 0)
                    {
                        trainingWordsearchImages.Add(wordsearchImage);
                    }
                    else if(oldImagesNum % 3 == 1)
                    {
                        crossValidationWordsearchImages.Add(wordsearchImage);
                    }
                    else
                    {
                        evaluationWordsearchImages.Add(wordsearchImage);
                    }
                    oldImagesNum++;
                }
                else //Otherwise this image in in the new image directory and should be put in the evaluation data set
                {
                    evaluationWordsearchImages.Add(wordsearchImage);
                }
            }

            DefaultLog.Info("Data split into Training, Cross-Validation & Evalutaion data");

            //If we're evaluating neural networks
            if (EVALUATE_NEURAL_NETWORKS)
            {
                //Evaluate all of the neural network combo's
                DefaultLog.Info("Starting to evaluate all Neural Networks . . .");
                IDictionary<string, NeuralNetworkEvaluator> neuralNetworkEvalResults =
                    EvaluateNeuralNetworks.evaluateNeuralNetworks(trainingWordsearchImages,
                    crossValidationWordsearchImages, evaluationWordsearchImages);
                DefaultLog.Info("Evaluation of all Neural Networks completed");

                //Write out the evaluation results
                if (!Directory.Exists(EVALUATION_RESULTS_DIR_PATH))
                {
                    DefaultLog.Info("Evaluation Results Directory didn't exist, creating . . .");
                    Directory.CreateDirectory(EVALUATION_RESULTS_DIR_PATH);
                }

                DefaultLog.Info("Writing out Neural Network Evaluation Results . . .");
                foreach (KeyValuePair<string, NeuralNetworkEvaluator> pair in neuralNetworkEvalResults)
                {
                    string networkName = pair.Key;
                    ConfusionMatrix cm = pair.Value.ConfusionMatrix;

                    DefaultLog.Info("Network \"{0}\" misclassified {1}/{2}", networkName,
                        cm.NumMisclassifications, cm.TotalClassifications);

                    try
                    {
                        cm.WriteToCsv(EVALUATION_RESULTS_DIR_PATH + "/" + networkName + ".csv");
                    }
                    catch (Exception e)
                    {
                        DefaultLog.Error("Error writing Confusion Matrix to file " + EVALUATION_RESULTS_DIR_PATH + "/" + networkName + ".csv");
                        Console.WriteLine(e);
                    }
                }
                DefaultLog.Info("Neural Network Evaluation results written out successfully");
            }
            
            //If we're evaluating Wordsearch rotation correction
            if(EVALUATE_WORDSEARCH_ROTATION_CORRECTION)
            {
                DefaultLog.Info("Starting to evaluate Wordsearch Rotation Correction");

                //Get the Feature Reduction Algorithm to be used
                DefaultLog.Info("Loading Feature Extraction Algorithm . . .");
                //Load a pre-trained feature reduction algorithm rather than training on the training data every time
                FeatureExtractionPCA featureExtractionAlgorithm = (FeatureExtractionPCA)TrainableFeatureExtractionAlgorithm.Load(
                    FEATURE_EXTRACTORS_PATH + PCA_ALL_FEATURES_FILE_NAME + FEATURE_EXTRACTORS_FILE_EXTENSION);
                DefaultLog.Info("Feature Extraction Algorithm Loaded");

                //Get the classifier to be used
                DefaultLog.Info("Loading Classifier . . .");
                Classifier classifier = new AForgeActivationNeuralNetClassifier(featureExtractionAlgorithm, 
                    NEURAL_NETWORKS_PATH + "SingleLayer Sigmoid BkPropLearn PCAAllFeatures" + NEURAL_NETWORK_FILE_EXTENSION);
                DefaultLog.Info("Classifier Loaded");

                //Evaluate the wordsearch Image Rotation Correction
                double rotationCorrectionRate = EvaluateWordsearchRotationCorrection.Evaluate(evaluationWordsearchImages, classifier);
                DefaultLog.Info("Wordsearch Rotation Correction returned the correct answer {0}% of the time", rotationCorrectionRate * 100);

                DefaultLog.Info("Wordsearch Rotation Correction Evaluation complete");
            }

            //If we're evaluating Wordsearch Segmentation
            if(EVALUATE_WORDSEARCH_SEGMENTATION)
            {
                /*
                 * Note that here all Wordsearch Images are used for evaluation (essentially they are all in the evaluation data set)
                 * This is because the training & cross validation data haven't been used to develop the algorithms, meaning that they 
                 * are fresh data and can be used for evaluation
                 */

                DefaultLog.Info("Starting to evaluate Wordsearch Segmentation");

                //Evaluate by the number of rows and cols returned
                Dictionary<string, double> scoresByNumRowsCols = EvaluateWordsearchSegmentation.EvaluateByNumRowsAndCols(wordsearchImages);

                //Print out scores
                DefaultLog.Info("Scores for Evaluation by Number of Rows and Cols");
                printScores(scoresByNumRowsCols);

                DefaultLog.Info("Wordsearch Segmentation Evaluation complete");
            }

            //If we're evaluating Wordsearch Recognition
            if(EVALUATE_WORDSEARCH_DETECTION)
            {
                /*
                 * Note that here all Images are used for evaluation (essentially they are all in the evaluation data set)
                 * This is because the training & cross validation data haven't been used to develop the algorithms, meaning
                 * that they are fresh data and can be used for evaluation
                 */

                DefaultLog.Info("Starting to evaluate Wordsearch Recognition");

                Dictionary<string, double> scores = EvaluateWordsearchDetection.EvaluateReturnsWordsearch(ImageMarkupDatabase.GetImages());

                //Print out scores
                DefaultLog.Info("Scores for Evaluation based on a single wordsearch returned (the best candidate)");
                printScores(scores);

                DefaultLog.Info("Wordsearch Recognition Evaluation Complete");
            }

            //If we're evaluating the Full System
            if(EVALUATE_FULL_SYSTEM)
            {
                /*
                 * This evaluation stage uses Images rather than WordsearchImages, and the Images used must not contain any WordsearchImages 
                 * from the training or cross-validation data sets
                 */

                DefaultLog.Info("Building the collection of Evaluation Images . . .");

                //Compile a list of the hashes of the Images that contain each WordsearchImage in the training & cross-validation data sets
                HashSet<string> usedImageHashes = new HashSet<string>();
                List<WordsearchImage> usedWordsearchImages = new List<WordsearchImage>(trainingWordsearchImages);
                usedWordsearchImages.AddRange(crossValidationWordsearchImages);
                foreach(WordsearchImage wordsearchImage in usedWordsearchImages)
                {
                    usedImageHashes.Add(wordsearchImage.FromImageHash);
                }

                //Now build a list of Images whose hash aren't present in the set of used hashes
                List<Image> allImages = ImageMarkupDatabase.GetImages();
                List<Image> evaluationImages = new List<Image>();
                foreach(Image image in allImages)
                {
                    if(!usedImageHashes.Contains(image.Hash))
                    {
                        evaluationImages.Add(image);
                    }
                }

                DefaultLog.Info("Collection of Evaluation Images built");

                DefaultLog.Info("Starting to Evaluate the Full System");

                Dictionary<string, double> scores = EvaluateFullSystem.Evaluate(evaluationImages);

                //Print out scores
                DefaultLog.Info("Scores for Evaluation of the Full System");
                printScores(scores);

                DefaultLog.Info("Full System Evaluation Complete");
            }

            //If we're evaluating the stages after Wordsearch Detection (Segmentation to Solver)
            if(EVALUATE_STAGES_SEGMENTATION_TO_SOLVER)
            {
                DefaultLog.Info("Starting to Evaluate the stages from Segmentation until Solving");

                Dictionary<string, double> scores = EvaluateSegmentationToSolver.Evaluate(evaluationWordsearchImages);

                //Print out scores
                DefaultLog.Info("Scores for Evaluation of the stages Segmentation to Solver");
                printScores(scores);

                DefaultLog.Info("Evaluation of stages Segmentation to Solving complete");
            }
        }
Esempio n. 3
0
        //Get the Classifier object. This encapsulates the classifier itself & the Feature Extractor,
        //  so we pass two parent menu items, one for each
        private Classifier getSelectedClassifier(ToolStripMenuItem parentFeatureExtractor, ToolStripMenuItem parentClassifier)
        {
            //There is only one classifier at the minute, so just always use it
            string classifierName = "Single Layer Neural Network";

            //Get the Feature Extraction Algorithm
            FeatureExtractionAlgorithm featureExtractor = getSelectedFeatureExtractionAlgorithm(parentFeatureExtractor);

            //Work out where to load the Neural Network from. There are different trained versions of the network
            //  for each different Feature Extraction Algorithm

            //Get the name of the Feature Extractor (don't need to handle case of it not being found as the above call to
            //  getSelectedFeatureExtractionAlgorithm will have already thrown an Exception if no menu item was checked)
            string featureExtractorName = null;
            foreach(ToolStripMenuItem menuItem in parentFeatureExtractor.DropDownItems)
            {
                if(menuItem.Checked)
                {
                    featureExtractorName = menuItem.Text;
                    break;
                }
            }

            //Get the file names for the Feature Extractor & Classifier
            string featureExtractorFileName = TRAINED_FEATURE_EXTRACTOR_FILE_NAMES[featureExtractorName];
            string classifierFileName = TRAINED_CLASSIFIER_FILE_NAMES[classifierName];

            //Combine the feature extractor & classifier file names into one which is what the classifier to be loaded is actually called
            string fileName = classifierFileName + "_" + featureExtractorFileName;

            //Construct the full path for the file
            string filePath = TRAINED_CLASSIFIERS_PATH + fileName + TRAINED_CLASSIFIER_FILE_EXTENSION;

            //Construct the Classifier object (which will load the classifier from the specified file)
            Classifier classifier = new AForgeActivationNeuralNetClassifier(featureExtractor, filePath);

            return classifier;
        }