コード例 #1
0
        private static string getStringFromLevelCodeImage(Image <Bgr, byte> image)
        {
            Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image);

            ocrReadyImage = cropLineOfText(ocrReadyImage, new Size(37, 3));
            List <Mat> characters = segmentCharacters(ocrReadyImage);

            // 11 characters expected in a level code (XXX-XXX-XXX)
            if (characters.Count == 11)
            {
                // Remove the dashes
                characters.RemoveAt(7);
                characters.RemoveAt(3);

                string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY");

                // format code
                levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}";

                return(levelCode);
            }
            else
            {
                // fallback to original OCR if segmentation has unexpected number of characters
                log.Debug($"segmentChacters - level code - detected {characters.Count} characters, falling back to original OCR method");
                return(doOCROnLevelCodeImage(ocrReadyImage));
            }
        }
コード例 #2
0
ファイル: Form1.cs プロジェクト: underscore-zi/MarioMaker2OCR
        private void startButton_Click(object sender, EventArgs e)
        {
            if (deviceComboBox.SelectedItem == null)
            {
                MessageBox.Show("Please select a video device first.");
                return;
            }
            try
            {
                // resize reference image based on current resolution
                levelSelectScreen = ImageLibrary.ChangeSize(levelSelectScreen720, resolution720, SelectedResolution);

                // resize rectangles based on current resolution
                levelCodeArea   = ImageLibrary.ChangeSize(levelCodeArea720p, resolution720, SelectedResolution);
                creatorNameArea = ImageLibrary.ChangeSize(creatorNameArea720, resolution720, SelectedResolution);
                levelTitleArea  = ImageLibrary.ChangeSize(levelTitleArea720, resolution720, SelectedResolution);

                SMMServer.port = Decimal.ToUInt16(numPort.Value);
                log.Info(String.Format("Start Web Server on http://localhost:{0}/", SMMServer.port));
                SMMServer.Start();

                processor              = new VideoProcessor(deviceComboBox.SelectedIndex, SelectedResolution);
                processor.BlackScreen += VideoProcessor_BlackScreen;
                processor.ClearScreen += VideoProcessor_ClearScreen;
                processor.NewFrame    += VideoProcessor_NewFrame;
                processor.Start();
                lockForm();
            }
            catch (Exception ex)
            {
                processException("Error starting video device", ex);
            }
        }
コード例 #3
0
 private static string getStringFromImage(Image <Bgr, byte> image)
 {
     using (Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image))
     {
         return(doOCROnImage(ocrReadyImage));
     }
 }
コード例 #4
0
 private static void updateLevelBoundryResolutions(Size newResolution)
 {
     // only update if a new resolution is detected
     if (newResolution.Height != levelCodeBoundry.Height)
     {
         levelCodeBoundry   = ImageLibrary.ChangeSize(levelCodeBoundry720, resolution720, newResolution);
         creatorNameBoundry = ImageLibrary.ChangeSize(creatorNameBoundry720, resolution720, newResolution);
         levelTitleBoundry  = ImageLibrary.ChangeSize(levelTitleBoundry720, resolution720, newResolution);
     }
 }
コード例 #5
0
ファイル: OCRLibrary.cs プロジェクト: Vukkk/MarioMaker2OCR
        /// <summary>
        /// Read clear time from frame
        /// </summary>
        /// <param name="frame"></param>
        /// <param name="commentsEnabled">If true, then use a different screen region to scan for time</param>
        /// <returns></returns>
        internal static string GetClearTimeFromFrame(Image <Bgr, byte> frame, bool commentsEnabled)
        {
            // Update boundry sizes relative to the current resolution
            int idx = commentsEnabled ? 1 : 0;

            clearTimeBoundry = ImageLibrary.ChangeSize(clearTimeBoundry720[idx], resolution720, frame.Size);

            // Set ROI
            frame.ROI = clearTimeBoundry;

            // Prepare Image for OCR
            Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(frame);

            // Segment characters
            List <Mat> characters = segmentCharacters(ocrReadyImage);

            frame.ROI = Rectangle.Empty;

            // expect time to be 9 characters, quote reads as 2 chars (ex: 01'34''789)
            if (characters.Count == 10)
            {
                // Different Time Formats
                // 12:34,567 | 12:34.567 | 12'34"567

                // Remove punctuation - can identify by height
                // Make relative to max character size incase we ever resize images.
                int maxCharHeight = characters.Max(p => p.Height);
                int minimumHeight = (int)Math.Floor(maxCharHeight * .60);
                characters.RemoveAll(p => p.Height < minimumHeight);

                // Do OCR
                string clearTime = doOCROnCharacterImages(characters, "0123456789");

                // format clear time
                try
                {
                    if (clearTime.Length >= 7)
                    {
                        clearTime = $"{clearTime.Substring(0, 2)}'{clearTime.Substring(2, 2)}\"{clearTime.Substring(4, 3)}";
                    }
                }
                catch (ArgumentOutOfRangeException ex)
                {
                    log.Error($"ArgumentOutOfRangeException formatting clear time: {clearTime}");
                    clearTime = "";
                }
                return(clearTime);
            }
            else
            {
                log.Debug($"segmentChacters for time - {characters.Count} characters detected, expected 10, falling back to original OCR method");
                return(doOCROnImage(ocrReadyImage));
            }
        }
コード例 #6
0
ファイル: Form1.cs プロジェクト: CeBkCn/MarioMaker2OCR
        /// <summary>
        /// Event callback for the Clear Screen event generatead by the VideoProcessor
        /// </summary>
        private void VideoProcessor_ClearScreen(object sender, VideoProcessor.ClearScreenEventArgs e)
        {
            log.Debug("Detected Level Clear");

            Image <Gray, byte> grayscaleFrame = e.currentFrame.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic);
            //e.currentFrame.Save("clearmatch_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png");

            Dictionary <String, bool> events = new Dictionary <String, bool>
            {
                { "worldrecord", false },
                { "firstclear", false },
            };
            List <Rectangle> boundaries = new List <Rectangle>();

            foreach (EventTemplate tmpl in clearTemplates)
            {
                if (events[tmpl.eventType])
                {
                    continue;
                }
                Point loc = tmpl.getLocation(grayscaleFrame);
                if (!loc.IsEmpty)
                {
                    events[tmpl.eventType] = true;
                    boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, e.currentFrame.Size));
                    previewer.SetLastMatch(e.currentFrame, boundaries.ToArray());
                }
            }

            foreach (var evt in events)
            {
                if (evt.Value)
                {
                    log.Info(String.Format("Detected {0}.", evt.Key));
                    SMMServer.BroadcastEvent(evt.Key);
                }
            }

            if (Properties.Settings.Default.WarpWorldEnabled)
            {
                WarpWorld?.win();
            }

            // Read time from screen
            string clearTime = OCRLibrary.GetClearTimeFromFrame(e.currentFrame, e.commentsEnabled);

            SMMServer.BroadcastDataEvent("clear", clearTime);
        }
コード例 #7
0
ファイル: Form1.cs プロジェクト: CeBkCn/MarioMaker2OCR
        private void startButton_Click(object sender, EventArgs e)
        {
            if (deviceComboBox.SelectedItem == null)
            {
                MessageBox.Show("Please select a video device first.");
                return;
            }
            try
            {
                templates.Clear();
                templates.AddRange(engTemplates);

                // Add language neutral templates if selected.
                if (langNeutralcheckBox.Checked)
                {
                    templates.AddRange(langNeutralTemplates);
                }

                // resize reference image based on current resolution
                levelDetailScreen = ImageLibrary.ChangeSize(levelSelectScreen720, resolution720, SelectedResolution);

                SMMServer.port = decimal.ToUInt16(numPort.Value);
                log.Info(string.Format("Start Web Server on http://localhost:{0}/", SMMServer.port));
                SMMServer.Start();

                if (Properties.Settings.Default.WarpWorldEnabled)
                {
                    WarpWorld = new WarpWorldAPI(Properties.Settings.Default.WarpWorldUsername, Properties.Settings.Default.WarpWorldToken);
                }
                else
                {
                    WarpWorld = null;
                }

                processor              = new VideoProcessor(deviceComboBox.SelectedIndex, SelectedResolution);
                processor.BlackScreen += VideoProcessor_BlackScreen;
                processor.ClearScreen += VideoProcessor_ClearScreen;
                processor.NewFrame    += VideoProcessor_NewFrame;
                processor.Start();
                lockForm();
            }
            catch (Exception ex)
            {
                processException("Error starting video device", ex);
            }
        }
コード例 #8
0
ファイル: Form1.cs プロジェクト: CeBkCn/MarioMaker2OCR
        /// <summary>
        /// Event Callback for the Black Screen event generated by the VideoProcessor
        /// </summary>
        private void VideoProcessor_BlackScreen(object sender, VideoProcessor.BlackScreenEventArgs e)
        {
            log.Debug(String.Format("Detected a black screen [{0}]", e.seconds));
            BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing black screen..."));

            double imageMatchPercent = ImageLibrary.CompareImages(e.currentFrame, levelDetailScreen);

            // Is this frame a 90% match to a level screen?
            if (imageMatchPercent > 0.90)
            {
                log.Info(String.Format("Detected new level. [{0}]", e.seconds));

                BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing level screen..."));

                Level level = OCRLibrary.GetLevelFromFrame(e.currentFrame);
                writeLevelToFile(level);
                SMMServer.BroadcastLevel(level);

                BeginInvoke((MethodInvoker)(() => ocrTextBox.Text = level.code + "  |  " + level.author + "  |  " + level.name));
                BeginInvoke((MethodInvoker)(() => processingLabel.Text = ""));

                previewer.SetLastMatch(e.currentFrame);
            }
            else
            {
                // Not a new level, see if we can detect a template.
                Dictionary <String, bool> events = new Dictionary <String, bool>
                {
                    { "death", false },
                    { "restart", false },
                    { "exit", false },
                    { "gameover", false },
                    { "skip", false }
                };

                BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing events..."));
                foreach (Image <Bgr, byte> f in e.frameBuffer)
                {
                    // Skip any empty frames in the buffer
                    if (f == null)
                    {
                        continue;
                    }

                    Image <Gray, byte> grayscaleFrame = f.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic);
                    //grayscaleFrame.Save("frame_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); // XXX: Useful for debugging template false-negatives, and for getting templates

                    List <Rectangle> boundaries = new List <Rectangle>();
                    foreach (EventTemplate tmpl in templates)
                    {
                        if (events[tmpl.eventType])
                        {
                            continue;
                        }
                        Point loc = tmpl.getLocation(grayscaleFrame);
                        if (!loc.IsEmpty)
                        {
                            events[tmpl.eventType] = true;
                            boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, f.Size));
                            previewer.SetLastMatch(f, boundaries.ToArray());
                        }
                    }
                }
                BeginInvoke((MethodInvoker)(() => processingLabel.Text = ""));

                foreach (var evt in events)
                {
                    if (evt.Value)
                    {
                        log.Info(String.Format("Detected {0} [{1}].", evt.Key, e.seconds));
                        SMMServer.BroadcastEvent(evt.Key);

                        // Even though this will get sent on exiting after a clear, it only matters if a entry is active, and after marking it as a win it goes inactive.
                        if (evt.Key == "exit")
                        {
                            if (Properties.Settings.Default.WarpWorldEnabled)
                            {
                                WarpWorld?.lose();
                            }
                            if (JsonSettings.ClearOnExit)
                            {
                                clearJsonFile();
                            }
                        }
                        if (evt.Key == "skip" && JsonSettings.ClearOnSkip)
                        {
                            clearJsonFile();
                        }
                        if (evt.Key == "gameover" && JsonSettings.ClearOnGameover)
                        {
                            clearJsonFile();
                        }
                    }
                }
            }
        }
コード例 #9
0
ファイル: OCRLibrary.cs プロジェクト: Vukkk/MarioMaker2OCR
        private static string getStringFromLevelCodeImage(Image <Bgr, byte> image)
        {
            Image <Gray, byte> ocrReadyImage = null;
            Image <Gray, byte> croppedImage  = null;

            try
            {
                ocrReadyImage = ImageLibrary.PrepareImageForOCR(image);
                croppedImage  = cropLineOfText(ocrReadyImage, new Size(37, 3));
                List <Mat> characters = segmentCharacters(croppedImage);

                // 11 characters expected in a level code (XXX-XXX-XXX)
                if (characters.Count == 11)
                {
                    // Remove the dashes
                    characters.RemoveAt(7);
                    characters.RemoveAt(3);

                    string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY");

                    // format code
                    levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}";

                    return(levelCode);
                }
                else
                {
                    // before falling back to line based OCR, take one more shot at saving this code with segmenting characters.
                    // This is an extremely crude method which will split character images in half if they are much wider than
                    // median char width. This is needed for adjacent "TT" characters in a level code.
                    if (characters.Count < 11 && characters.Count > 7)
                    {
                        log.Debug($"getStringFromLevelCodeImage - level code - detected {characters.Count} characters, manually splitting larger characters");

                        // Remove dashes - can identify by height
                        int maxCharHeight = characters.Max(p => p.Height);
                        int minimumHeight = (int)Math.Floor(maxCharHeight * .60);
                        characters.RemoveAll(p => p.Height < minimumHeight);

                        double medianCharacterWidth = characters.OrderBy(p => p.Width).Skip(characters.Count / 2).Take(1).FirstOrDefault().Width;

                        for (int i = 0; i < characters.Count; i++)
                        {
                            var character = characters[i];

                            double perc = medianCharacterWidth / character.Width;

                            // Current letter is significantly larger than median width
                            if (perc < .65)
                            {
                                Rectangle leftRect  = new Rectangle(0, 0, character.Width / 2, character.Height);
                                Rectangle rightRect = new Rectangle(character.Width / 2, 0, character.Width / 2, character.Height);

                                // Left character
                                Mat leftCharacter = new Mat(character, leftRect);
                                // Right character
                                characters[i] = new Mat(character, rightRect);

                                // Insert left character into array
                                characters.Insert(i, leftCharacter);
                            }

                            // If there are 9 characters then we are done
                            if (characters.Count == 9)
                            {
                                break;
                            }
                        }

                        string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY");
                        levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}";
                        return(levelCode);
                    }

                    // fallback to original OCR if segmentation has unexpected number of characters
                    log.Debug($"segmentChacters - level code - detected {characters.Count} characters, falling back to original OCR method");
                    return(doOCROnLevelCodeImage(ocrReadyImage));
                }
            }
            catch (Exception ex)
            {
                log.Error("Exception in getStringFromLevelCodeImage - " + ex.Message);
                return("");
            }
            finally
            {
                ocrReadyImage?.Dispose();
                croppedImage?.Dispose();
            }
        }
コード例 #10
0
ファイル: Form1.cs プロジェクト: underscore-zi/MarioMaker2OCR
        /// <summary>
        /// Event Callback for the Black Screen event generated by the VideoProcessor
        /// </summary>
        private void VideoProcessor_BlackScreen(object sender, VideoProcessor.VideoEventArgs e)
        {
            log.Debug("Detected a black screen");
            BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing black screen..."));

            // Is this a new level?
            Image <Bgr, byte> frame             = e.frameBuffer[e.frameBuffer.Length - 5];
            double            imageMatchPercent = ImageLibrary.CompareImages(frame, levelSelectScreen);

            if (imageMatchPercent > 0.94)
            {
                log.Info("Detected new level.");
                previewer.SetLastMatch(frame, new Rectangle[] { levelCodeArea, creatorNameArea, levelTitleArea });
                BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing level screen..."));

                Level level = getLevelFromCurrentFrame(frame);
                writeLevelToFile(level);
                SMMServer.BroadcastLevel(level);

                BeginInvoke((MethodInvoker)(() => ocrTextBox.Text = level.code + "  |  " + level.author + "  |  " + level.name));
                BeginInvoke((MethodInvoker)(() => processingLabel.Text = ""));
            }
            else
            {
                // Not a new level, see if we can detect a template.
                Dictionary <String, bool> events = new Dictionary <String, bool>
                {
                    { "death", false },
                    { "restart", false },
                    { "exit", false },
                };


                BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing events..."));
                foreach (Image <Bgr, byte> f in e.frameBuffer)
                {
                    Image <Gray, byte> grayscaleFrame = f.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic);
                    //grayscaleFrame.Save("frame_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); // XXX: Useful for debugging template false-negatives, and for getting templates

                    List <Rectangle> boundaries = new List <Rectangle>();
                    foreach (EventTemplate tmpl in templates)
                    {
                        if (events[tmpl.eventType])
                        {
                            continue;
                        }
                        Point loc = tmpl.getLocation(grayscaleFrame);
                        if (!loc.IsEmpty)
                        {
                            events[tmpl.eventType] = true;
                            boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, f.Size));
                            previewer.SetLastMatch(f, boundaries.ToArray());
                        }
                    }
                }
                BeginInvoke((MethodInvoker)(() => processingLabel.Text = ""));

                foreach (var evt in events)
                {
                    if (evt.Value)
                    {
                        log.Info(String.Format("Detected {0}.", evt.Key));
                        SMMServer.BroadcastEvent(evt.Key);
                    }
                }
            }
        }
コード例 #11
0
ファイル: Form1.cs プロジェクト: underscore-zi/MarioMaker2OCR
        private static string getStringFromLevelCodeImage(Image <Bgr, byte> image)
        {
            Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image);

            return(OCRLibrary.GetStringFromLevelCodeImage(ocrReadyImage));
        }
コード例 #12
0
        private void previewMatch(object sender, VideoProcessor.TemplateMatchEventArgs e)
        {
            var boundary = ImageLibrary.ChangeSize(new Rectangle(e.location, e.template.size), processor.TEMPLATE_FRAME_SIZE, processor.frameSize);

            previewer.SetLastMatch(e.frame, new Rectangle[] { boundary });
        }