private static string getStringFromLevelCodeImage(Image <Bgr, byte> image) { Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image); ocrReadyImage = cropLineOfText(ocrReadyImage, new Size(37, 3)); List <Mat> characters = segmentCharacters(ocrReadyImage); // 11 characters expected in a level code (XXX-XXX-XXX) if (characters.Count == 11) { // Remove the dashes characters.RemoveAt(7); characters.RemoveAt(3); string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY"); // format code levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}"; return(levelCode); } else { // fallback to original OCR if segmentation has unexpected number of characters log.Debug($"segmentChacters - level code - detected {characters.Count} characters, falling back to original OCR method"); return(doOCROnLevelCodeImage(ocrReadyImage)); } }
private void startButton_Click(object sender, EventArgs e) { if (deviceComboBox.SelectedItem == null) { MessageBox.Show("Please select a video device first."); return; } try { // resize reference image based on current resolution levelSelectScreen = ImageLibrary.ChangeSize(levelSelectScreen720, resolution720, SelectedResolution); // resize rectangles based on current resolution levelCodeArea = ImageLibrary.ChangeSize(levelCodeArea720p, resolution720, SelectedResolution); creatorNameArea = ImageLibrary.ChangeSize(creatorNameArea720, resolution720, SelectedResolution); levelTitleArea = ImageLibrary.ChangeSize(levelTitleArea720, resolution720, SelectedResolution); SMMServer.port = Decimal.ToUInt16(numPort.Value); log.Info(String.Format("Start Web Server on http://localhost:{0}/", SMMServer.port)); SMMServer.Start(); processor = new VideoProcessor(deviceComboBox.SelectedIndex, SelectedResolution); processor.BlackScreen += VideoProcessor_BlackScreen; processor.ClearScreen += VideoProcessor_ClearScreen; processor.NewFrame += VideoProcessor_NewFrame; processor.Start(); lockForm(); } catch (Exception ex) { processException("Error starting video device", ex); } }
private static string getStringFromImage(Image <Bgr, byte> image) { using (Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image)) { return(doOCROnImage(ocrReadyImage)); } }
private static void updateLevelBoundryResolutions(Size newResolution) { // only update if a new resolution is detected if (newResolution.Height != levelCodeBoundry.Height) { levelCodeBoundry = ImageLibrary.ChangeSize(levelCodeBoundry720, resolution720, newResolution); creatorNameBoundry = ImageLibrary.ChangeSize(creatorNameBoundry720, resolution720, newResolution); levelTitleBoundry = ImageLibrary.ChangeSize(levelTitleBoundry720, resolution720, newResolution); } }
/// <summary> /// Read clear time from frame /// </summary> /// <param name="frame"></param> /// <param name="commentsEnabled">If true, then use a different screen region to scan for time</param> /// <returns></returns> internal static string GetClearTimeFromFrame(Image <Bgr, byte> frame, bool commentsEnabled) { // Update boundry sizes relative to the current resolution int idx = commentsEnabled ? 1 : 0; clearTimeBoundry = ImageLibrary.ChangeSize(clearTimeBoundry720[idx], resolution720, frame.Size); // Set ROI frame.ROI = clearTimeBoundry; // Prepare Image for OCR Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(frame); // Segment characters List <Mat> characters = segmentCharacters(ocrReadyImage); frame.ROI = Rectangle.Empty; // expect time to be 9 characters, quote reads as 2 chars (ex: 01'34''789) if (characters.Count == 10) { // Different Time Formats // 12:34,567 | 12:34.567 | 12'34"567 // Remove punctuation - can identify by height // Make relative to max character size incase we ever resize images. int maxCharHeight = characters.Max(p => p.Height); int minimumHeight = (int)Math.Floor(maxCharHeight * .60); characters.RemoveAll(p => p.Height < minimumHeight); // Do OCR string clearTime = doOCROnCharacterImages(characters, "0123456789"); // format clear time try { if (clearTime.Length >= 7) { clearTime = $"{clearTime.Substring(0, 2)}'{clearTime.Substring(2, 2)}\"{clearTime.Substring(4, 3)}"; } } catch (ArgumentOutOfRangeException ex) { log.Error($"ArgumentOutOfRangeException formatting clear time: {clearTime}"); clearTime = ""; } return(clearTime); } else { log.Debug($"segmentChacters for time - {characters.Count} characters detected, expected 10, falling back to original OCR method"); return(doOCROnImage(ocrReadyImage)); } }
/// <summary> /// Event callback for the Clear Screen event generatead by the VideoProcessor /// </summary> private void VideoProcessor_ClearScreen(object sender, VideoProcessor.ClearScreenEventArgs e) { log.Debug("Detected Level Clear"); Image <Gray, byte> grayscaleFrame = e.currentFrame.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic); //e.currentFrame.Save("clearmatch_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); Dictionary <String, bool> events = new Dictionary <String, bool> { { "worldrecord", false }, { "firstclear", false }, }; List <Rectangle> boundaries = new List <Rectangle>(); foreach (EventTemplate tmpl in clearTemplates) { if (events[tmpl.eventType]) { continue; } Point loc = tmpl.getLocation(grayscaleFrame); if (!loc.IsEmpty) { events[tmpl.eventType] = true; boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, e.currentFrame.Size)); previewer.SetLastMatch(e.currentFrame, boundaries.ToArray()); } } foreach (var evt in events) { if (evt.Value) { log.Info(String.Format("Detected {0}.", evt.Key)); SMMServer.BroadcastEvent(evt.Key); } } if (Properties.Settings.Default.WarpWorldEnabled) { WarpWorld?.win(); } // Read time from screen string clearTime = OCRLibrary.GetClearTimeFromFrame(e.currentFrame, e.commentsEnabled); SMMServer.BroadcastDataEvent("clear", clearTime); }
private void startButton_Click(object sender, EventArgs e) { if (deviceComboBox.SelectedItem == null) { MessageBox.Show("Please select a video device first."); return; } try { templates.Clear(); templates.AddRange(engTemplates); // Add language neutral templates if selected. if (langNeutralcheckBox.Checked) { templates.AddRange(langNeutralTemplates); } // resize reference image based on current resolution levelDetailScreen = ImageLibrary.ChangeSize(levelSelectScreen720, resolution720, SelectedResolution); SMMServer.port = decimal.ToUInt16(numPort.Value); log.Info(string.Format("Start Web Server on http://localhost:{0}/", SMMServer.port)); SMMServer.Start(); if (Properties.Settings.Default.WarpWorldEnabled) { WarpWorld = new WarpWorldAPI(Properties.Settings.Default.WarpWorldUsername, Properties.Settings.Default.WarpWorldToken); } else { WarpWorld = null; } processor = new VideoProcessor(deviceComboBox.SelectedIndex, SelectedResolution); processor.BlackScreen += VideoProcessor_BlackScreen; processor.ClearScreen += VideoProcessor_ClearScreen; processor.NewFrame += VideoProcessor_NewFrame; processor.Start(); lockForm(); } catch (Exception ex) { processException("Error starting video device", ex); } }
/// <summary> /// Event Callback for the Black Screen event generated by the VideoProcessor /// </summary> private void VideoProcessor_BlackScreen(object sender, VideoProcessor.BlackScreenEventArgs e) { log.Debug(String.Format("Detected a black screen [{0}]", e.seconds)); BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing black screen...")); double imageMatchPercent = ImageLibrary.CompareImages(e.currentFrame, levelDetailScreen); // Is this frame a 90% match to a level screen? if (imageMatchPercent > 0.90) { log.Info(String.Format("Detected new level. [{0}]", e.seconds)); BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing level screen...")); Level level = OCRLibrary.GetLevelFromFrame(e.currentFrame); writeLevelToFile(level); SMMServer.BroadcastLevel(level); BeginInvoke((MethodInvoker)(() => ocrTextBox.Text = level.code + " | " + level.author + " | " + level.name)); BeginInvoke((MethodInvoker)(() => processingLabel.Text = "")); previewer.SetLastMatch(e.currentFrame); } else { // Not a new level, see if we can detect a template. Dictionary <String, bool> events = new Dictionary <String, bool> { { "death", false }, { "restart", false }, { "exit", false }, { "gameover", false }, { "skip", false } }; BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing events...")); foreach (Image <Bgr, byte> f in e.frameBuffer) { // Skip any empty frames in the buffer if (f == null) { continue; } Image <Gray, byte> grayscaleFrame = f.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic); //grayscaleFrame.Save("frame_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); // XXX: Useful for debugging template false-negatives, and for getting templates List <Rectangle> boundaries = new List <Rectangle>(); foreach (EventTemplate tmpl in templates) { if (events[tmpl.eventType]) { continue; } Point loc = tmpl.getLocation(grayscaleFrame); if (!loc.IsEmpty) { events[tmpl.eventType] = true; boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, f.Size)); previewer.SetLastMatch(f, boundaries.ToArray()); } } } BeginInvoke((MethodInvoker)(() => processingLabel.Text = "")); foreach (var evt in events) { if (evt.Value) { log.Info(String.Format("Detected {0} [{1}].", evt.Key, e.seconds)); SMMServer.BroadcastEvent(evt.Key); // Even though this will get sent on exiting after a clear, it only matters if a entry is active, and after marking it as a win it goes inactive. if (evt.Key == "exit") { if (Properties.Settings.Default.WarpWorldEnabled) { WarpWorld?.lose(); } if (JsonSettings.ClearOnExit) { clearJsonFile(); } } if (evt.Key == "skip" && JsonSettings.ClearOnSkip) { clearJsonFile(); } if (evt.Key == "gameover" && JsonSettings.ClearOnGameover) { clearJsonFile(); } } } } }
private static string getStringFromLevelCodeImage(Image <Bgr, byte> image) { Image <Gray, byte> ocrReadyImage = null; Image <Gray, byte> croppedImage = null; try { ocrReadyImage = ImageLibrary.PrepareImageForOCR(image); croppedImage = cropLineOfText(ocrReadyImage, new Size(37, 3)); List <Mat> characters = segmentCharacters(croppedImage); // 11 characters expected in a level code (XXX-XXX-XXX) if (characters.Count == 11) { // Remove the dashes characters.RemoveAt(7); characters.RemoveAt(3); string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY"); // format code levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}"; return(levelCode); } else { // before falling back to line based OCR, take one more shot at saving this code with segmenting characters. // This is an extremely crude method which will split character images in half if they are much wider than // median char width. This is needed for adjacent "TT" characters in a level code. if (characters.Count < 11 && characters.Count > 7) { log.Debug($"getStringFromLevelCodeImage - level code - detected {characters.Count} characters, manually splitting larger characters"); // Remove dashes - can identify by height int maxCharHeight = characters.Max(p => p.Height); int minimumHeight = (int)Math.Floor(maxCharHeight * .60); characters.RemoveAll(p => p.Height < minimumHeight); double medianCharacterWidth = characters.OrderBy(p => p.Width).Skip(characters.Count / 2).Take(1).FirstOrDefault().Width; for (int i = 0; i < characters.Count; i++) { var character = characters[i]; double perc = medianCharacterWidth / character.Width; // Current letter is significantly larger than median width if (perc < .65) { Rectangle leftRect = new Rectangle(0, 0, character.Width / 2, character.Height); Rectangle rightRect = new Rectangle(character.Width / 2, 0, character.Width / 2, character.Height); // Left character Mat leftCharacter = new Mat(character, leftRect); // Right character characters[i] = new Mat(character, rightRect); // Insert left character into array characters.Insert(i, leftCharacter); } // If there are 9 characters then we are done if (characters.Count == 9) { break; } } string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY"); levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}"; return(levelCode); } // fallback to original OCR if segmentation has unexpected number of characters log.Debug($"segmentChacters - level code - detected {characters.Count} characters, falling back to original OCR method"); return(doOCROnLevelCodeImage(ocrReadyImage)); } } catch (Exception ex) { log.Error("Exception in getStringFromLevelCodeImage - " + ex.Message); return(""); } finally { ocrReadyImage?.Dispose(); croppedImage?.Dispose(); } }
/// <summary> /// Event Callback for the Black Screen event generated by the VideoProcessor /// </summary> private void VideoProcessor_BlackScreen(object sender, VideoProcessor.VideoEventArgs e) { log.Debug("Detected a black screen"); BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing black screen...")); // Is this a new level? Image <Bgr, byte> frame = e.frameBuffer[e.frameBuffer.Length - 5]; double imageMatchPercent = ImageLibrary.CompareImages(frame, levelSelectScreen); if (imageMatchPercent > 0.94) { log.Info("Detected new level."); previewer.SetLastMatch(frame, new Rectangle[] { levelCodeArea, creatorNameArea, levelTitleArea }); BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing level screen...")); Level level = getLevelFromCurrentFrame(frame); writeLevelToFile(level); SMMServer.BroadcastLevel(level); BeginInvoke((MethodInvoker)(() => ocrTextBox.Text = level.code + " | " + level.author + " | " + level.name)); BeginInvoke((MethodInvoker)(() => processingLabel.Text = "")); } else { // Not a new level, see if we can detect a template. Dictionary <String, bool> events = new Dictionary <String, bool> { { "death", false }, { "restart", false }, { "exit", false }, }; BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing events...")); foreach (Image <Bgr, byte> f in e.frameBuffer) { Image <Gray, byte> grayscaleFrame = f.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic); //grayscaleFrame.Save("frame_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); // XXX: Useful for debugging template false-negatives, and for getting templates List <Rectangle> boundaries = new List <Rectangle>(); foreach (EventTemplate tmpl in templates) { if (events[tmpl.eventType]) { continue; } Point loc = tmpl.getLocation(grayscaleFrame); if (!loc.IsEmpty) { events[tmpl.eventType] = true; boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, f.Size)); previewer.SetLastMatch(f, boundaries.ToArray()); } } } BeginInvoke((MethodInvoker)(() => processingLabel.Text = "")); foreach (var evt in events) { if (evt.Value) { log.Info(String.Format("Detected {0}.", evt.Key)); SMMServer.BroadcastEvent(evt.Key); } } } }
private static string getStringFromLevelCodeImage(Image <Bgr, byte> image) { Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image); return(OCRLibrary.GetStringFromLevelCodeImage(ocrReadyImage)); }
private void previewMatch(object sender, VideoProcessor.TemplateMatchEventArgs e) { var boundary = ImageLibrary.ChangeSize(new Rectangle(e.location, e.template.size), processor.TEMPLATE_FRAME_SIZE, processor.frameSize); previewer.SetLastMatch(e.frame, new Rectangle[] { boundary }); }