private static string getStringFromLevelCodeImage(Image <Bgr, byte> image)
{
Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image);
ocrReadyImage = cropLineOfText(ocrReadyImage, new Size(37, 3));
List <Mat> characters = segmentCharacters(ocrReadyImage);
// 11 characters expected in a level code (XXX-XXX-XXX)
if (characters.Count == 11)
{
// Remove the dashes
characters.RemoveAt(7);
characters.RemoveAt(3);
string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY");
// format code
levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}";
return(levelCode);
}
else
{
// fallback to original OCR if segmentation has unexpected number of characters
log.Debug($"segmentChacters - level code - detected {characters.Count} characters, falling back to original OCR method");
return(doOCROnLevelCodeImage(ocrReadyImage));
}
}
private void startButton_Click(object sender, EventArgs e)
{
if (deviceComboBox.SelectedItem == null)
{
MessageBox.Show("Please select a video device first.");
return;
}
try
{
// resize reference image based on current resolution
levelSelectScreen = ImageLibrary.ChangeSize(levelSelectScreen720, resolution720, SelectedResolution);
// resize rectangles based on current resolution
levelCodeArea = ImageLibrary.ChangeSize(levelCodeArea720p, resolution720, SelectedResolution);
creatorNameArea = ImageLibrary.ChangeSize(creatorNameArea720, resolution720, SelectedResolution);
levelTitleArea = ImageLibrary.ChangeSize(levelTitleArea720, resolution720, SelectedResolution);
SMMServer.port = Decimal.ToUInt16(numPort.Value);
log.Info(String.Format("Start Web Server on http://localhost:{0}/", SMMServer.port));
SMMServer.Start();
processor = new VideoProcessor(deviceComboBox.SelectedIndex, SelectedResolution);
processor.BlackScreen += VideoProcessor_BlackScreen;
processor.ClearScreen += VideoProcessor_ClearScreen;
processor.NewFrame += VideoProcessor_NewFrame;
processor.Start();
lockForm();
}
catch (Exception ex)
{
processException("Error starting video device", ex);
}
}
private static string getStringFromImage(Image <Bgr, byte> image)
{
using (Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image))
{
return(doOCROnImage(ocrReadyImage));
}
}
private static void updateLevelBoundryResolutions(Size newResolution)
{
// only update if a new resolution is detected
if (newResolution.Height != levelCodeBoundry.Height)
{
levelCodeBoundry = ImageLibrary.ChangeSize(levelCodeBoundry720, resolution720, newResolution);
creatorNameBoundry = ImageLibrary.ChangeSize(creatorNameBoundry720, resolution720, newResolution);
levelTitleBoundry = ImageLibrary.ChangeSize(levelTitleBoundry720, resolution720, newResolution);
}
}
/// <summary>
/// Read clear time from frame
/// </summary>
/// <param name="frame"></param>
/// <param name="commentsEnabled">If true, then use a different screen region to scan for time</param>
/// <returns></returns>
internal static string GetClearTimeFromFrame(Image <Bgr, byte> frame, bool commentsEnabled)
{
// Update boundry sizes relative to the current resolution
int idx = commentsEnabled ? 1 : 0;
clearTimeBoundry = ImageLibrary.ChangeSize(clearTimeBoundry720[idx], resolution720, frame.Size);
// Set ROI
frame.ROI = clearTimeBoundry;
// Prepare Image for OCR
Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(frame);
// Segment characters
List <Mat> characters = segmentCharacters(ocrReadyImage);
frame.ROI = Rectangle.Empty;
// expect time to be 9 characters, quote reads as 2 chars (ex: 01'34''789)
if (characters.Count == 10)
{
// Different Time Formats
// 12:34,567 | 12:34.567 | 12'34"567
// Remove punctuation - can identify by height
// Make relative to max character size incase we ever resize images.
int maxCharHeight = characters.Max(p => p.Height);
int minimumHeight = (int)Math.Floor(maxCharHeight * .60);
characters.RemoveAll(p => p.Height < minimumHeight);
// Do OCR
string clearTime = doOCROnCharacterImages(characters, "0123456789");
// format clear time
try
{
if (clearTime.Length >= 7)
{
clearTime = $"{clearTime.Substring(0, 2)}'{clearTime.Substring(2, 2)}\"{clearTime.Substring(4, 3)}";
}
}
catch (ArgumentOutOfRangeException ex)
{
log.Error($"ArgumentOutOfRangeException formatting clear time: {clearTime}");
clearTime = "";
}
return(clearTime);
}
else
{
log.Debug($"segmentChacters for time - {characters.Count} characters detected, expected 10, falling back to original OCR method");
return(doOCROnImage(ocrReadyImage));
}
}
/// <summary>
/// Event callback for the Clear Screen event generatead by the VideoProcessor
/// </summary>
private void VideoProcessor_ClearScreen(object sender, VideoProcessor.ClearScreenEventArgs e)
{
log.Debug("Detected Level Clear");
Image <Gray, byte> grayscaleFrame = e.currentFrame.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic);
//e.currentFrame.Save("clearmatch_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png");
Dictionary <String, bool> events = new Dictionary <String, bool>
{
{ "worldrecord", false },
{ "firstclear", false },
};
List <Rectangle> boundaries = new List <Rectangle>();
foreach (EventTemplate tmpl in clearTemplates)
{
if (events[tmpl.eventType])
{
continue;
}
Point loc = tmpl.getLocation(grayscaleFrame);
if (!loc.IsEmpty)
{
events[tmpl.eventType] = true;
boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, e.currentFrame.Size));
previewer.SetLastMatch(e.currentFrame, boundaries.ToArray());
}
}
foreach (var evt in events)
{
if (evt.Value)
{
log.Info(String.Format("Detected {0}.", evt.Key));
SMMServer.BroadcastEvent(evt.Key);
}
}
if (Properties.Settings.Default.WarpWorldEnabled)
{
WarpWorld?.win();
}
// Read time from screen
string clearTime = OCRLibrary.GetClearTimeFromFrame(e.currentFrame, e.commentsEnabled);
SMMServer.BroadcastDataEvent("clear", clearTime);
}
private void startButton_Click(object sender, EventArgs e)
{
if (deviceComboBox.SelectedItem == null)
{
MessageBox.Show("Please select a video device first.");
return;
}
try
{
templates.Clear();
templates.AddRange(engTemplates);
// Add language neutral templates if selected.
if (langNeutralcheckBox.Checked)
{
templates.AddRange(langNeutralTemplates);
}
// resize reference image based on current resolution
levelDetailScreen = ImageLibrary.ChangeSize(levelSelectScreen720, resolution720, SelectedResolution);
SMMServer.port = decimal.ToUInt16(numPort.Value);
log.Info(string.Format("Start Web Server on http://localhost:{0}/", SMMServer.port));
SMMServer.Start();
if (Properties.Settings.Default.WarpWorldEnabled)
{
WarpWorld = new WarpWorldAPI(Properties.Settings.Default.WarpWorldUsername, Properties.Settings.Default.WarpWorldToken);
}
else
{
WarpWorld = null;
}
processor = new VideoProcessor(deviceComboBox.SelectedIndex, SelectedResolution);
processor.BlackScreen += VideoProcessor_BlackScreen;
processor.ClearScreen += VideoProcessor_ClearScreen;
processor.NewFrame += VideoProcessor_NewFrame;
processor.Start();
lockForm();
}
catch (Exception ex)
{
processException("Error starting video device", ex);
}
}
/// <summary>
/// Event Callback for the Black Screen event generated by the VideoProcessor
/// </summary>
private void VideoProcessor_BlackScreen(object sender, VideoProcessor.BlackScreenEventArgs e)
{
log.Debug(String.Format("Detected a black screen [{0}]", e.seconds));
BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing black screen..."));
double imageMatchPercent = ImageLibrary.CompareImages(e.currentFrame, levelDetailScreen);
// Is this frame a 90% match to a level screen?
if (imageMatchPercent > 0.90)
{
log.Info(String.Format("Detected new level. [{0}]", e.seconds));
BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing level screen..."));
Level level = OCRLibrary.GetLevelFromFrame(e.currentFrame);
writeLevelToFile(level);
SMMServer.BroadcastLevel(level);
BeginInvoke((MethodInvoker)(() => ocrTextBox.Text = level.code + " | " + level.author + " | " + level.name));
BeginInvoke((MethodInvoker)(() => processingLabel.Text = ""));
previewer.SetLastMatch(e.currentFrame);
}
else
{
// Not a new level, see if we can detect a template.
Dictionary <String, bool> events = new Dictionary <String, bool>
{
{ "death", false },
{ "restart", false },
{ "exit", false },
{ "gameover", false },
{ "skip", false }
};
BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing events..."));
foreach (Image <Bgr, byte> f in e.frameBuffer)
{
// Skip any empty frames in the buffer
if (f == null)
{
continue;
}
Image <Gray, byte> grayscaleFrame = f.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic);
//grayscaleFrame.Save("frame_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); // XXX: Useful for debugging template false-negatives, and for getting templates
List <Rectangle> boundaries = new List <Rectangle>();
foreach (EventTemplate tmpl in templates)
{
if (events[tmpl.eventType])
{
continue;
}
Point loc = tmpl.getLocation(grayscaleFrame);
if (!loc.IsEmpty)
{
events[tmpl.eventType] = true;
boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, f.Size));
previewer.SetLastMatch(f, boundaries.ToArray());
}
}
}
BeginInvoke((MethodInvoker)(() => processingLabel.Text = ""));
foreach (var evt in events)
{
if (evt.Value)
{
log.Info(String.Format("Detected {0} [{1}].", evt.Key, e.seconds));
SMMServer.BroadcastEvent(evt.Key);
// Even though this will get sent on exiting after a clear, it only matters if a entry is active, and after marking it as a win it goes inactive.
if (evt.Key == "exit")
{
if (Properties.Settings.Default.WarpWorldEnabled)
{
WarpWorld?.lose();
}
if (JsonSettings.ClearOnExit)
{
clearJsonFile();
}
}
if (evt.Key == "skip" && JsonSettings.ClearOnSkip)
{
clearJsonFile();
}
if (evt.Key == "gameover" && JsonSettings.ClearOnGameover)
{
clearJsonFile();
}
}
}
}
}
private static string getStringFromLevelCodeImage(Image <Bgr, byte> image)
{
Image <Gray, byte> ocrReadyImage = null;
Image <Gray, byte> croppedImage = null;
try
{
ocrReadyImage = ImageLibrary.PrepareImageForOCR(image);
croppedImage = cropLineOfText(ocrReadyImage, new Size(37, 3));
List <Mat> characters = segmentCharacters(croppedImage);
// 11 characters expected in a level code (XXX-XXX-XXX)
if (characters.Count == 11)
{
// Remove the dashes
characters.RemoveAt(7);
characters.RemoveAt(3);
string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY");
// format code
levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}";
return(levelCode);
}
else
{
// before falling back to line based OCR, take one more shot at saving this code with segmenting characters.
// This is an extremely crude method which will split character images in half if they are much wider than
// median char width. This is needed for adjacent "TT" characters in a level code.
if (characters.Count < 11 && characters.Count > 7)
{
log.Debug($"getStringFromLevelCodeImage - level code - detected {characters.Count} characters, manually splitting larger characters");
// Remove dashes - can identify by height
int maxCharHeight = characters.Max(p => p.Height);
int minimumHeight = (int)Math.Floor(maxCharHeight * .60);
characters.RemoveAll(p => p.Height < minimumHeight);
double medianCharacterWidth = characters.OrderBy(p => p.Width).Skip(characters.Count / 2).Take(1).FirstOrDefault().Width;
for (int i = 0; i < characters.Count; i++)
{
var character = characters[i];
double perc = medianCharacterWidth / character.Width;
// Current letter is significantly larger than median width
if (perc < .65)
{
Rectangle leftRect = new Rectangle(0, 0, character.Width / 2, character.Height);
Rectangle rightRect = new Rectangle(character.Width / 2, 0, character.Width / 2, character.Height);
// Left character
Mat leftCharacter = new Mat(character, leftRect);
// Right character
characters[i] = new Mat(character, rightRect);
// Insert left character into array
characters.Insert(i, leftCharacter);
}
// If there are 9 characters then we are done
if (characters.Count == 9)
{
break;
}
}
string levelCode = doOCROnCharacterImages(characters, "0123456789ABCDEFGHJKLMNPQRSTUVWXY");
levelCode = $"{levelCode.Substring(0, 3)}-{levelCode.Substring(3, 3)}-{levelCode.Substring(6, 3)}";
return(levelCode);
}
// fallback to original OCR if segmentation has unexpected number of characters
log.Debug($"segmentChacters - level code - detected {characters.Count} characters, falling back to original OCR method");
return(doOCROnLevelCodeImage(ocrReadyImage));
}
}
catch (Exception ex)
{
log.Error("Exception in getStringFromLevelCodeImage - " + ex.Message);
return("");
}
finally
{
ocrReadyImage?.Dispose();
croppedImage?.Dispose();
}
}
/// <summary>
/// Event Callback for the Black Screen event generated by the VideoProcessor
/// </summary>
private void VideoProcessor_BlackScreen(object sender, VideoProcessor.VideoEventArgs e)
{
log.Debug("Detected a black screen");
BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing black screen..."));
// Is this a new level?
Image <Bgr, byte> frame = e.frameBuffer[e.frameBuffer.Length - 5];
double imageMatchPercent = ImageLibrary.CompareImages(frame, levelSelectScreen);
if (imageMatchPercent > 0.94)
{
log.Info("Detected new level.");
previewer.SetLastMatch(frame, new Rectangle[] { levelCodeArea, creatorNameArea, levelTitleArea });
BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing level screen..."));
Level level = getLevelFromCurrentFrame(frame);
writeLevelToFile(level);
SMMServer.BroadcastLevel(level);
BeginInvoke((MethodInvoker)(() => ocrTextBox.Text = level.code + " | " + level.author + " | " + level.name));
BeginInvoke((MethodInvoker)(() => processingLabel.Text = ""));
}
else
{
// Not a new level, see if we can detect a template.
Dictionary <String, bool> events = new Dictionary <String, bool>
{
{ "death", false },
{ "restart", false },
{ "exit", false },
};
BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing events..."));
foreach (Image <Bgr, byte> f in e.frameBuffer)
{
Image <Gray, byte> grayscaleFrame = f.Mat.ToImage <Gray, byte>().Resize(640, 480, Inter.Cubic);
//grayscaleFrame.Save("frame_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); // XXX: Useful for debugging template false-negatives, and for getting templates
List <Rectangle> boundaries = new List <Rectangle>();
foreach (EventTemplate tmpl in templates)
{
if (events[tmpl.eventType])
{
continue;
}
Point loc = tmpl.getLocation(grayscaleFrame);
if (!loc.IsEmpty)
{
events[tmpl.eventType] = true;
boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, f.Size));
previewer.SetLastMatch(f, boundaries.ToArray());
}
}
}
BeginInvoke((MethodInvoker)(() => processingLabel.Text = ""));
foreach (var evt in events)
{
if (evt.Value)
{
log.Info(String.Format("Detected {0}.", evt.Key));
SMMServer.BroadcastEvent(evt.Key);
}
}
}
}
private static string getStringFromLevelCodeImage(Image <Bgr, byte> image)
{
Image <Gray, byte> ocrReadyImage = ImageLibrary.PrepareImageForOCR(image);
return(OCRLibrary.GetStringFromLevelCodeImage(ocrReadyImage));
}
private void previewMatch(object sender, VideoProcessor.TemplateMatchEventArgs e)
{
var boundary = ImageLibrary.ChangeSize(new Rectangle(e.location, e.template.size), processor.TEMPLATE_FRAME_SIZE, processor.frameSize);
previewer.SetLastMatch(e.frame, new Rectangle[] { boundary });
}
