private void Run()
{
Utils.setDebugMode(true);
er_filter1 = OpenCVForUnity.Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);
er_filter2 = OpenCVForUnity.Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);
Mat transition_p = new Mat(62, 62, CvType.CV_64FC1);
// string filename = "OCRHMM_transitions_table.xml";
// FileStorage fs(filename, FileStorage::READ);
// fs["transition_probabilities"] >> transition_p;
// fs.release();
//Load TransitionProbabilitiesData.
transition_p.put(0, 0, GetTransitionProbabilitiesData(OCRHMM_transitions_table_xml_filepath));
Mat emission_p = Mat.eye(62, 62, CvType.CV_64FC1);
string voc = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
decoder = OCRHMMDecoder.create(OCRHMM_knn_model_data_xml_gz_filepath,
voc, transition_p, emission_p);
webCamTextureToMatHelper = gameObject.GetComponent <WebCamTextureToMatHelper> ();
webCamTextureToMatHelper.Initialize();
flipVerticalToggle.isOn = webCamTextureToMatHelper.flipVertical;
flipHorizontalToggle.isOn = webCamTextureToMatHelper.flipHorizontal;
}
private static String OcrImage(Tesseract ocr, Mat image, OCRMode mode, Mat imageColor)
{
Bgr drawCharColor = new Bgr(Color.Red);
if (image.NumberOfChannels == 1)
{
CvInvoke.CvtColor(image, imageColor, ColorConversion.Gray2Bgr);
}
else
{
image.CopyTo(imageColor);
}
if (mode == OCRMode.FullPage)
{
ocr.SetImage(imageColor);
if (ocr.Recognize() != 0)
{
throw new Exception("Failed to recognizer image");
}
Tesseract.Character[] characters = ocr.GetCharacters();
if (characters.Length == 0)
{
Mat imgGrey = new Mat();
CvInvoke.CvtColor(image, imgGrey, ColorConversion.Bgr2Gray);
Mat imgThresholded = new Mat();
CvInvoke.Threshold(imgGrey, imgThresholded, 65, 255, ThresholdType.Binary);
ocr.SetImage(imgThresholded);
characters = ocr.GetCharacters();
imageColor = imgThresholded;
if (characters.Length == 0)
{
CvInvoke.Threshold(image, imgThresholded, 190, 255, ThresholdType.Binary);
ocr.SetImage(imgThresholded);
characters = ocr.GetCharacters();
imageColor = imgThresholded;
}
}
foreach (Tesseract.Character c in characters)
{
CvInvoke.Rectangle(imageColor, c.Region, drawCharColor.MCvScalar);
}
return(ocr.GetUTF8Text());
}
else
{
bool checkInvert = true;
Rectangle[] regions;
using (
ERFilterNM1 er1 = new ERFilterNM1("trained_classifierNM1.xml", 8, 0.00025f, 0.13f, 0.4f, true, 0.1f))
using (ERFilterNM2 er2 = new ERFilterNM2("trained_classifierNM2.xml", 0.3f))
{
int channelCount = image.NumberOfChannels;
UMat[] channels = new UMat[checkInvert ? channelCount * 2 : channelCount];
for (int i = 0; i < channelCount; i++)
{
UMat c = new UMat();
CvInvoke.ExtractChannel(image, c, i);
channels[i] = c;
}
if (checkInvert)
{
for (int i = 0; i < channelCount; i++)
{
UMat c = new UMat();
CvInvoke.BitwiseNot(channels[i], c);
channels[i + channelCount] = c;
}
}
VectorOfERStat[] regionVecs = new VectorOfERStat[channels.Length];
for (int i = 0; i < regionVecs.Length; i++)
{
regionVecs[i] = new VectorOfERStat();
}
try
{
for (int i = 0; i < channels.Length; i++)
{
er1.Run(channels[i], regionVecs[i]);
er2.Run(channels[i], regionVecs[i]);
}
using (VectorOfUMat vm = new VectorOfUMat(channels))
{
regions = ERFilter.ERGrouping(image, vm, regionVecs, ERFilter.GroupingMethod.OrientationHoriz,
"trained_classifier_erGrouping.xml", 0.5f);
}
}
finally
{
foreach (UMat tmp in channels)
{
if (tmp != null)
{
tmp.Dispose();
}
}
foreach (VectorOfERStat tmp in regionVecs)
{
if (tmp != null)
{
tmp.Dispose();
}
}
}
Rectangle imageRegion = new Rectangle(Point.Empty, imageColor.Size);
for (int i = 0; i < regions.Length; i++)
{
Rectangle r = ScaleRectangle(regions[i], 1.1);
r.Intersect(imageRegion);
regions[i] = r;
}
}
List <Tesseract.Character> allChars = new List <Tesseract.Character>();
String allText = String.Empty;
foreach (Rectangle rect in regions)
{
using (Mat region = new Mat(image, rect))
{
ocr.SetImage(region);
if (ocr.Recognize() != 0)
{
throw new Exception("Failed to recognize image");
}
Tesseract.Character[] characters = ocr.GetCharacters();
//convert the coordinates from the local region to global
for (int i = 0; i < characters.Length; i++)
{
Rectangle charRegion = characters[i].Region;
charRegion.Offset(rect.Location);
characters[i].Region = charRegion;
}
allChars.AddRange(characters);
allText += ocr.GetUTF8Text() + Environment.NewLine;
}
}
Bgr drawRegionColor = new Bgr(Color.Red);
foreach (Rectangle rect in regions)
{
CvInvoke.Rectangle(imageColor, rect, drawRegionColor.MCvScalar);
}
foreach (Tesseract.Character c in allChars)
{
CvInvoke.Rectangle(imageColor, c.Region, drawCharColor.MCvScalar);
}
return(allText);
}
}
private void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat frame = Imgcodecs.imread(scenetext01_jpg_filepath);
#if !UNITY_WSA_10_0
if (frame.empty())
{
Debug.LogError("text/scenetext01.jpg is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
}
#endif
Mat binaryMat = new Mat();
Mat maskMat = new Mat();
List <MatOfPoint> regions = new List <MatOfPoint> ();
ERFilter er_filter1 = Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);
ERFilter er_filter2 = Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);
Mat transition_p = new Mat(62, 62, CvType.CV_64FC1);
// string filename = "OCRHMM_transitions_table.xml";
// FileStorage fs(filename, FileStorage::READ);
// fs["transition_probabilities"] >> transition_p;
// fs.release();
//Load TransitionProbabilitiesData.
transition_p.put(0, 0, GetTransitionProbabilitiesData(OCRHMM_transitions_table_xml_filepath));
Mat emission_p = Mat.eye(62, 62, CvType.CV_64FC1);
string voc = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
OCRHMMDecoder decoder = OCRHMMDecoder.create(
OCRHMM_knn_model_data_xml_gz_filepath,
voc, transition_p, emission_p);
//Text Detection
Imgproc.cvtColor(frame, frame, Imgproc.COLOR_BGR2RGB);
Imgproc.cvtColor(frame, binaryMat, Imgproc.COLOR_RGB2GRAY);
Imgproc.threshold(binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
Core.absdiff(binaryMat, new Scalar(255), maskMat);
Text.detectRegions(binaryMat, er_filter1, er_filter2, regions);
Debug.Log("regions.Count " + regions.Count);
MatOfRect groups_rects = new MatOfRect();
List <OpenCVForUnity.Rect> rects = new List <OpenCVForUnity.Rect> ();
Text.erGrouping(frame, binaryMat, regions, groups_rects);
for (int i = 0; i < regions.Count; i++)
{
regions [i].Dispose();
}
regions.Clear();
rects.AddRange(groups_rects.toList());
groups_rects.Dispose();
//Text Recognition (OCR)
List <Mat> detections = new List <Mat> ();
for (int i = 0; i < (int)rects.Count; i++)
{
Mat group_img = new Mat();
maskMat.submat(rects [i]).copyTo(group_img);
Core.copyMakeBorder(group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar(0));
detections.Add(group_img);
}
Debug.Log("detections.Count " + detections.Count);
//#Visualization
for (int i = 0; i < rects.Count; i++)
{
Imgproc.rectangle(frame, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0), 2);
Imgproc.rectangle(frame, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 255, 255), 1);
string output = decoder.run(detections [i], 0);
if (!string.IsNullOrEmpty(output))
{
Debug.Log("output " + output);
Imgproc.putText(frame, output, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 255), 1, Imgproc.LINE_AA, false);
}
}
Texture2D texture = new Texture2D(frame.cols(), frame.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(frame, texture);
// Texture2D texture = new Texture2D (detections [0].cols (), detections [0].rows (), TextureFormat.RGBA32, false);
//
// Utils.matToTexture2D (detections [0], texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
for (int i = 0; i < detections.Count; i++)
{
detections [i].Dispose();
}
binaryMat.Dispose();
maskMat.Dispose();
Utils.setDebugMode(false);
}
private void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(scenetext01_jpg_filepath);
#if !UNITY_WSA_10_0
if (img.empty())
{
Debug.LogError("text/scenetext01.jpg is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
}
#endif
//# for visualization
Mat vis = new Mat();
img.copyTo(vis);
Imgproc.cvtColor(vis, vis, Imgproc.COLOR_BGR2RGB);
//# Extract channels to be processed individually
List <Mat> channels = new List <Mat> ();
Text.computeNMChannels(img, channels);
//# Append negative channels to detect ER- (bright regions over dark background)
int cn = channels.Count;
for (int i = 0; i < cn; i++)
{
channels.Add(new Scalar(255) - channels [i]);
}
//# Apply the default cascade classifier to each independent channel (could be done in parallel)
Debug.Log("Extracting Class Specific Extremal Regions from " + channels.Count + " channels ...");
Debug.Log(" (...) this may take a while (...)");
foreach (var channel in channels)
{
ERFilter er1 = Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 16, 0.00015f, 0.13f, 0.2f, true, 0.1f);
ERFilter er2 = Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);
List <MatOfPoint> regions = new List <MatOfPoint> ();
Text.detectRegions(channel, er1, er2, regions);
MatOfRect matOfRects = new MatOfRect();
Text.erGrouping(img, channel, regions, matOfRects);
// Text.erGrouping (img, channel, regions, matOfRects, Text.ERGROUPING_ORIENTATION_ANY, Utils.getFilePath ("text/trained_classifier_erGrouping.xml"), 0.5f);
List <OpenCVForUnity.Rect> rects = matOfRects.toList();
//#Visualization
foreach (var rect in rects)
{
Imgproc.rectangle(vis, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(255, 0, 0), 2);
Imgproc.rectangle(vis, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(255, 255, 255), 1);
}
}
Texture2D texture = new Texture2D(vis.cols(), vis.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(vis, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Utils.setDebugMode(false);
}
private void loadImageButton_Click(object sender, EventArgs e)
{
if (openImageFileDialog.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
fileNameTextBox.Text = openImageFileDialog.FileName;
imageBox1.Image = null;
ocrTextBox.Text = String.Empty;
hocrTextBox.Text = String.Empty;
Bgr drawCharColor = new Bgr(Color.Blue);
try
{
Mat image = new Mat(openImageFileDialog.FileName, ImreadModes.AnyColor);
Mat imageColor = new Mat();
if (image.NumberOfChannels == 1)
{
CvInvoke.CvtColor(image, imageColor, ColorConversion.Gray2Bgr);
}
else
{
image.CopyTo(imageColor);
}
if (Mode == OCRMode.FullPage)
{
_ocr.Recognize(image);
Tesseract.Character[] characters = _ocr.GetCharacters();
if (characters.Length == 0)
{
Mat imgGrey = new Mat();
CvInvoke.CvtColor(image, imgGrey, ColorConversion.Bgr2Gray);
Mat imgThresholded = new Mat();
CvInvoke.Threshold(imgGrey, imgThresholded, 65, 255, ThresholdType.Binary);
_ocr.Recognize(imgThresholded);
characters = _ocr.GetCharacters();
imageColor = imgThresholded;
if (characters.Length == 0)
{
CvInvoke.Threshold(image, imgThresholded, 190, 255, ThresholdType.Binary);
_ocr.Recognize(imgThresholded);
characters = _ocr.GetCharacters();
imageColor = imgThresholded;
}
}
foreach (Tesseract.Character c in characters)
{
CvInvoke.Rectangle(imageColor, c.Region, drawCharColor.MCvScalar);
}
imageBox1.Image = imageColor;
String text = _ocr.GetText();
ocrTextBox.Text = text;
String hocrText = _ocr.GetHOCRText();
hocrTextBox.Text = hocrText;
}
else
{
bool checkInvert = true;
Rectangle[] regions;
using (ERFilterNM1 er1 = new ERFilterNM1("trained_classifierNM1.xml", 8, 0.00025f, 0.13f, 0.4f, true, 0.1f))
using (ERFilterNM2 er2 = new ERFilterNM2("trained_classifierNM2.xml", 0.3f))
{
int channelCount = image.NumberOfChannels;
UMat[] channels = new UMat[checkInvert ? channelCount * 2 : channelCount];
for (int i = 0; i < channelCount; i++)
{
UMat c = new UMat();
CvInvoke.ExtractChannel(image, c, i);
channels[i] = c;
}
if (checkInvert)
{
for (int i = 0; i < channelCount; i++)
{
UMat c = new UMat();
CvInvoke.BitwiseNot(channels[i], c);
channels[i + channelCount] = c;
}
}
VectorOfERStat[] regionVecs = new VectorOfERStat[channels.Length];
for (int i = 0; i < regionVecs.Length; i++)
{
regionVecs[i] = new VectorOfERStat();
}
try
{
for (int i = 0; i < channels.Length; i++)
{
er1.Run(channels[i], regionVecs[i]);
er2.Run(channels[i], regionVecs[i]);
}
using (VectorOfUMat vm = new VectorOfUMat(channels))
{
regions = ERFilter.ERGrouping(image, vm, regionVecs, ERFilter.GroupingMethod.OrientationHoriz, "trained_classifier_erGrouping.xml", 0.5f);
}
}
finally
{
foreach (UMat tmp in channels)
{
if (tmp != null)
{
tmp.Dispose();
}
}
foreach (VectorOfERStat tmp in regionVecs)
{
if (tmp != null)
{
tmp.Dispose();
}
}
}
Rectangle imageRegion = new Rectangle(Point.Empty, imageColor.Size);
for (int i = 0; i < regions.Length; i++)
{
Rectangle r = ScaleRectangle(regions[i], 1.1);
r.Intersect(imageRegion);
regions[i] = r;
}
}
List <Tesseract.Character> allChars = new List <Tesseract.Character>();
String allText = String.Empty;
foreach (Rectangle rect in regions)
{
using (Mat region = new Mat(image, rect))
{
_ocr.Recognize(region);
Tesseract.Character[] characters = _ocr.GetCharacters();
//convert the coordinates from the local region to global
for (int i = 0; i < characters.Length; i++)
{
Rectangle charRegion = characters[i].Region;
charRegion.Offset(rect.Location);
characters[i].Region = charRegion;
}
allChars.AddRange(characters);
allText += _ocr.GetText() + Environment.NewLine;
}
}
Bgr drawRegionColor = new Bgr(Color.Red);
foreach (Rectangle rect in regions)
{
CvInvoke.Rectangle(imageColor, rect, drawRegionColor.MCvScalar);
}
foreach (Tesseract.Character c in allChars)
{
CvInvoke.Rectangle(imageColor, c.Region, drawCharColor.MCvScalar);
}
imageBox1.Image = imageColor;
ocrTextBox.Text = allText;
}
}
catch (Exception exception)
{
MessageBox.Show(exception.Message);
}
}
}
internal static extern void CvERGrouping(
IntPtr image, IntPtr channels,
IntPtr regions, int count,
IntPtr groups, IntPtr groupRects,
ERFilter.GroupingMethod method, IntPtr fileName, float minProbability);
public string Recognize(Mat image)
{
Rectangle[] regions;
Bgr drawCharColor = new Bgr(Color.Red);
using (var er1 = new ERFilterNM1("Assets\\trained_classifierNM1.xml", 8, 0.00025f, 0.13f, 0.4f, true, 0.1f)) {
using (var er2 = new ERFilterNM2("Assets\\trained_classifierNM2.xml", 0.3f)) {
var channelCount = image.NumberOfChannels;
var channels = new UMat[channelCount * 2];
for (int i = 0; i < channelCount; i++)
{
var c = new UMat();
CvInvoke.ExtractChannel(image, c, i);
channels[i] = c;
}
for (int i = 0; i < channelCount; i++)
{
var c = new UMat();
CvInvoke.BitwiseNot(channels[i], c);
channels[i + channelCount] = c;
}
var regionVecs = new VectorOfERStat[channels.Length];
for (int i = 0; i < regionVecs.Length; i++)
{
regionVecs[i] = new VectorOfERStat();
}
try {
for (int i = 0; i < channels.Length; i++)
{
er1.Run(channels[i], regionVecs[i]);
er2.Run(channels[i], regionVecs[i]);
}
using (var vm = new VectorOfUMat(channels)) {
regions = ERFilter.ERGrouping(image, vm, regionVecs, ERFilter.GroupingMethod.OrientationHoriz,
"Assets\\trained_classifier_erGrouping.xml", 0.5f);
}
}
finally {
foreach (UMat tmp in channels)
{
if (tmp != null)
{
tmp.Dispose();
}
}
foreach (VectorOfERStat tmp in regionVecs)
{
if (tmp != null)
{
tmp.Dispose();
}
}
}
Rectangle imageRegion = new Rectangle(Point.Empty, image.Size);
for (int i = 0; i < regions.Length; i++)
{
Rectangle r = ScaleRectangle(regions[i], 1.1);
r.Intersect(imageRegion);
regions[i] = r;
}
}
var allChars = new List <Tesseract.Character>();
String allText = String.Empty;
foreach (Rectangle rect in regions)
{
using (Mat region = new Mat(image, rect)) {
_ocr.SetImage(region);
if (_ocr.Recognize() != 0)
{
return(null);
}
//var characters = _ocr.GetCharacters();
////convert the coordinates from the local region to global
//for (int i = 0; i < characters.Length; i++) {
// Rectangle charRegion = characters[i].Region;
// charRegion.Offset(rect.Location);
// characters[i].Region = charRegion;
//}
//allChars.AddRange(characters);
allText += _ocr.GetUTF8Text() + "|";
}
}
//Bgr drawRegionColor = new Bgr(Color.Red);
//foreach (Rectangle rect in regions) {
// CvInvoke.Rectangle(image, rect, drawRegionColor.MCvScalar);
//}
//foreach (Tesseract.Character c in allChars) {
// CvInvoke.Rectangle(image, c.Region, drawCharColor.MCvScalar);
//}
return(allText);
}
}
