private void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat frame = Imgcodecs.imread(scenetext01_jpg_filepath);
#if !UNITY_WSA_10_0
if (frame.empty())
{
Debug.LogError("text/scenetext01.jpg is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
}
#endif
Mat binaryMat = new Mat();
Mat maskMat = new Mat();
List <MatOfPoint> regions = new List <MatOfPoint> ();
ERFilter er_filter1 = Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);
ERFilter er_filter2 = Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);
Mat transition_p = new Mat(62, 62, CvType.CV_64FC1);
// string filename = "OCRHMM_transitions_table.xml";
// FileStorage fs(filename, FileStorage::READ);
// fs["transition_probabilities"] >> transition_p;
// fs.release();
//Load TransitionProbabilitiesData.
transition_p.put(0, 0, GetTransitionProbabilitiesData(OCRHMM_transitions_table_xml_filepath));
Mat emission_p = Mat.eye(62, 62, CvType.CV_64FC1);
string voc = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
OCRHMMDecoder decoder = OCRHMMDecoder.create(
OCRHMM_knn_model_data_xml_gz_filepath,
voc, transition_p, emission_p);
//Text Detection
Imgproc.cvtColor(frame, frame, Imgproc.COLOR_BGR2RGB);
Imgproc.cvtColor(frame, binaryMat, Imgproc.COLOR_RGB2GRAY);
Imgproc.threshold(binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
Core.absdiff(binaryMat, new Scalar(255), maskMat);
Text.detectRegions(binaryMat, er_filter1, er_filter2, regions);
Debug.Log("regions.Count " + regions.Count);
MatOfRect groups_rects = new MatOfRect();
List <OpenCVForUnity.Rect> rects = new List <OpenCVForUnity.Rect> ();
Text.erGrouping(frame, binaryMat, regions, groups_rects);
for (int i = 0; i < regions.Count; i++)
{
regions [i].Dispose();
}
regions.Clear();
rects.AddRange(groups_rects.toList());
groups_rects.Dispose();
//Text Recognition (OCR)
List <Mat> detections = new List <Mat> ();
for (int i = 0; i < (int)rects.Count; i++)
{
Mat group_img = new Mat();
maskMat.submat(rects [i]).copyTo(group_img);
Core.copyMakeBorder(group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar(0));
detections.Add(group_img);
}
Debug.Log("detections.Count " + detections.Count);
//#Visualization
for (int i = 0; i < rects.Count; i++)
{
Imgproc.rectangle(frame, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0), 2);
Imgproc.rectangle(frame, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 255, 255), 1);
string output = decoder.run(detections [i], 0);
if (!string.IsNullOrEmpty(output))
{
Debug.Log("output " + output);
Imgproc.putText(frame, output, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 255), 1, Imgproc.LINE_AA, false);
}
}
Texture2D texture = new Texture2D(frame.cols(), frame.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(frame, texture);
// Texture2D texture = new Texture2D (detections [0].cols (), detections [0].rows (), TextureFormat.RGBA32, false);
//
// Utils.matToTexture2D (detections [0], texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
for (int i = 0; i < detections.Count; i++)
{
detections [i].Dispose();
}
binaryMat.Dispose();
maskMat.Dispose();
Utils.setDebugMode(false);
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
/*Text Detection*/
Imgproc.cvtColor(rgbMat, binaryMat, Imgproc.COLOR_RGB2GRAY);
Imgproc.threshold(binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
Core.absdiff(binaryMat, new Scalar(255), maskMat);
List <MatOfPoint> regions = new List <MatOfPoint> ();
OpenCVForUnity.Text.detectRegions(binaryMat, er_filter1, er_filter2, regions);
// Debug.Log ("regions.Count " + regions.Count);
MatOfRect groups_rects = new MatOfRect();
List <OpenCVForUnity.Rect> rects = new List <OpenCVForUnity.Rect> ();
if (regions.Count > 0)
{
OpenCVForUnity.Text.erGrouping(rgbMat, binaryMat, regions, groups_rects);
}
for (int i = 0; i < regions.Count; i++)
{
regions [i].Dispose();
}
regions.Clear();
rects.AddRange(groups_rects.toList());
groups_rects.Dispose();
/*Text Recognition (OCR)*/
List <Mat> detections = new List <Mat> ();
for (int i = 0; i < (int)rects.Count; i++)
{
Mat group_img = new Mat();
maskMat.submat(rects [i]).copyTo(group_img);
Core.copyMakeBorder(group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar(0));
detections.Add(group_img);
}
// Debug.Log ("detections.Count " + detections.Count);
// Debug.Log ("rects.Count " + rects.Count);
//#Visualization
for (int i = 0; i < rects.Count; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 255, 255, 255), 1);
Imgproc.putText(rgbaMat, "" + i, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false);
}
for (int i = 0; i < detections.Count; i++)
{
string output = decoder.run(detections [i], 0);
Debug.Log("output " + output);
if (string.IsNullOrEmpty(output))
{
Debug.LogError("IsNullOrEmpty output " + output);
}
else
{
Imgproc.putText(rgbaMat, " " + output, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false);
}
}
for (int i = 0; i < detections.Count; i++)
{
detections [i].Dispose();
}
detections.Clear();
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}