private void Run()
        {
            Utils.setDebugMode(true);


            er_filter1 = OpenCVForUnity.Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);

            er_filter2 = OpenCVForUnity.Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);


            Mat transition_p = new Mat(62, 62, CvType.CV_64FC1);

            //            string filename = "OCRHMM_transitions_table.xml";
            //            FileStorage fs(filename, FileStorage::READ);
            //            fs["transition_probabilities"] >> transition_p;
            //            fs.release();

            //Load TransitionProbabilitiesData.
            transition_p.put(0, 0, GetTransitionProbabilitiesData(OCRHMM_transitions_table_xml_filepath));


            Mat    emission_p = Mat.eye(62, 62, CvType.CV_64FC1);
            string voc        = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";

            decoder = OCRHMMDecoder.create(OCRHMM_knn_model_data_xml_gz_filepath,
                                           voc, transition_p, emission_p);


            webCamTextureToMatHelper = gameObject.GetComponent <WebCamTextureToMatHelper> ();
            webCamTextureToMatHelper.Initialize();

            flipVerticalToggle.isOn   = webCamTextureToMatHelper.flipVertical;
            flipHorizontalToggle.isOn = webCamTextureToMatHelper.flipHorizontal;
        }
Esempio n. 2
0
        private static String OcrImage(Tesseract ocr, Mat image, OCRMode mode, Mat imageColor)
        {
            Bgr drawCharColor = new Bgr(Color.Red);

            if (image.NumberOfChannels == 1)
            {
                CvInvoke.CvtColor(image, imageColor, ColorConversion.Gray2Bgr);
            }
            else
            {
                image.CopyTo(imageColor);
            }

            if (mode == OCRMode.FullPage)
            {
                ocr.SetImage(imageColor);

                if (ocr.Recognize() != 0)
                {
                    throw new Exception("Failed to recognizer image");
                }

                Tesseract.Character[] characters = ocr.GetCharacters();
                if (characters.Length == 0)
                {
                    Mat imgGrey = new Mat();
                    CvInvoke.CvtColor(image, imgGrey, ColorConversion.Bgr2Gray);
                    Mat imgThresholded = new Mat();
                    CvInvoke.Threshold(imgGrey, imgThresholded, 65, 255, ThresholdType.Binary);
                    ocr.SetImage(imgThresholded);
                    characters = ocr.GetCharacters();
                    imageColor = imgThresholded;
                    if (characters.Length == 0)
                    {
                        CvInvoke.Threshold(image, imgThresholded, 190, 255, ThresholdType.Binary);
                        ocr.SetImage(imgThresholded);
                        characters = ocr.GetCharacters();
                        imageColor = imgThresholded;
                    }
                }
                foreach (Tesseract.Character c in characters)
                {
                    CvInvoke.Rectangle(imageColor, c.Region, drawCharColor.MCvScalar);
                }

                return(ocr.GetUTF8Text());
            }
            else
            {
                bool checkInvert = true;

                Rectangle[] regions;

                using (
                    ERFilterNM1 er1 = new ERFilterNM1("trained_classifierNM1.xml", 8, 0.00025f, 0.13f, 0.4f, true, 0.1f))
                    using (ERFilterNM2 er2 = new ERFilterNM2("trained_classifierNM2.xml", 0.3f))
                    {
                        int    channelCount = image.NumberOfChannels;
                        UMat[] channels     = new UMat[checkInvert ? channelCount * 2 : channelCount];

                        for (int i = 0; i < channelCount; i++)
                        {
                            UMat c = new UMat();
                            CvInvoke.ExtractChannel(image, c, i);
                            channels[i] = c;
                        }

                        if (checkInvert)
                        {
                            for (int i = 0; i < channelCount; i++)
                            {
                                UMat c = new UMat();
                                CvInvoke.BitwiseNot(channels[i], c);
                                channels[i + channelCount] = c;
                            }
                        }

                        VectorOfERStat[] regionVecs = new VectorOfERStat[channels.Length];
                        for (int i = 0; i < regionVecs.Length; i++)
                        {
                            regionVecs[i] = new VectorOfERStat();
                        }

                        try
                        {
                            for (int i = 0; i < channels.Length; i++)
                            {
                                er1.Run(channels[i], regionVecs[i]);
                                er2.Run(channels[i], regionVecs[i]);
                            }
                            using (VectorOfUMat vm = new VectorOfUMat(channels))
                            {
                                regions = ERFilter.ERGrouping(image, vm, regionVecs, ERFilter.GroupingMethod.OrientationHoriz,
                                                              "trained_classifier_erGrouping.xml", 0.5f);
                            }
                        }
                        finally
                        {
                            foreach (UMat tmp in channels)
                            {
                                if (tmp != null)
                                {
                                    tmp.Dispose();
                                }
                            }
                            foreach (VectorOfERStat tmp in regionVecs)
                            {
                                if (tmp != null)
                                {
                                    tmp.Dispose();
                                }
                            }
                        }

                        Rectangle imageRegion = new Rectangle(Point.Empty, imageColor.Size);
                        for (int i = 0; i < regions.Length; i++)
                        {
                            Rectangle r = ScaleRectangle(regions[i], 1.1);

                            r.Intersect(imageRegion);
                            regions[i] = r;
                        }
                    }


                List <Tesseract.Character> allChars = new List <Tesseract.Character>();
                String allText = String.Empty;
                foreach (Rectangle rect in regions)
                {
                    using (Mat region = new Mat(image, rect))
                    {
                        ocr.SetImage(region);
                        if (ocr.Recognize() != 0)
                        {
                            throw new Exception("Failed to recognize image");
                        }
                        Tesseract.Character[] characters = ocr.GetCharacters();

                        //convert the coordinates from the local region to global
                        for (int i = 0; i < characters.Length; i++)
                        {
                            Rectangle charRegion = characters[i].Region;
                            charRegion.Offset(rect.Location);
                            characters[i].Region = charRegion;
                        }
                        allChars.AddRange(characters);

                        allText += ocr.GetUTF8Text() + Environment.NewLine;
                    }
                }

                Bgr drawRegionColor = new Bgr(Color.Red);
                foreach (Rectangle rect in regions)
                {
                    CvInvoke.Rectangle(imageColor, rect, drawRegionColor.MCvScalar);
                }
                foreach (Tesseract.Character c in allChars)
                {
                    CvInvoke.Rectangle(imageColor, c.Region, drawCharColor.MCvScalar);
                }

                return(allText);
            }
        }
        private void Run()
        {
            //if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
            Utils.setDebugMode(true);


            Mat frame = Imgcodecs.imread(scenetext01_jpg_filepath);

            #if !UNITY_WSA_10_0
            if (frame.empty())
            {
                Debug.LogError("text/scenetext01.jpg is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
            }
            #endif

            Mat binaryMat = new Mat();
            Mat maskMat   = new Mat();


            List <MatOfPoint> regions = new List <MatOfPoint> ();

            ERFilter er_filter1 = Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);

            ERFilter er_filter2 = Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);


            Mat transition_p = new Mat(62, 62, CvType.CV_64FC1);
            //            string filename = "OCRHMM_transitions_table.xml";
            //            FileStorage fs(filename, FileStorage::READ);
            //            fs["transition_probabilities"] >> transition_p;
            //            fs.release();

            //Load TransitionProbabilitiesData.
            transition_p.put(0, 0, GetTransitionProbabilitiesData(OCRHMM_transitions_table_xml_filepath));

            Mat           emission_p = Mat.eye(62, 62, CvType.CV_64FC1);
            string        voc        = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
            OCRHMMDecoder decoder    = OCRHMMDecoder.create(
                OCRHMM_knn_model_data_xml_gz_filepath,
                voc, transition_p, emission_p);

            //Text Detection
            Imgproc.cvtColor(frame, frame, Imgproc.COLOR_BGR2RGB);
            Imgproc.cvtColor(frame, binaryMat, Imgproc.COLOR_RGB2GRAY);
            Imgproc.threshold(binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
            Core.absdiff(binaryMat, new Scalar(255), maskMat);


            Text.detectRegions(binaryMat, er_filter1, er_filter2, regions);
            Debug.Log("regions.Count " + regions.Count);


            MatOfRect groups_rects           = new MatOfRect();
            List <OpenCVForUnity.Rect> rects = new List <OpenCVForUnity.Rect> ();
            Text.erGrouping(frame, binaryMat, regions, groups_rects);


            for (int i = 0; i < regions.Count; i++)
            {
                regions [i].Dispose();
            }
            regions.Clear();


            rects.AddRange(groups_rects.toList());

            groups_rects.Dispose();


            //Text Recognition (OCR)

            List <Mat> detections = new List <Mat> ();

            for (int i = 0; i < (int)rects.Count; i++)
            {
                Mat group_img = new Mat();
                maskMat.submat(rects [i]).copyTo(group_img);
                Core.copyMakeBorder(group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar(0));
                detections.Add(group_img);
            }

            Debug.Log("detections.Count " + detections.Count);


            //#Visualization
            for (int i = 0; i < rects.Count; i++)
            {
                Imgproc.rectangle(frame, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0), 2);
                Imgproc.rectangle(frame, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 255, 255), 1);

                string output = decoder.run(detections [i], 0);
                if (!string.IsNullOrEmpty(output))
                {
                    Debug.Log("output " + output);
                    Imgproc.putText(frame, output, new Point(rects [i].x, rects [i].y), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 255), 1, Imgproc.LINE_AA, false);
                }
            }


            Texture2D texture = new Texture2D(frame.cols(), frame.rows(), TextureFormat.RGBA32, false);

            Utils.matToTexture2D(frame, texture);

//            Texture2D texture = new Texture2D (detections [0].cols (), detections [0].rows (), TextureFormat.RGBA32, false);
//
//            Utils.matToTexture2D (detections [0], texture);

            gameObject.GetComponent <Renderer> ().material.mainTexture = texture;


            for (int i = 0; i < detections.Count; i++)
            {
                detections [i].Dispose();
            }
            binaryMat.Dispose();
            maskMat.Dispose();

            Utils.setDebugMode(false);
        }
        private void Run()
        {
            //if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
            Utils.setDebugMode(true);


            Mat img = Imgcodecs.imread(scenetext01_jpg_filepath);

            #if !UNITY_WSA_10_0
            if (img.empty())
            {
                Debug.LogError("text/scenetext01.jpg is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
            }
            #endif

            //# for visualization
            Mat vis = new Mat();
            img.copyTo(vis);
            Imgproc.cvtColor(vis, vis, Imgproc.COLOR_BGR2RGB);


            //# Extract channels to be processed individually
            List <Mat> channels = new List <Mat> ();
            Text.computeNMChannels(img, channels);

            //# Append negative channels to detect ER- (bright regions over dark background)
            int cn = channels.Count;
            for (int i = 0; i < cn; i++)
            {
                channels.Add(new Scalar(255) - channels [i]);
            }

            //# Apply the default cascade classifier to each independent channel (could be done in parallel)
            Debug.Log("Extracting Class Specific Extremal Regions from " + channels.Count + " channels ...");
            Debug.Log("    (...) this may take a while (...)");
            foreach (var channel in channels)
            {
                ERFilter er1 = Text.createERFilterNM1(trained_classifierNM1_xml_filepath, 16, 0.00015f, 0.13f, 0.2f, true, 0.1f);

                ERFilter er2 = Text.createERFilterNM2(trained_classifierNM2_xml_filepath, 0.5f);

                List <MatOfPoint> regions = new List <MatOfPoint> ();
                Text.detectRegions(channel, er1, er2, regions);

                MatOfRect matOfRects = new MatOfRect();
                Text.erGrouping(img, channel, regions, matOfRects);
//                Text.erGrouping (img, channel, regions, matOfRects, Text.ERGROUPING_ORIENTATION_ANY, Utils.getFilePath ("text/trained_classifier_erGrouping.xml"), 0.5f);

                List <OpenCVForUnity.Rect> rects = matOfRects.toList();

                //#Visualization
                foreach (var rect in rects)
                {
                    Imgproc.rectangle(vis, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(255, 0, 0), 2);
                    Imgproc.rectangle(vis, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(255, 255, 255), 1);
                }
            }

            Texture2D texture = new Texture2D(vis.cols(), vis.rows(), TextureFormat.RGBA32, false);

            Utils.matToTexture2D(vis, texture);

            gameObject.GetComponent <Renderer> ().material.mainTexture = texture;


            Utils.setDebugMode(false);
        }
Esempio n. 5
0
        private void loadImageButton_Click(object sender, EventArgs e)
        {
            if (openImageFileDialog.ShowDialog() == System.Windows.Forms.DialogResult.OK)
            {
                fileNameTextBox.Text = openImageFileDialog.FileName;
                imageBox1.Image      = null;
                ocrTextBox.Text      = String.Empty;
                hocrTextBox.Text     = String.Empty;

                Bgr drawCharColor = new Bgr(Color.Blue);
                try
                {
                    Mat image = new Mat(openImageFileDialog.FileName, ImreadModes.AnyColor);

                    Mat imageColor = new Mat();
                    if (image.NumberOfChannels == 1)
                    {
                        CvInvoke.CvtColor(image, imageColor, ColorConversion.Gray2Bgr);
                    }
                    else
                    {
                        image.CopyTo(imageColor);
                    }

                    if (Mode == OCRMode.FullPage)
                    {
                        _ocr.Recognize(image);
                        Tesseract.Character[] characters = _ocr.GetCharacters();
                        if (characters.Length == 0)
                        {
                            Mat imgGrey = new Mat();
                            CvInvoke.CvtColor(image, imgGrey, ColorConversion.Bgr2Gray);
                            Mat imgThresholded = new Mat();
                            CvInvoke.Threshold(imgGrey, imgThresholded, 65, 255, ThresholdType.Binary);
                            _ocr.Recognize(imgThresholded);
                            characters = _ocr.GetCharacters();
                            imageColor = imgThresholded;
                            if (characters.Length == 0)
                            {
                                CvInvoke.Threshold(image, imgThresholded, 190, 255, ThresholdType.Binary);
                                _ocr.Recognize(imgThresholded);
                                characters = _ocr.GetCharacters();
                                imageColor = imgThresholded;
                            }
                        }
                        foreach (Tesseract.Character c in characters)
                        {
                            CvInvoke.Rectangle(imageColor, c.Region, drawCharColor.MCvScalar);
                        }

                        imageBox1.Image = imageColor;

                        String text = _ocr.GetText();
                        ocrTextBox.Text = text;
                        String hocrText = _ocr.GetHOCRText();
                        hocrTextBox.Text = hocrText;
                    }
                    else
                    {
                        bool checkInvert = true;

                        Rectangle[] regions;

                        using (ERFilterNM1 er1 = new ERFilterNM1("trained_classifierNM1.xml", 8, 0.00025f, 0.13f, 0.4f, true, 0.1f))
                            using (ERFilterNM2 er2 = new ERFilterNM2("trained_classifierNM2.xml", 0.3f))
                            {
                                int    channelCount = image.NumberOfChannels;
                                UMat[] channels     = new UMat[checkInvert ? channelCount * 2 : channelCount];

                                for (int i = 0; i < channelCount; i++)
                                {
                                    UMat c = new UMat();
                                    CvInvoke.ExtractChannel(image, c, i);
                                    channels[i] = c;
                                }

                                if (checkInvert)
                                {
                                    for (int i = 0; i < channelCount; i++)
                                    {
                                        UMat c = new UMat();
                                        CvInvoke.BitwiseNot(channels[i], c);
                                        channels[i + channelCount] = c;
                                    }
                                }

                                VectorOfERStat[] regionVecs = new VectorOfERStat[channels.Length];
                                for (int i = 0; i < regionVecs.Length; i++)
                                {
                                    regionVecs[i] = new VectorOfERStat();
                                }

                                try
                                {
                                    for (int i = 0; i < channels.Length; i++)
                                    {
                                        er1.Run(channels[i], regionVecs[i]);
                                        er2.Run(channels[i], regionVecs[i]);
                                    }
                                    using (VectorOfUMat vm = new VectorOfUMat(channels))
                                    {
                                        regions = ERFilter.ERGrouping(image, vm, regionVecs, ERFilter.GroupingMethod.OrientationHoriz, "trained_classifier_erGrouping.xml", 0.5f);
                                    }
                                }
                                finally
                                {
                                    foreach (UMat tmp in channels)
                                    {
                                        if (tmp != null)
                                        {
                                            tmp.Dispose();
                                        }
                                    }
                                    foreach (VectorOfERStat tmp in regionVecs)
                                    {
                                        if (tmp != null)
                                        {
                                            tmp.Dispose();
                                        }
                                    }
                                }

                                Rectangle imageRegion = new Rectangle(Point.Empty, imageColor.Size);
                                for (int i = 0; i < regions.Length; i++)
                                {
                                    Rectangle r = ScaleRectangle(regions[i], 1.1);

                                    r.Intersect(imageRegion);
                                    regions[i] = r;
                                }
                            }


                        List <Tesseract.Character> allChars = new List <Tesseract.Character>();
                        String allText = String.Empty;
                        foreach (Rectangle rect in regions)
                        {
                            using (Mat region = new Mat(image, rect))
                            {
                                _ocr.Recognize(region);
                                Tesseract.Character[] characters = _ocr.GetCharacters();

                                //convert the coordinates from the local region to global
                                for (int i = 0; i < characters.Length; i++)
                                {
                                    Rectangle charRegion = characters[i].Region;
                                    charRegion.Offset(rect.Location);
                                    characters[i].Region = charRegion;
                                }
                                allChars.AddRange(characters);

                                allText += _ocr.GetText() + Environment.NewLine;
                            }
                        }

                        Bgr drawRegionColor = new Bgr(Color.Red);
                        foreach (Rectangle rect in regions)
                        {
                            CvInvoke.Rectangle(imageColor, rect, drawRegionColor.MCvScalar);
                        }
                        foreach (Tesseract.Character c in allChars)
                        {
                            CvInvoke.Rectangle(imageColor, c.Region, drawCharColor.MCvScalar);
                        }
                        imageBox1.Image = imageColor;
                        ocrTextBox.Text = allText;
                    }
                }
                catch (Exception exception)
                {
                    MessageBox.Show(exception.Message);
                }
            }
        }
Esempio n. 6
0
 internal static extern void CvERGrouping(
    IntPtr image, IntPtr channels,
    IntPtr regions, int count,
    IntPtr groups, IntPtr groupRects,
    ERFilter.GroupingMethod method, IntPtr fileName, float minProbability);
        public string Recognize(Mat image)
        {
            Rectangle[] regions;
            Bgr         drawCharColor = new Bgr(Color.Red);

            using (var er1 = new ERFilterNM1("Assets\\trained_classifierNM1.xml", 8, 0.00025f, 0.13f, 0.4f, true, 0.1f)) {
                using (var er2 = new ERFilterNM2("Assets\\trained_classifierNM2.xml", 0.3f)) {
                    var channelCount = image.NumberOfChannels;
                    var channels     = new UMat[channelCount * 2];

                    for (int i = 0; i < channelCount; i++)
                    {
                        var c = new UMat();
                        CvInvoke.ExtractChannel(image, c, i);
                        channels[i] = c;
                    }

                    for (int i = 0; i < channelCount; i++)
                    {
                        var c = new UMat();
                        CvInvoke.BitwiseNot(channels[i], c);
                        channels[i + channelCount] = c;
                    }

                    var regionVecs = new VectorOfERStat[channels.Length];
                    for (int i = 0; i < regionVecs.Length; i++)
                    {
                        regionVecs[i] = new VectorOfERStat();
                    }
                    try {
                        for (int i = 0; i < channels.Length; i++)
                        {
                            er1.Run(channels[i], regionVecs[i]);
                            er2.Run(channels[i], regionVecs[i]);
                        }
                        using (var vm = new VectorOfUMat(channels)) {
                            regions = ERFilter.ERGrouping(image, vm, regionVecs, ERFilter.GroupingMethod.OrientationHoriz,
                                                          "Assets\\trained_classifier_erGrouping.xml", 0.5f);
                        }
                    }
                    finally {
                        foreach (UMat tmp in channels)
                        {
                            if (tmp != null)
                            {
                                tmp.Dispose();
                            }
                        }
                        foreach (VectorOfERStat tmp in regionVecs)
                        {
                            if (tmp != null)
                            {
                                tmp.Dispose();
                            }
                        }
                    }
                    Rectangle imageRegion = new Rectangle(Point.Empty, image.Size);
                    for (int i = 0; i < regions.Length; i++)
                    {
                        Rectangle r = ScaleRectangle(regions[i], 1.1);

                        r.Intersect(imageRegion);
                        regions[i] = r;
                    }
                }

                var    allChars = new List <Tesseract.Character>();
                String allText  = String.Empty;
                foreach (Rectangle rect in regions)
                {
                    using (Mat region = new Mat(image, rect)) {
                        _ocr.SetImage(region);
                        if (_ocr.Recognize() != 0)
                        {
                            return(null);
                        }
                        //var characters = _ocr.GetCharacters();

                        ////convert the coordinates from the local region to global
                        //for (int i = 0; i < characters.Length; i++) {
                        //    Rectangle charRegion = characters[i].Region;
                        //    charRegion.Offset(rect.Location);
                        //    characters[i].Region = charRegion;
                        //}
                        //allChars.AddRange(characters);
                        allText += _ocr.GetUTF8Text() + "|";
                    }
                }

                //Bgr drawRegionColor = new Bgr(Color.Red);
                //foreach (Rectangle rect in regions) {
                //    CvInvoke.Rectangle(image, rect, drawRegionColor.MCvScalar);
                //}
                //foreach (Tesseract.Character c in allChars) {
                //    CvInvoke.Rectangle(image, c.Region, drawCharColor.MCvScalar);
                //}

                return(allText);
            }
        }