public void FindARMarker(Mat imgMat)
        {
            PatternTrackingInfo patternTrackingInfo = new PatternTrackingInfo();

            foreach (string patternName in _Patterns.Keys)
            {
                bool patternFound = _PatternDetectors[patternName].findPattern(imgMat, patternTrackingInfo);
                // Debug.Log ("PatternFound " + patternFound);

                if (patternFound)
                {
                    patternTrackingInfo.computePose(_Patterns[patternName], _CamMatrix, _DistCoeffs);

                    Matrix4x4 transformationM = patternTrackingInfo.pose3d; // Marker to Camera Coordinate System Convert Matrix

                    Matrix4x4 scaleMat = _ARObjectScaleMatrix[patternName];
                    Matrix4x4 ARM      = ARCamera.transform.localToWorldMatrix * scaleMat * _InvertYMat * transformationM * _InvertZMat;

                    GameObject ARGameObject;
                    if (!_ARObjects.TryGetValue(patternName, out ARGameObject))
                    {
                        ARGameObject            = GameObject.Instantiate(ARObjectPrefab, Vector3.zero, Quaternion.identity);
                        ARGameObject.name       = ARObjectPrefab.name + "_" + patternName;
                        _ARObjects[patternName] = ARGameObject;

                        Material material = ARGameObject.GetComponentInChildren <MeshRenderer>().material;
                        material.mainTexture = _TextureImages[patternName];
                    }

                    ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
                    ARGameObject.transform.Rotate(ARObjLocalRotEuler);
                }
            }
        }
Пример #2
0
        void FindARMarker(Mat imgMat, Matrix4x4 cameraToWorldMatrix)
        {
            PatternTrackingInfo patternTrackingInfo = new PatternTrackingInfo();

            foreach (string patternName in _Patterns.Keys)
            {
                bool patternFound = _PatternDetectors[patternName].findPattern(imgMat, patternTrackingInfo);
                // Debug.Log ("PatternFound " + patternFound);

                if (patternFound)
                {
                    patternTrackingInfo.computePose(_Patterns[patternName], _CamMatrix, _DistCoeffs);

                    Matrix4x4 transformationM = patternTrackingInfo.pose3d; // Marker to Camera Coordinate System Convert Matrix

                    Matrix4x4 scaleMat = _ARObjectScaleMatrix[patternName];

                    // _ARObjectTransformMatrix[patternName] = cameraToWorldMatrix * scaleMat * _InvertYMat * transformationM * _InvertZMat;
                    _ARObjectTransformMatrix[patternName] = cameraToWorldMatrix * scaleMat * _InvertZMat * _InvertYMat * transformationM * _InvertZMat;

                    _ARObjectHasUpdate[patternName] = true;
                }
                else
                {
                    _ARObjectHasUpdate[patternName] = false;
                }
            }
        }
Пример #3
0
        static void VideoRun()
        {
            var cap    = new VideoCapture(@"rabit3V.mp4");
            Mat MARKER = new Mat("rabits.jpg");

            Cv2.Resize(MARKER, MARKER, new OpenCvSharp.Size(500, 500));
            Pattern             pattern             = new Pattern();
            PatternTrackingInfo patternTrackinginfo = new PatternTrackingInfo();
            PatternDetector     detector            = new PatternDetector(true);

            detector.buildPatternFromImage(MARKER, pattern);
            detector.train();



            int sleepTime = 1;

            using (Window window = new Window("capture"))
                using (Mat image = new Mat())
                {
                    while (true)
                    {
                        cap.Read(image);
                        if (image.Empty())
                        {
                            break;
                        }
                        var img = image.Clone();

                        if (detector.findPattern(img, patternTrackinginfo))
                        {
                            patternTrackinginfo.computePose(pattern);

                            var temp = patternTrackinginfo.campos.Get <Point3d>(0);

                            string camposInfo = "x:" + Math.Round(temp.X, 5) + "\ny:" + Math.Round(temp.Y, 5) + "\nz:" + Math.Round(temp.Z, 5);
                            Cv2.PutText(img,
                                        camposInfo,
                                        new OpenCvSharp.Point(0, 80),
                                        HersheyFonts.HersheyComplex,
                                        0.5,
                                        Scalar.White);

                            for (int i = 0; i < patternTrackinginfo.points2d.Rows; i++)
                            {
                                //Console.WriteLine(" x"+(int)patternTrackinginfo.points2d.Get<Point2d>(i).X+" "+ (int)patternTrackinginfo.points2d.Get<Point2d>(i).Y);
                                Cv2.Circle(img, (int)patternTrackinginfo.points2d.Get <Point2d>(i).X, (int)patternTrackinginfo.points2d.Get <Point2d>(i).Y, 5, Scalar.Black, 3);
                            }
                        }


                        window.ShowImage(img);
                        Cv2.WaitKey(sleepTime);
                        img.Release();
                    }
                }
            cap.Release();
        }
Пример #4
0
        private void EstimatePoseMarkerLess(GameObject ARGameObject)
        {
            patternTrackingInfo.computePose(pattern, camMatrix, distCoeffsMarkerLess, rgbMat);

            transformationM = patternTrackingInfo.pose3d;
            Matrix4x4 ARM = ARCamera.transform.localToWorldMatrix * invertYM * transformationM * invertYM;

            ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
            ARGameObject.SetActive(true);
        }
Пример #5
0
        // Update is called once per frame
        void Update()
        {
            if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                Mat rgbaMat = webCamTextureToMatHelper.GetMat();

                Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);


                bool patternFound = patternDetector.findPattern(grayMat, patternTrackingInfo);

                //Debug.Log ("patternFound " + patternFound);
                if (patternFound)
                {
                    patternTrackingInfo.computePose(pattern, camMatrix, distCoeffs);

                    //Marker to Camera Coordinate System Convert Matrix
                    Matrix4x4 transformationM = patternTrackingInfo.pose3d;


                    // right-handed coordinates system (OpenCV) to left-handed one (Unity)
                    // https://stackoverflow.com/questions/30234945/change-handedness-of-a-row-major-4x4-transformation-matrix
                    Matrix4x4 ARM = invertYM * transformationM * invertYM;

                    // Apply Y-axis and Z-axis refletion matrix. (Adjust the posture of the AR object)
                    ARM = ARM * invertYM * invertZM;

                    if (shouldMoveARCamera)
                    {
                        ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;

                        //Debug.Log("ARM " + ARM.ToString());

                        ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
                    }
                    else
                    {
                        ARM = ARCamera.transform.localToWorldMatrix * ARM;

                        //Debug.Log("ARM " + ARM.ToString());

                        ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
                    }

                    ARGameObject.GetComponent <DelayableSetActive>().SetActive(true);
                }
                else
                {
                    ARGameObject.GetComponent <DelayableSetActive>().SetActive(false, 0.5f);
                }

                Utils.fastMatToTexture2D(rgbaMat, texture);
            }
        }
Пример #6
0
        // Update is called once per frame
        void Update()
        {
            if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                Mat rgbaMat = webCamTextureToMatHelper.GetMat();

                Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);


                bool patternFound = patternDetector.findPattern(grayMat, patternTrackingInfo);

//              Debug.Log ("patternFound " + patternFound);
                if (patternFound)
                {
                    patternTrackingInfo.computePose(pattern, camMatrix, distCoeffs);

                    //Marker to Camera Coordinate System Convert Matrix
                    transformationM = patternTrackingInfo.pose3d;
                    //Debug.Log ("transformationM " + transformationM.ToString ());

                    if (shouldMoveARCamera)
                    {
                        ARM = ARGameObject.transform.localToWorldMatrix * invertZM * transformationM.inverse * invertYM;
                        //Debug.Log ("ARM " + ARM.ToString ());

                        ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
                    }
                    else
                    {
                        ARM = ARCamera.transform.localToWorldMatrix * invertYM * transformationM * invertZM;
                        //Debug.Log ("ARM " + ARM.ToString ());

                        ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
                    }

                    ARGameObject.GetComponent <DelayableSetActive> ().SetActive(true);
                }
                else
                {
                    ARGameObject.GetComponent <DelayableSetActive> ().SetActive(false, 0.5f);
                }


                Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
            }
        }
Пример #7
0
        static void imgRun()
        {
            string path   = @"D:\Code_Resource\IMAGE\";
            Mat    img    = new Mat(path + "rabit3.jpg");
            Mat    MARKER = new Mat(path + "rabits.jpg");

            Cv2.Resize(MARKER, MARKER, new OpenCvSharp.Size(500, 500));
            Pattern             pattern             = new Pattern();
            PatternTrackingInfo patternTrackinginfo = new PatternTrackingInfo();
            PatternDetector     detector            = new PatternDetector(true);

            detector.buildPatternFromImage(MARKER, pattern);
            detector.train();
            if (detector.findPattern(img, patternTrackinginfo))
            {
                patternTrackinginfo.computePose(pattern);

                var temp = patternTrackinginfo.campos.Get <Point3d>(0);

                string camposInfo = "x:" + Math.Round(temp.X, 5) + "y:" + Math.Round(temp.Y, 5) + "z:" + Math.Round(temp.Z, 5);
                Cv2.PutText(img,
                            camposInfo,
                            new OpenCvSharp.Point(0, 80),
                            HersheyFonts.HersheyComplex,
                            0.5,
                            Scalar.White);

                for (int i = 0; i < 4; i++)
                {
                    Cv2.Circle(img, (int)patternTrackinginfo.points2d.Get <Point2d>(i).X, (int)patternTrackinginfo.points2d.Get <Point2d>(i).Y, 5, Scalar.Black, 3);
                }
            }
            Cv2.ImShow("result", img);

            Cv2.WaitKey(100000);
            img.Release();
        }
        // Use this for initialization
        void Start()
        {
            Mat patternMat = new Mat(patternTexture.height, patternTexture.width, CvType.CV_8UC4);

            Utils.texture2DToMat(patternTexture, patternMat);
            Debug.Log("patternMat dst ToString " + patternMat.ToString());

            patternRawImage.texture = patternTexture;
            patternRawImage.rectTransform.localScale = new Vector3(1.0f, (float)patternMat.height() / (float)patternMat.width(), 1.0f);


            Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);

            Utils.texture2DToMat(imgTexture, imgMat);
            Debug.Log("imgMat dst ToString " + imgMat.ToString());

            gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
            Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);


            float width  = imgMat.width();
            float height = imgMat.height();

            float imageSizeScale = 1.0f;
            float widthScale     = (float)Screen.width / width;
            float heightScale    = (float)Screen.height / height;

            if (widthScale < heightScale)
            {
                Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
                imageSizeScale = (float)Screen.height / (float)Screen.width;
            }
            else
            {
                Camera.main.orthographicSize = height / 2;
            }


            //set cameraparam
            int    max_d     = (int)Mathf.Max(width, height);
            double fx        = max_d;
            double fy        = max_d;
            double cx        = width / 2.0f;
            double cy        = height / 2.0f;
            Mat    camMatrix = new Mat(3, 3, CvType.CV_64FC1);

            camMatrix.put(0, 0, fx);
            camMatrix.put(0, 1, 0);
            camMatrix.put(0, 2, cx);
            camMatrix.put(1, 0, 0);
            camMatrix.put(1, 1, fy);
            camMatrix.put(1, 2, cy);
            camMatrix.put(2, 0, 0);
            camMatrix.put(2, 1, 0);
            camMatrix.put(2, 2, 1.0f);
            Debug.Log("camMatrix " + camMatrix.dump());


            MatOfDouble distCoeffs = new MatOfDouble(0, 0, 0, 0);

            Debug.Log("distCoeffs " + distCoeffs.dump());


            //calibration camera
            Size   imageSize      = new Size(width * imageSizeScale, height * imageSizeScale);
            double apertureWidth  = 0;
            double apertureHeight = 0;

            double[] fovx           = new double[1];
            double[] fovy           = new double[1];
            double[] focalLength    = new double[1];
            Point    principalPoint = new Point(0, 0);

            double[] aspectratio = new double[1];

            Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);

            Debug.Log("imageSize " + imageSize.ToString());
            Debug.Log("apertureWidth " + apertureWidth);
            Debug.Log("apertureHeight " + apertureHeight);
            Debug.Log("fovx " + fovx [0]);
            Debug.Log("fovy " + fovy [0]);
            Debug.Log("focalLength " + focalLength [0]);
            Debug.Log("principalPoint " + principalPoint.ToString());
            Debug.Log("aspectratio " + aspectratio [0]);


            //To convert the difference of the FOV value of the OpenCV and Unity.
            double fovXScale = (2.0 * Mathf.Atan((float)(imageSize.width / (2.0 * fx)))) / (Mathf.Atan2((float)cx, (float)fx) + Mathf.Atan2((float)(imageSize.width - cx), (float)fx));
            double fovYScale = (2.0 * Mathf.Atan((float)(imageSize.height / (2.0 * fy)))) / (Mathf.Atan2((float)cy, (float)fy) + Mathf.Atan2((float)(imageSize.height - cy), (float)fy));

            Debug.Log("fovXScale " + fovXScale);
            Debug.Log("fovYScale " + fovYScale);


            //Adjust Unity Camera FOV https://github.com/opencv/opencv/commit/8ed1945ccd52501f5ab22bdec6aa1f91f1e2cfd4
            if (widthScale < heightScale)
            {
                ARCamera.fieldOfView = (float)(fovx [0] * fovXScale);
            }
            else
            {
                ARCamera.fieldOfView = (float)(fovy [0] * fovYScale);
            }



            //Learning the feature points of the pattern image.
            Pattern             pattern             = new Pattern();
            PatternTrackingInfo patternTrackingInfo = new PatternTrackingInfo();

            PatternDetector patternDetector = new PatternDetector(null, null, null, true);

            patternDetector.buildPatternFromImage(patternMat, pattern);
            patternDetector.train(pattern);



            bool patternFound = patternDetector.findPattern(imgMat, patternTrackingInfo);

            Debug.Log("patternFound " + patternFound);

            if (patternFound)
            {
                patternTrackingInfo.computePose(pattern, camMatrix, distCoeffs);

                Matrix4x4 transformationM = patternTrackingInfo.pose3d;
                Debug.Log("transformationM " + transformationM.ToString());

                Matrix4x4 invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
                Debug.Log("invertZM " + invertZM.ToString());

                Matrix4x4 invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
                Debug.Log("invertYM " + invertYM.ToString());


                if (shouldMoveARCamera)
                {
                    Matrix4x4 ARM = ARGameObject.transform.localToWorldMatrix * invertZM * transformationM.inverse * invertYM;
                    Debug.Log("ARM " + ARM.ToString());

                    ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
                }
                else
                {
                    Matrix4x4 ARM = ARCamera.transform.localToWorldMatrix * invertYM * transformationM * invertZM;
                    Debug.Log("ARM " + ARM.ToString());

                    ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
                }
            }

            Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false);

            Utils.matToTexture2D(imgMat, texture);

            gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
        }