Example #1
0
        private void DetectInRegion(Mat img, Rect region, List <Rect> detectedObjectsInRegions, FaceLandmarkDetector landmarkDetector)
        {
            Rect r0 = new Rect(new Point(), img.size());
            Rect r1 = new Rect(region.x, region.y, region.width, region.height);

            Rect.inflate(r1, (int)((r1.width * coeffTrackingWindowSize) - r1.width) / 2,
                         (int)((r1.height * coeffTrackingWindowSize) - r1.height) / 2);
            r1 = Rect.intersect(r0, r1);

            if ((r1.width <= 0) || (r1.height <= 0))
            {
                Debug.Log("detectInRegion: Empty intersection");
                return;
            }

            using (Mat img1_roi = new Mat(img, r1))
                using (Mat img1 = new Mat(r1.size(), img.type()))
                {
                    img1_roi.copyTo(img1);

                    OpenCVForUnityUtils.SetImage(landmarkDetector, img1);

                    List <UnityEngine.Rect> detectResult = landmarkDetector.Detect();

                    int len = detectResult.Count;
                    for (int i = 0; i < len; i++)
                    {
                        UnityEngine.Rect tmp = detectResult[i];
                        Rect             r   = new Rect((int)(tmp.x + r1.x), (int)(tmp.y + r1.y), (int)tmp.width, (int)tmp.height);
                        detectedObjectsInRegions.Add(r);
                    }
                }
        }
Example #2
0
        private void DetectObject(Mat img, out List <Rect> detectedObjects, FaceLandmarkDetector landmarkDetector)
        {
            OpenCVForUnityUtils.SetImage(landmarkDetector, img);

            List <UnityEngine.Rect> detectResult = landmarkDetector.Detect();

            detectedObjects = new List <Rect>();

            int len = detectResult.Count;

            for (int i = 0; i < len; i++)
            {
                UnityEngine.Rect r = detectResult[i];
                detectedObjects.Add(new Rect((int)r.x, (int)r.y, (int)r.width, (int)r.height));
            }
        }
Example #3
0
        void OnPhotoCaptured(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
        {
            Matrix4x4 cameraToWorldMatrix;

            photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
            Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;

            Matrix4x4 projectionMatrix;

            photoCaptureFrame.TryGetProjectionMatrix(out projectionMatrix);

            photoCaptureFrame.UploadImageDataToTexture(m_Texture);


            OpenCVForUnity.Utils.texture2DToMat(m_Texture, rgbaMat);

            // fill all black.
//            Imgproc.rectangle (rgbaMat, new Point (0, 0), new Point (rgbaMat.width (), rgbaMat.height ()), new Scalar (0, 0, 0, 0), -1);
            // draw an edge lines.
            Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(255, 0, 0, 255), 2);
            // draw a diagonal line.
//            Imgproc.line (rgbaMat, new Point (0, 0), new Point (rgbaMat.cols (), rgbaMat.rows ()), new Scalar (255, 0, 0, 255));


            OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);

            //detect face
            List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection();

            foreach (var r in detectResult)
            {
                Debug.Log("rect : " + r.rect);

                //detect landmark points
                List <Vector2> points = faceLandmarkDetector.DetectLandmark(r.rect);

                Debug.Log("face points count : " + points.Count);
                //draw landmark points
                OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);

                //draw face rect
                OpenCVForUnityUtils.DrawFaceRect(rgbaMat, r.rect, new Scalar(255, 0, 0, 255), 2);
            }


            Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.5, new Scalar(0, 255, 0, 255), 2, Imgproc.LINE_AA, false);

            OpenCVForUnity.Utils.matToTexture2D(rgbaMat, m_Texture, colors);



            m_Texture.wrapMode = TextureWrapMode.Clamp;

            m_CanvasRenderer.enabled = true;
            m_CanvasRenderer.sharedMaterial.SetTexture("_MainTex", m_Texture);
            m_CanvasRenderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
            m_CanvasRenderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix);
            m_CanvasRenderer.sharedMaterial.SetFloat("_VignetteScale", 0.0f);

            // Position the canvas object slightly in front
            // of the real world web camera.
            Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);

            // Rotate the canvas object so that it faces the user.
            Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));

            m_Canvas.transform.position = position;
            m_Canvas.transform.rotation = rotation;

            Debug.Log("Took picture!");
            Debug.Log("projectionMatrix:\n" + projectionMatrix.ToString());
            m_CapturingPhoto = false;
        }
Example #4
0
        // Update is called once per frame
        void Update()
        {
            if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());


                Mat rgbaMatClipROI = new Mat(rgbaMat, processingAreaRect);

                rgbaMatClipROI.copyTo(processingAreaMat);


                // fill all black.
                Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);


                OpenCVForUnityUtils.SetImage(faceLandmarkDetector, processingAreaMat);

                // detect faces.
                List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
                if (isUsingDlibFaceDetecter)
                {
                    List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();

                    foreach (var unityRect in result)
                    {
                        detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
                    }
                }
                else
                {
                    // convert image to greyscale.
                    Imgproc.cvtColor(processingAreaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);


                    Imgproc.equalizeHist(grayMat, grayMat);

                    cascade.detectMultiScale(grayMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(grayMat.cols() * 0.15, grayMat.cols() * 0.15), new Size());

                    detectResult = faces.toList();


                    // Adjust to Dilb's result.
                    foreach (OpenCVForUnity.Rect r in detectResult)
                    {
                        r.y += (int)(r.height * 0.1f);
                    }
                }


                foreach (var rect in detectResult)
                {
                    //detect landmark points
                    List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                    //draw landmark points
                    OpenCVForUnityUtils.DrawFaceLandmark(rgbaMatClipROI, points, new Scalar(0, 255, 0, 255), 2);

                    //draw face rect
                    OpenCVForUnityUtils.DrawFaceRect(rgbaMatClipROI, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
                }

                Imgproc.putText(rgbaMatClipROI, "W:" + rgbaMatClipROI.width() + " H:" + rgbaMatClipROI.height() + " SO:" + Screen.orientation, new Point(5, rgbaMatClipROI.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false);


                Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(255, 0, 0, 255), 2);

                // Draw prosessing area rectangle.
                Imgproc.rectangle(rgbaMat, processingAreaRect.tl(), processingAreaRect.br(), new Scalar(255, 255, 0, 255), 2);

                OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture);

                rgbaMatClipROI.Dispose();
            }

            if (webCamTextureToMatHelper.IsPlaying())
            {
                Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
                ;
                Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;

                texture.wrapMode = TextureWrapMode.Clamp;

                quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);

                // Position the canvas object slightly in front
                // of the real world web camera.
                Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);

                // Rotate the canvas object so that it faces the user.
                Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));

                gameObject.transform.position = position;
                gameObject.transform.rotation = rotation;
            }
        }
Example #5
0
        // Update is called once per frame
        void Update()
        {
            lock (sync) {
                while (ExecuteOnMainThread.Count > 0)
                {
                    ExecuteOnMainThread.Dequeue().Invoke();
                }
            }

            if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                Mat rgbaMat = imageOptimizationHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());

                Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
                Imgproc.equalizeHist(grayMat, grayMat);

                if (enableDetection && !isDetecting)
                {
                    isDetecting = true;

                    grayMat.copyTo(grayMat4Thread);

                    StartThread(ThreadWorker);
                }

                OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                Rect[] rects;
                if (!useSeparateDetection)
                {
                    if (hasUpdatedDetectionResult)
                    {
                        hasUpdatedDetectionResult = false;

                        rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
                    }

                    rectangleTracker.GetObjects(resultObjects, true);

                    rects = rectangleTracker.CreateCorrectionBySpeedOfRects();

                    if (rects.Length > 0)
                    {
                        OpenCVForUnity.Rect rect = rects [0];

                        // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
                        rect.y += (int)(rect.height * 0.1f);

                        //detect landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                        if (enableOpticalFlowFilter)
                        {
                            opticalFlowFilter.Process(rgbaMat, points, points, false);
                        }

                        UpdateARHeadTransform(points, arCamera.cameraToWorldMatrix);

                        if (displayCameraPreview)
                        {
                            //draw landmark points
                            OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
                        }
                    }
                }
                else
                {
                    if (hasUpdatedDetectionResult)
                    {
                        hasUpdatedDetectionResult = false;

                        //Debug.Log("process: get rectsWhereRegions were got from detectionResult");
                        rectsWhereRegions = detectionResult.toArray();
                    }
                    else
                    {
                        //Debug.Log("process: get rectsWhereRegions from previous positions");
                        rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
                    }

                    detectedObjectsInRegions.Clear();
                    if (rectsWhereRegions.Length > 0)
                    {
                        int len = rectsWhereRegions.Length;
                        for (int i = 0; i < len; i++)
                        {
                            DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
                        }
                    }

                    rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
                    rectangleTracker.GetObjects(resultObjects, true);

                    if (resultObjects.Count > 0)
                    {
                        OpenCVForUnity.Rect rect = resultObjects [0];

                        // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
                        rect.y += (int)(rect.height * 0.1f);

                        //detect landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                        if (enableOpticalFlowFilter)
                        {
                            opticalFlowFilter.Process(rgbaMat, points, points, false);
                        }

                        UpdateARHeadTransform(points, arCamera.cameraToWorldMatrix);

                        if (displayCameraPreview)
                        {
                            //draw landmark points
                            OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
                        }
                    }
                }

                if (displayCameraPreview)
                {
                    OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture);
                }
            }
        }
Example #6
0
        public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
        {
            Mat downScaleFrameMat = imageOptimizationHelper.GetDownScaleMat(bgraMat);

            Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY);
            Imgproc.equalizeHist(grayMat, grayMat);

            if (enableDetection && !isDetecting)
            {
                isDetecting = true;

                grayMat.copyTo(grayMat4Thread);

                System.Threading.Tasks.Task.Run(() => {
                    isThreadRunning = true;
                    DetectObject();
                    isThreadRunning = false;
                    OnDetectionDone();
                });
            }

            OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

            Mat bgraMat4preview = null;

            if (displayCameraPreview)
            {
                bgraMat4preview = new Mat();
                downScaleFrameMat.copyTo(bgraMat4preview);
            }

            List <Vector2> points = null;

            Rect[] rects;
            if (!useSeparateDetection)
            {
                if (hasUpdatedDetectionResult)
                {
                    hasUpdatedDetectionResult = false;

                    lock (rectangleTracker) {
                        rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
                    }
                }

                lock (rectangleTracker) {
                    rectangleTracker.GetObjects(resultObjects, true);
                }
                rects = resultObjects.ToArray();

                if (rects.Length > 0)
                {
                    OpenCVForUnity.Rect rect = rects [0];

                    // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
                    rect.y += (int)(rect.height * 0.1f);

                    //detect landmark points
                    points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                    if (enableOpticalFlowFilter)
                    {
                        opticalFlowFilter.Process(bgraMat, points, points, false);
                    }

                    if (displayCameraPreview && bgraMat4preview != null)
                    {
                        //draw landmark points
                        OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2);
                    }
                }
            }
            else
            {
                if (hasUpdatedDetectionResult)
                {
                    hasUpdatedDetectionResult = false;

                    //UnityEngine.WSA.Application.InvokeOnAppThread (() => {
                    //    Debug.Log("process: get rectsWhereRegions were got from detectionResult");
                    //}, true);

                    lock (rectangleTracker) {
                        rectsWhereRegions = detectionResult.toArray();
                    }
                }
                else
                {
                    //UnityEngine.WSA.Application.InvokeOnAppThread (() => {
                    //    Debug.Log("process: get rectsWhereRegions from previous positions");
                    //}, true);

                    lock (rectangleTracker) {
                        rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
                    }
                }

                detectedObjectsInRegions.Clear();
                if (rectsWhereRegions.Length > 0)
                {
                    int len = rectsWhereRegions.Length;
                    for (int i = 0; i < len; i++)
                    {
                        DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
                    }
                }

                lock (rectangleTracker) {
                    rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
                    rectangleTracker.GetObjects(resultObjects, true);
                }

                if (resultObjects.Count > 0)
                {
                    OpenCVForUnity.Rect rect = resultObjects [0];

                    // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
                    rect.y += (int)(rect.height * 0.1f);

                    //detect landmark points
                    points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                    if (enableOpticalFlowFilter)
                    {
                        opticalFlowFilter.Process(bgraMat, points, points, false);
                    }

                    if (displayCameraPreview && bgraMat4preview != null)
                    {
                        //draw landmark points
                        OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2);
                    }
                }
            }


            UnityEngine.WSA.Application.InvokeOnAppThread(() => {
                if (!webCamTextureToMatHelper.IsPlaying())
                {
                    return;
                }

                if (displayCameraPreview && bgraMat4preview != null)
                {
                    OpenCVForUnity.Utils.fastMatToTexture2D(bgraMat4preview, texture);
                }

                if (points != null)
                {
                    UpdateARHeadTransform(points, cameraToWorldMatrix);
                }

                bgraMat.Dispose();
                if (bgraMat4preview != null)
                {
                    bgraMat4preview.Dispose();
                }
            }, false);
        }
Example #7
0
        // Update is called once per frame
        void Update()
        {
            lock (sync) {
                while (ExecuteOnMainThread.Count > 0)
                {
                    ExecuteOnMainThread.Dequeue().Invoke();
                }
            }

            if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());

                Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
                Imgproc.equalizeHist(grayMat, grayMat);

                if (enable && !detecting)
                {
                    detecting = true;

                    grayMat.copyTo(grayMat4Thread);

                    StartThread(ThreadWorker);
                }

                OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                Rect[] rects;
                if (!isUsingSeparateDetection)
                {
                    if (didUpdateTheDetectionResult)
                    {
                        didUpdateTheDetectionResult = false;

                        rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
                    }

                    rectangleTracker.GetObjects(resultObjects, true);

                    rects = rectangleTracker.CreateCorrectionBySpeedOfRects();

                    if (rects.Length > 0)
                    {
                        OpenCVForUnity.Rect rect = rects [0];

                        // Adjust to Dilb's result.
                        rect.y += (int)(rect.height * 0.1f);

                        //detect landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                        UpdateARHeadTransform(points);
                    }
                }
                else
                {
                    if (didUpdateTheDetectionResult)
                    {
                        didUpdateTheDetectionResult = false;

                        //Debug.Log("process: get rectsWhereRegions were got from detectionResult");
                        rectsWhereRegions = detectionResult.toArray();
                    }
                    else
                    {
                        //Debug.Log("process: get rectsWhereRegions from previous positions");
                        rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
                    }

                    detectedObjectsInRegions.Clear();
                    if (rectsWhereRegions.Length > 0)
                    {
                        int len = rectsWhereRegions.Length;
                        for (int i = 0; i < len; i++)
                        {
                            detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
                        }
                    }

                    rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
                    rectangleTracker.GetObjects(resultObjects, true);

                    if (resultObjects.Count > 0)
                    {
                        OpenCVForUnity.Rect rect = resultObjects [0];

                        // Adjust to Dilb's result.
                        rect.y += (int)(rect.height * 0.1f);

                        //detect landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                        UpdateARHeadTransform(points);
                    }
                }
            }
        }
Example #8
0
        // Update is called once per frame
        void Update()
        {
            lock (ExecuteOnMainThread)
            {
                while (ExecuteOnMainThread.Count > 0)
                {
                    ExecuteOnMainThread.Dequeue().Invoke();
                }
            }

            if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                Mat rgbaMat = webCamTextureToMatHelper.GetMat();
                Imgproc.cvtColor(rgbaMat, grayMat, CVTCOLOR_CODE);

                Mat downScaleGrayMat = imageOptimizationHelper.GetDownScaleMat(grayMat);

                if (useOpenCVDetector)
                {
                    Imgproc.equalizeHist(downScaleGrayMat, downScaleGrayMat);
                }

                if (enableDetection && !isDetecting)
                {
                    isDetecting = true;

                    downScaleGrayMat.copyTo(grayMat4Thread);

                    StartThread(ThreadWorker);
                }

                if (!displayCameraImage)
                {
                    // fill all black
                    Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
                }

                if (!useSeparateDetection)
                {
                    if (hasUpdatedDetectionResult)
                    {
                        hasUpdatedDetectionResult = false;

                        rectangleTracker.UpdateTrackedObjects(detectionResult);
                    }

                    rectangleTracker.GetObjects(resultObjects, true);

                    // set original size image
                    OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                    int len = resultObjects.Count;
                    for (int i = 0; i < len; i++)
                    {
                        Rect rect = resultObjects[i];

                        // restore to original size rect
                        float downscaleRatio = imageOptimizationHelper.downscaleRatio;
                        rect.x      = (int)(rect.x * downscaleRatio);
                        rect.y      = (int)(rect.y * downscaleRatio);
                        rect.width  = (int)(rect.width * downscaleRatio);
                        rect.height = (int)(rect.height * downscaleRatio);

                        // detect face landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                        // draw face landmark points
                        OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);

                        if (displayDetectedFaceRect)
                        {
                            // draw face rect
                            OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
                        }
                    }
                }
                else
                {
                    Rect[] rectsWhereRegions;

                    if (hasUpdatedDetectionResult)
                    {
                        hasUpdatedDetectionResult = false;

                        //Debug.Log("process: get rectsWhereRegions were got from detectionResult");
                        rectsWhereRegions = detectionResult.ToArray();

                        if (displayDetectedFaceRect)
                        {
                            DrawDownScaleFaceRects(rgbaMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_BLUE, 1);
                        }
                    }
                    else
                    {
                        //Debug.Log("process: get rectsWhereRegions from previous positions");
                        if (useOpenCVDetector)
                        {
                            rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
                        }
                        else
                        {
                            rectsWhereRegions = rectangleTracker.CreateRawRects();
                        }

                        if (displayDetectedFaceRect)
                        {
                            DrawDownScaleFaceRects(rgbaMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_GREEN, 1);
                        }
                    }

                    detectedObjectsInRegions.Clear();
                    int len = rectsWhereRegions.Length;
                    for (int i = 0; i < len; i++)
                    {
                        if (useOpenCVDetector)
                        {
                            DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade, true);
                        }
                        else
                        {
                            DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, faceLandmarkDetector);
                        }
                    }

                    rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
                    rectangleTracker.GetObjects(resultObjects, false);

                    // set original size image
                    OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                    len = resultObjects.Count;
                    for (int i = 0; i < len; i++)
                    {
                        Rect rect = resultObjects[i];

                        // restore to original size rect
                        float downscaleRatio = imageOptimizationHelper.downscaleRatio;
                        rect.x      = (int)(rect.x * downscaleRatio);
                        rect.y      = (int)(rect.y * downscaleRatio);
                        rect.width  = (int)(rect.width * downscaleRatio);
                        rect.height = (int)(rect.height * downscaleRatio);

                        // detect face landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                        // draw face landmark points
                        OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);

                        if (displayDetectedFaceRect)
                        {
                            //draw face rect
                            OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
                        }
                    }
                }

                Utils.fastMatToTexture2D(rgbaMat, texture);
            }

            if (webCamTextureToMatHelper.IsPlaying())
            {
                Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
                Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;

                quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);

                // Position the canvas object slightly in front
                // of the real world web camera.
                Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;

                // Rotate the canvas object so that it faces the user.
                Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));

                gameObject.transform.position = position;
                gameObject.transform.rotation = rotation;
            }
        }
Example #9
0
        public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
        {
            Imgproc.cvtColor(bgraMat, grayMat, CVTCOLOR_CODE);

            Mat downScaleGrayMat = imageOptimizationHelper.GetDownScaleMat(grayMat);

            if (useOpenCVDetector)
            {
                Imgproc.equalizeHist(downScaleGrayMat, downScaleGrayMat);
            }

            if (enableDetection && !isDetecting)
            {
                isDetecting = true;

                downScaleGrayMat.copyTo(grayMat4Thread);

                System.Threading.Tasks.Task.Run(() =>
                {
                    isThreadRunning = true;

                    if (useOpenCVDetector)
                    {
                        DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true);
                    }
                    else
                    {
                        DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread);
                    }

                    isThreadRunning = false;
                    OnDetectionDone();
                });
            }


            if (!displayCameraImage)
            {
                // fill all black.
                Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1);
            }


            if (!useSeparateDetection)
            {
                if (hasUpdatedDetectionResult)
                {
                    hasUpdatedDetectionResult = false;

                    lock (rectangleTracker)
                    {
                        rectangleTracker.UpdateTrackedObjects(detectionResult);
                    }
                }

                lock (rectangleTracker)
                {
                    rectangleTracker.GetObjects(resultObjects, true);
                }

                // set original size image
                OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                int len = resultObjects.Count;
                for (int i = 0; i < len; i++)
                {
                    Rect rect = resultObjects[i];

                    // restore to original size rect
                    float downscaleRatio = imageOptimizationHelper.downscaleRatio;
                    rect.x      = (int)(rect.x * downscaleRatio);
                    rect.y      = (int)(rect.y * downscaleRatio);
                    rect.width  = (int)(rect.width * downscaleRatio);
                    rect.height = (int)(rect.height * downscaleRatio);

                    // detect face landmark points
                    List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                    // draw face landmark points
                    OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2);

                    if (displayDetectedFaceRect)
                    {
                        // draw face rect
                        OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
                    }
                }
            }
            else
            {
                Rect[] rectsWhereRegions;

                if (hasUpdatedDetectionResult)
                {
                    hasUpdatedDetectionResult = false;

                    //UnityEngine.WSA.Application.InvokeOnAppThread (() => {
                    //    Debug.Log("process: get rectsWhereRegions were got from detectionResult");
                    //}, true);

                    lock (rectangleTracker)
                    {
                        rectsWhereRegions = detectionResult.ToArray();
                    }

                    if (displayDetectedFaceRect)
                    {
                        DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_BLUE, 1);
                    }
                }
                else
                {
                    //UnityEngine.WSA.Application.InvokeOnAppThread (() => {
                    //    Debug.Log("process: get rectsWhereRegions from previous positions");
                    //}, true);

                    if (useOpenCVDetector)
                    {
                        lock (rectangleTracker)
                        {
                            rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
                        }
                    }
                    else
                    {
                        lock (rectangleTracker)
                        {
                            rectsWhereRegions = rectangleTracker.CreateRawRects();
                        }
                    }

                    if (displayDetectedFaceRect)
                    {
                        DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_GREEN, 1);
                    }
                }

                detectedObjectsInRegions.Clear();
                int len = rectsWhereRegions.Length;
                for (int i = 0; i < len; i++)
                {
                    if (useOpenCVDetector)
                    {
                        DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade, true);
                    }
                    else
                    {
                        DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, faceLandmarkDetector);
                    }
                }

                lock (rectangleTracker)
                {
                    rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
                    rectangleTracker.GetObjects(resultObjects, false);
                }

                // set original size image
                OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                len = resultObjects.Count;
                for (int i = 0; i < len; i++)
                {
                    Rect rect = resultObjects[i];

                    // restore to original size rect
                    float downscaleRatio = imageOptimizationHelper.downscaleRatio;
                    rect.x      = (int)(rect.x * downscaleRatio);
                    rect.y      = (int)(rect.y * downscaleRatio);
                    rect.width  = (int)(rect.width * downscaleRatio);
                    rect.height = (int)(rect.height * downscaleRatio);

                    // detect face landmark points
                    List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                    // draw face landmark points
                    OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2);

                    if (displayDetectedFaceRect)
                    {
                        // draw face rect
                        OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
                    }
                }
            }


            Enqueue(() =>
            {
                if (!webCamTextureToMatHelper.IsPlaying())
                {
                    return;
                }

                Utils.fastMatToTexture2D(bgraMat, texture);
                bgraMat.Dispose();

                Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;

                quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);

                // Position the canvas object slightly in front
                // of the real world web camera.
                Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;

                // Rotate the canvas object so that it faces the user.
                Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));

                gameObject.transform.position = position;
                gameObject.transform.rotation = rotation;
            });
        }
Example #10
0
        // Update is called once per frame
        void Update()
        {
            lock (sync) {
                while (ExecuteOnMainThread.Count > 0)
                {
                    ExecuteOnMainThread.Dequeue().Invoke();
                }
            }

            if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());

                Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
                Imgproc.equalizeHist(grayMat, grayMat);

                if (enable && !detecting)
                {
                    detecting = true;

                    grayMat.copyTo(grayMat4Thread);

                    StartThread(ThreadWorker);
                }

                OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                if (!isShowingWebCamImage)
                {
                    // fill all black.
                    Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
                }

                Rect[] rects;
                if (!isUsingSeparateDetection)
                {
                    if (didUpdateTheDetectionResult)
                    {
                        didUpdateTheDetectionResult = false;

                        rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
                    }

                    rectangleTracker.GetObjects(resultObjects, true);
//                    rects = resultObjects.ToArray ();

                    rects = rectangleTracker.CreateCorrectionBySpeedOfRects();

                    if (isShowingDetectedFaceRect)
                    {
                        for (int i = 0; i < rects.Length; i++)
                        {
                            //Debug.Log ("detected face[" + i + "] " + rects [i]);
                            Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
                        }
                    }

                    // Adjust to Dilb's result.
                    foreach (OpenCVForUnity.Rect r in resultObjects)
                    {
                        r.y += (int)(r.height * 0.1f);
                    }

                    foreach (var rect in resultObjects)
                    {
                        //detect landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                        //draw landmark points
                        OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);

                        if (isShowingDetectedFaceRect)
                        {
                            //draw face rect
                            OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
                        }
                    }
                }
                else
                {
                    if (didUpdateTheDetectionResult)
                    {
                        didUpdateTheDetectionResult = false;

                        //Debug.Log("process: get rectsWhereRegions were got from detectionResult");
                        rectsWhereRegions = detectionResult.toArray();

                        if (isShowingDetectedFaceRect)
                        {
                            rects = rectsWhereRegions;
                            for (int i = 0; i < rects.Length; i++)
                            {
                                Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 1);
                            }
                        }
                    }
                    else
                    {
                        //Debug.Log("process: get rectsWhereRegions from previous positions");
                        rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();

                        if (isShowingDetectedFaceRect)
                        {
                            rects = rectsWhereRegions;
                            for (int i = 0; i < rects.Length; i++)
                            {
                                Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1);
                            }
                        }
                    }

                    detectedObjectsInRegions.Clear();
                    if (rectsWhereRegions.Length > 0)
                    {
                        int len = rectsWhereRegions.Length;
                        for (int i = 0; i < len; i++)
                        {
                            detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
                        }
                    }

                    rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
                    rectangleTracker.GetObjects(resultObjects, true);

                    if (isShowingDetectedFaceRect)
                    {
                        rects = resultObjects.ToArray();
                        for (int i = 0; i < rects.Length; i++)
                        {
                            //Debug.Log ("detected face[" + i + "] " + rects [i]);
                            Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
                        }
                    }

                    // Adjust to Dilb's result.
                    foreach (OpenCVForUnity.Rect r in resultObjects)
                    {
                        r.y += (int)(r.height * 0.1f);
                    }

                    foreach (var rect in resultObjects)
                    {
                        //detect landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                        //draw landmark points
                        OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);

                        if (isShowingDetectedFaceRect)
                        {
                            //draw face rect
                            OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
                        }
                    }
                }

                OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
            }

            if (webCamTextureToMatHelper.IsPlaying())
            {
                Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
                Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;

                texture.wrapMode = TextureWrapMode.Clamp;

                quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);

                // Position the canvas object slightly in front
                // of the real world web camera.
                Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);

                // Rotate the canvas object so that it faces the user.
                Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));

                gameObject.transform.position = position;
                gameObject.transform.rotation = rotation;
            }
        }
        // Update is called once per frame
        void Update()
        {
            lock (ExecuteOnMainThread)
            {
                while (ExecuteOnMainThread.Count > 0)
                {
                    ExecuteOnMainThread.Dequeue().Invoke();
                }
            }

            if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                DebugUtils.VideoTick();

                Mat grayMat = webCamTextureToMatHelper.GetMat();

                Mat   downScaleMat = null;
                float DOWNSCALE_RATIO;
                if (enableDownScale)
                {
                    downScaleMat    = imageOptimizationHelper.GetDownScaleMat(grayMat);
                    DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
                }
                else
                {
                    downScaleMat    = grayMat;
                    DOWNSCALE_RATIO = 1.0f;
                }

                if (useOpenCVDetector)
                {
                    Imgproc.equalizeHist(downScaleMat, downScaleMat);
                }

                if (enableDetection && !isDetecting)
                {
                    isDetecting = true;

                    downScaleMat.copyTo(grayMat4Thread);

                    StartThread(ThreadWorker);
                }

                if (!useSeparateDetection)
                {
                    if (hasUpdatedDetectionResult)
                    {
                        hasUpdatedDetectionResult = false;

                        rectangleTracker.UpdateTrackedObjects(detectionResult);
                    }

                    rectangleTracker.GetObjects(resultObjects, true);

                    // set original size image
                    OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                    resultFaceLandmarkPoints.Clear();
                    foreach (Rect rect in resultObjects)
                    {
                        // restore to original size rect
                        rect.x      = (int)(rect.x * DOWNSCALE_RATIO);
                        rect.y      = (int)(rect.y * DOWNSCALE_RATIO);
                        rect.width  = (int)(rect.width * DOWNSCALE_RATIO);
                        rect.height = (int)(rect.height * DOWNSCALE_RATIO);

                        // detect face landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
                        resultFaceLandmarkPoints.Add(points);
                    }

                    if (!displayCameraImage)
                    {
                        // fill all black.
                        Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
                    }

                    if (displayDetectedFaceRect)
                    {
                        // draw face rects
                        foreach (Rect rect in resultObjects)
                        {
                            OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2);
                        }
                    }

                    // draw face landmark points
                    foreach (List <Vector2> points in resultFaceLandmarkPoints)
                    {
                        OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4);
                    }
                }
                else
                {
                    Rect[] rectsWhereRegions;

                    if (hasUpdatedDetectionResult)
                    {
                        hasUpdatedDetectionResult = false;

                        //Debug.Log("process: get rectsWhereRegions were got from detectionResult");
                        rectsWhereRegions = detectionResult.ToArray();
                    }
                    else
                    {
                        //Debug.Log("process: get rectsWhereRegions from previous positions");
                        if (useOpenCVDetector)
                        {
                            rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
                        }
                        else
                        {
                            rectsWhereRegions = rectangleTracker.CreateRawRects();
                        }
                    }

                    detectedObjectsInRegions.Clear();
                    foreach (Rect rect in rectsWhereRegions)
                    {
                        if (useOpenCVDetector)
                        {
                            DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, cascade, true);
                        }
                        else
                        {
                            DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, faceLandmarkDetector);
                        }
                    }

                    rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
                    rectangleTracker.GetObjects(resultObjects, false);

                    // set original size image
                    OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

                    resultFaceLandmarkPoints.Clear();
                    foreach (Rect rect in resultObjects)
                    {
                        // restore to original size rect
                        rect.x      = (int)(rect.x * DOWNSCALE_RATIO);
                        rect.y      = (int)(rect.y * DOWNSCALE_RATIO);
                        rect.width  = (int)(rect.width * DOWNSCALE_RATIO);
                        rect.height = (int)(rect.height * DOWNSCALE_RATIO);

                        // detect face landmark points
                        List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
                        resultFaceLandmarkPoints.Add(points);
                    }

                    if (!displayCameraImage)
                    {
                        // fill all black.
                        Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
                    }

                    if (displayDetectedFaceRect)
                    {
                        // draw previous rects
                        DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1);

                        // draw face rects
                        foreach (Rect rect in resultObjects)
                        {
                            OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2);
                        }
                    }

                    // draw face landmark points
                    foreach (List <Vector2> points in resultFaceLandmarkPoints)
                    {
                        OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4);
                    }
                }

                DebugUtils.TrackTick();

                Utils.fastMatToTexture2D(grayMat, texture);
            }

            if (webCamTextureToMatHelper.IsPlaying())
            {
                Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
                Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;

                quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);

                // Position the canvas object slightly in front
                // of the real world web camera.
                Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;

                // Rotate the canvas object so that it faces the user.
                Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));

                gameObject.transform.position = position;
                gameObject.transform.rotation = rotation;
            }
        }
Example #12
0
        public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
        {
            Mat downScaleFrameMat = bgraMat;

            if (imageOptimizationHelper.downscaleRatio != 1)
            {
                downScaleFrameMat = new Mat();
                Imgproc.resize(bgraMat, downScaleFrameMat, new Size(), 1.0 / imageOptimizationHelper.downscaleRatio, 1.0 / imageOptimizationHelper.downscaleRatio, Imgproc.INTER_LINEAR);
            }

            Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY);
            Imgproc.equalizeHist(grayMat, grayMat);

            if (enableDetection && !isDetecting)
            {
                isDetecting = true;

                grayMat.copyTo(grayMat4Thread);

                System.Threading.Tasks.Task.Run(() => {
                    isThreadRunning = true;
                    DetectObject();
                    isThreadRunning = false;
                    OnDetectionDone();
                });
            }

            OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);

            if (!displayCameraImage)
            {
                // fill all black.
                Imgproc.rectangle(downScaleFrameMat, new Point(0, 0), new Point(downScaleFrameMat.width(), downScaleFrameMat.height()), new Scalar(0, 0, 0, 0), -1);
            }


            Rect[] rects;
            if (!useSeparateDetection)
            {
                if (hasUpdatedDetectionResult)
                {
                    hasUpdatedDetectionResult = false;

                    lock (rectangleTracker) {
                        rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
                    }
                }

                lock (rectangleTracker) {
                    rectangleTracker.GetObjects(resultObjects, true);
                }
                rects = resultObjects.ToArray();

                if (displayDetectedFaceRect)
                {
                    for (int i = 0; i < rects.Length; i++)
                    {
                        //UnityEngine.WSA.Application.InvokeOnAppThread (() => {
                        //    Debug.Log ("detected face[" + i + "] " + rects [i]);
                        //}, true);

                        Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3);
                    }
                }

                // Adjust to Dilb's result.
                foreach (OpenCVForUnity.Rect r in resultObjects)
                {
                    r.y += (int)(r.height * 0.1f);
                }

                foreach (var rect in resultObjects)
                {
                    //detect landmark points
                    List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                    //draw landmark points
                    OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2);

                    if (displayDetectedFaceRect)
                    {
                        //draw face rect
                        OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2);
                    }
                }
            }
            else
            {
                if (hasUpdatedDetectionResult)
                {
                    hasUpdatedDetectionResult = false;

                    //UnityEngine.WSA.Application.InvokeOnAppThread (() => {
                    //    Debug.Log("process: get rectsWhereRegions were got from detectionResult");
                    //}, true);

                    lock (rectangleTracker) {
                        rectsWhereRegions = detectionResult.toArray();
                    }

                    if (displayDetectedFaceRect)
                    {
                        rects = rectsWhereRegions;
                        for (int i = 0; i < rects.Length; i++)
                        {
                            Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
                        }
                    }
                }
                else
                {
                    //UnityEngine.WSA.Application.InvokeOnAppThread (() => {
                    //    Debug.Log("process: get rectsWhereRegions from previous positions");
                    //}, true);

                    lock (rectangleTracker) {
                        rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
                    }

                    if (displayDetectedFaceRect)
                    {
                        rects = rectsWhereRegions;
                        for (int i = 0; i < rects.Length; i++)
                        {
                            Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1);
                        }
                    }
                }

                detectedObjectsInRegions.Clear();
                if (rectsWhereRegions.Length > 0)
                {
                    int len = rectsWhereRegions.Length;
                    for (int i = 0; i < len; i++)
                    {
                        DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
                    }
                }

                lock (rectangleTracker) {
                    rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
                    rectangleTracker.GetObjects(resultObjects, true);
                }

                if (displayDetectedFaceRect)
                {
                    rects = resultObjects.ToArray();
                    for (int i = 0; i < rects.Length; i++)
                    {
                        //UnityEngine.WSA.Application.InvokeOnAppThread (() => {
                        //    Debug.Log ("detected face[" + i + "] " + rects [i]);
                        //}, true);

                        Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3);
                    }
                }

                // Adjust to Dilb's result.
                foreach (OpenCVForUnity.Rect r in resultObjects)
                {
                    r.y += (int)(r.height * 0.1f);
                }

                foreach (var rect in resultObjects)
                {
                    //detect landmark points
                    List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));

                    //draw landmark points
                    OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2);

                    if (displayDetectedFaceRect)
                    {
                        //draw face rect
                        OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2);
                    }
                }
            }


            UnityEngine.WSA.Application.InvokeOnAppThread(() => {
                if (!webCamTextureToMatHelper.IsPlaying())
                {
                    return;
                }

                OpenCVForUnity.Utils.fastMatToTexture2D(downScaleFrameMat, texture);
                bgraMat.Dispose();
                if (imageOptimizationHelper.downscaleRatio != 1)
                {
                    downScaleFrameMat.Dispose();
                }

                Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;

                quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);

                // Position the canvas object slightly in front
                // of the real world web camera.
                Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);

                // Rotate the canvas object so that it faces the user.
                Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));

                gameObject.transform.position = position;
                gameObject.transform.rotation = rotation;
            }, false);
        }