// Update is called once per frame void Update() { lock (ExecuteOnMainThread) { while (ExecuteOnMainThread.Count > 0) { ExecuteOnMainThread.Dequeue().Invoke(); } } if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) { DebugUtils.VideoTick(); Mat grayMat = webCamTextureToMatHelper.GetMat(); Mat downScaleMat = null; float DOWNSCALE_RATIO; if (enableDownScale) { downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat); DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio; } else { downScaleMat = grayMat; DOWNSCALE_RATIO = 1.0f; } if (useOpenCVDetector) { Imgproc.equalizeHist(downScaleMat, downScaleMat); } if (enableDetection && !isDetecting) { isDetecting = true; downScaleMat.copyTo(grayMat4Thread); StartThread(ThreadWorker); } if (!useSeparateDetection) { if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; rectangleTracker.UpdateTrackedObjects(detectionResult); } rectangleTracker.GetObjects(resultObjects, true); // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); resultFaceLandmarkPoints.Clear(); foreach (Rect rect in resultObjects) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); resultFaceLandmarkPoints.Add(points); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } if (displayDetectedFaceRect) { // draw face rects foreach (Rect rect in resultObjects) { OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2); } } // draw face landmark points foreach (List <Vector2> points in resultFaceLandmarkPoints) { OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4); } } else { Rect[] rectsWhereRegions; if (hasUpdatedDetectionResult) { hasUpdatedDetectionResult = false; //Debug.Log("process: get rectsWhereRegions were got from detectionResult"); rectsWhereRegions = detectionResult.ToArray(); } else { //Debug.Log("process: get rectsWhereRegions from previous positions"); if (useOpenCVDetector) { rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects(); } else { rectsWhereRegions = rectangleTracker.CreateRawRects(); } } detectedObjectsInRegions.Clear(); foreach (Rect rect in rectsWhereRegions) { if (useOpenCVDetector) { DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, cascade, true); } else { DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, faceLandmarkDetector); } } rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions); rectangleTracker.GetObjects(resultObjects, false); // set original size image OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat); resultFaceLandmarkPoints.Clear(); foreach (Rect rect in resultObjects) { // restore to original size rect rect.x = (int)(rect.x * DOWNSCALE_RATIO); rect.y = (int)(rect.y * DOWNSCALE_RATIO); rect.width = (int)(rect.width * DOWNSCALE_RATIO); rect.height = (int)(rect.height * DOWNSCALE_RATIO); // detect face landmark points List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height)); resultFaceLandmarkPoints.Add(points); } if (!displayCameraImage) { // fill all black. Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1); } if (displayDetectedFaceRect) { // draw previous rects DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1); // draw face rects foreach (Rect rect in resultObjects) { OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2); } } // draw face landmark points foreach (List <Vector2> points in resultFaceLandmarkPoints) { OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4); } } DebugUtils.TrackTick(); Utils.fastMatToTexture2D(grayMat, texture); } if (webCamTextureToMatHelper.IsPlaying()) { Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix; Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse; quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix); // Position the canvas object slightly in front // of the real world web camera. Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f; // Rotate the canvas object so that it faces the user. Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1)); gameObject.transform.position = position; gameObject.transform.rotation = rotation; } }
/// <summary> /// Raises the web cam texture to mat helper initialized event. /// </summary> public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat grayMat = webCamTextureToMatHelper.GetMat(); texture = new Texture2D(grayMat.cols(), grayMat.rows(), TextureFormat.Alpha8, false); texture.wrapMode = TextureWrapMode.Clamp; quad_renderer = gameObject.GetComponent <Renderer>() as Renderer; quad_renderer.sharedMaterial.SetTexture("_MainTex", texture); quad_renderer.sharedMaterial.SetVector("_VignetteOffset", new Vector4(0, 0)); //Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); DebugUtils.AddDebugStr(webCamTextureToMatHelper.GetWidth() + " x " + webCamTextureToMatHelper.GetHeight() + " : " + webCamTextureToMatHelper.GetFPS()); if (enableDownScale) { DebugUtils.AddDebugStr("enableDownScale = true: " + imageOptimizationHelper.downscaleRatio + " / " + webCamTextureToMatHelper.GetWidth() / imageOptimizationHelper.downscaleRatio + " x " + webCamTextureToMatHelper.GetHeight() / imageOptimizationHelper.downscaleRatio); } Matrix4x4 projectionMatrix; #if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API projectionMatrix = webCamTextureToMatHelper.GetProjectionMatrix(); quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #else //This value is obtained from PhotoCapture's TryGetProjectionMatrix() method.I do not know whether this method is good. //Please see the discussion of this thread.Https://forums.hololens.com/discussion/782/live-stream-of-locatable-camera-webcam-in-unity projectionMatrix = Matrix4x4.identity; projectionMatrix.m00 = 2.31029f; projectionMatrix.m01 = 0.00000f; projectionMatrix.m02 = 0.09614f; projectionMatrix.m03 = 0.00000f; projectionMatrix.m10 = 0.00000f; projectionMatrix.m11 = 4.10427f; projectionMatrix.m12 = -0.06231f; projectionMatrix.m13 = 0.00000f; projectionMatrix.m20 = 0.00000f; projectionMatrix.m21 = 0.00000f; projectionMatrix.m22 = -1.00000f; projectionMatrix.m23 = 0.00000f; projectionMatrix.m30 = 0.00000f; projectionMatrix.m31 = 0.00000f; projectionMatrix.m32 = -1.00000f; projectionMatrix.m33 = 0.00000f; quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix); #endif quad_renderer.sharedMaterial.SetFloat("_VignetteScale", 0.0f); cascade = new CascadeClassifier(); cascade.load(Utils.getFilePath("lbpcascade_frontalface.xml")); #if !UNITY_WSA_10_0 || UNITY_EDITOR // "empty" method is not working on the UWP platform. if (cascade.empty()) { Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } #endif grayMat4Thread = new Mat(); cascade4Thread = new CascadeClassifier(); cascade4Thread.load(Utils.getFilePath("haarcascade_frontalface_alt.xml")); #if !UNITY_WSA_10_0 || UNITY_EDITOR // "empty" method is not working on the UWP platform. if (cascade4Thread.empty()) { Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } #endif }