/// <summary>
/// Raises the web cam texture to mat helper initialized event.
/// </summary>
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat webCamTextureMat = imageOptimizationHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = webCamTextureMat.width();
float height = webCamTextureMat.height();
#if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API
// HololensCameraStream always returns image data in BGRA format.
texture = new Texture2D((int)width, (int)height, TextureFormat.BGRA32, false);
#else
texture = new Texture2D((int)width, (int)height, TextureFormat.RGBA32, false);
#endif
previewQuad.GetComponent <MeshRenderer>().material.mainTexture = texture;
previewQuad.transform.localScale = new Vector3(1, height / width, 1);
previewQuad.SetActive(displayCameraPreview);
double fx = this.fx;
double fy = this.fy;
double cx = this.cx / imageOptimizationHelper.downscaleRatio;
double cy = this.cy / imageOptimizationHelper.downscaleRatio;
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, fx);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, cx);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, fy);
camMatrix.put(1, 2, cy);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
Debug.Log("camMatrix " + camMatrix.dump());
distCoeffs = new MatOfDouble(distCoeffs1, distCoeffs2, distCoeffs3, distCoeffs4, distCoeffs5);
Debug.Log("distCoeffs " + distCoeffs.dump());
//Calibration camera
Size imageSize = new Size(width, height);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point(0, 0);
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx [0]);
Debug.Log("fovy " + fovy [0]);
Debug.Log("focalLength " + focalLength [0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio [0]);
transformationM = new Matrix4x4();
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
axes.SetActive(false);
head.SetActive(false);
rightEye.SetActive(false);
leftEye.SetActive(false);
mouth.SetActive(false);
mouthParticleSystem = mouth.GetComponentsInChildren <ParticleSystem> (true);
//If WebCamera is frontFaceing,flip Mat.
if (webCamTextureToMatHelper.GetWebCamDevice().isFrontFacing)
{
webCamTextureToMatHelper.flipHorizontal = true;
}
grayMat = new Mat();
cascade = new CascadeClassifier();
cascade.load(OpenCVForUnity.Utils.getFilePath("lbpcascade_frontalface.xml"));
// "empty" method is not working on the UWP platform.
// if (cascade.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
grayMat4Thread = new Mat();
cascade4Thread = new CascadeClassifier();
cascade4Thread.load(OpenCVForUnity.Utils.getFilePath("haarcascade_frontalface_alt.xml"));
// "empty" method is not working on the UWP platform.
// if (cascade4Thread.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
detectionResult = new MatOfRect();
}
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics)
{
isDetectingInFrameArrivedThread = true;
DebugUtils.VideoTick();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
if (useOpenCVDetector)
{
Imgproc.equalizeHist(downScaleMat, downScaleMat);
}
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
if (useOpenCVDetector)
{
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true);
}
else
{
DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread);
}
isThreadRunning = false;
OnDetectionDone();
});
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
resultFaceLandmarkPoints.Clear();
foreach (Rect rect in resultObjects)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
resultFaceLandmarkPoints.Add(points);
}
if (!displayCameraImage)
{
// fill all black
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (displayDetectedFaceRect)
{
// draw face rects
foreach (Rect rect in resultObjects)
{
OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2);
}
}
// draw face landmark points
foreach (List <Vector2> points in resultFaceLandmarkPoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4);
}
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//});
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
}
else
{
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions from previous positions");
//});
if (useOpenCVDetector)
{
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
else
{
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateRawRects();
}
}
}
detectedObjectsInRegions.Clear();
foreach (Rect rect in rectsWhereRegions)
{
if (useOpenCVDetector)
{
DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, cascade, true);
}
else
{
DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, faceLandmarkDetector);
}
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, false);
}
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
resultFaceLandmarkPoints.Clear();
foreach (Rect rect in resultObjects)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
resultFaceLandmarkPoints.Add(points);
}
if (!displayCameraImage)
{
// fill all black
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (displayDetectedFaceRect)
{
// draw previous rects
DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1);
// draw face rects
foreach (Rect rect in resultObjects)
{
OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2);
}
}
// draw face landmark points
foreach (List <Vector2> points in resultFaceLandmarkPoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4);
}
}
DebugUtils.TrackTick();
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(grayMat, texture);
grayMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
isDetectingInFrameArrivedThread = false;
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Imgproc.cvtColor(bgraMat, grayMat, CVTCOLOR_CODE);
Mat downScaleGrayMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
if (useOpenCVDetector)
{
Imgproc.equalizeHist(downScaleGrayMat, downScaleGrayMat);
}
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleGrayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
if (useOpenCVDetector)
{
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true);
}
else
{
DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread);
}
isThreadRunning = false;
OnDetectionDone();
});
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
int len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
// restore to original size rect
float downscaleRatio = imageOptimizationHelper.downscaleRatio;
rect.x = (int)(rect.x * downscaleRatio);
rect.y = (int)(rect.y * downscaleRatio);
rect.width = (int)(rect.width * downscaleRatio);
rect.height = (int)(rect.height * downscaleRatio);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
// draw face landmark points
OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2);
if (displayDetectedFaceRect)
{
// draw face rect
OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
}
}
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
if (displayDetectedFaceRect)
{
DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_BLUE, 1);
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
if (useOpenCVDetector)
{
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
else
{
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateRawRects();
}
}
if (displayDetectedFaceRect)
{
DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_GREEN, 1);
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
if (useOpenCVDetector)
{
DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade, true);
}
else
{
DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, faceLandmarkDetector);
}
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, false);
}
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
// restore to original size rect
float downscaleRatio = imageOptimizationHelper.downscaleRatio;
rect.x = (int)(rect.x * downscaleRatio);
rect.y = (int)(rect.y * downscaleRatio);
rect.width = (int)(rect.width * downscaleRatio);
rect.height = (int)(rect.height * downscaleRatio);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
// draw face landmark points
OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2);
if (displayDetectedFaceRect)
{
// draw face rect
OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
}
}
}
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(bgraMat, texture);
bgraMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
}
/// <summary>
/// Raises the web cam texture to mat helper initialized event.
/// </summary>
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat webCamTextureMat = imageOptimizationHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
#if NETFX_CORE
// HololensCameraStream always returns image data in BGRA format.
texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.BGRA32, false);
#else
texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false);
#endif
texture.wrapMode = TextureWrapMode.Clamp;
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
quad_renderer = gameObject.GetComponent <Renderer> () as Renderer;
quad_renderer.sharedMaterial.SetTexture("_MainTex", texture);
quad_renderer.sharedMaterial.SetVector("_VignetteOffset", new Vector4(0, 0));
Matrix4x4 projectionMatrix;
#if NETFX_CORE
projectionMatrix = webCamTextureToMatHelper.GetProjectionMatrix();
quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix);
#else
//This value is obtained from PhotoCapture's TryGetProjectionMatrix() method.I do not know whether this method is good.
//Please see the discussion of this thread.Https://forums.hololens.com/discussion/782/live-stream-of-locatable-camera-webcam-in-unity
projectionMatrix = Matrix4x4.identity;
projectionMatrix.m00 = 2.31029f;
projectionMatrix.m01 = 0.00000f;
projectionMatrix.m02 = 0.09614f;
projectionMatrix.m03 = 0.00000f;
projectionMatrix.m10 = 0.00000f;
projectionMatrix.m11 = 4.10427f;
projectionMatrix.m12 = -0.06231f;
projectionMatrix.m13 = 0.00000f;
projectionMatrix.m20 = 0.00000f;
projectionMatrix.m21 = 0.00000f;
projectionMatrix.m22 = -1.00000f;
projectionMatrix.m23 = 0.00000f;
projectionMatrix.m30 = 0.00000f;
projectionMatrix.m31 = 0.00000f;
projectionMatrix.m32 = -1.00000f;
projectionMatrix.m33 = 0.00000f;
quad_renderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix);
#endif
quad_renderer.sharedMaterial.SetFloat("_VignetteScale", 0.0f);
grayMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC1);
cascade = new CascadeClassifier();
cascade.load(OpenCVForUnity.Utils.getFilePath("lbpcascade_frontalface.xml"));
// "empty" method is not working on the UWP platform.
// if (cascade.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
grayMat4Thread = new Mat();
cascade4Thread = new CascadeClassifier();
cascade4Thread.load(OpenCVForUnity.Utils.getFilePath("haarcascade_frontalface_alt.xml"));
// "empty" method is not working on the UWP platform.
// if (cascade4Thread.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
detectionResult = new MatOfRect();
}