/// <summary>
/// Raises the web cam texture to mat helper initialized event.
/// </summary>
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat rawSizeMat = webCamTextureToMatHelper.GetMat();
float rawSizeWidth = rawSizeMat.width();
float rawSizeHeight = rawSizeMat.height();
Mat webCamTextureMat = imageOptimizationHelper.GetDownScaleMat(rawSizeMat);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = webCamTextureMat.width();
float height = webCamTextureMat.height();
texture = new Texture2D((int)width, (int)height, TextureFormat.RGB24, false);
previewQuad.GetComponent <MeshRenderer>().material.mainTexture = texture;
previewQuad.transform.localScale = new Vector3(1, height / width, 1);
previewQuad.SetActive(displayCameraPreview);
// set camera parameters.
double fx;
double fy;
double cx;
double cy;
string loadDirectoryPath = Path.Combine(Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample");
string calibratonDirectoryName = "camera_parameters" + rawSizeWidth + "x" + rawSizeHeight;
string loadCalibratonFileDirectoryPath = Path.Combine(loadDirectoryPath, calibratonDirectoryName);
string loadPath = Path.Combine(loadCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml");
if (useStoredCameraParameters && File.Exists(loadPath))
{
CameraParameters param;
XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters));
using (var stream = new FileStream(loadPath, FileMode.Open)) {
param = (CameraParameters)serializer.Deserialize(stream);
}
fx = param.camera_matrix[0];
fy = param.camera_matrix[4];
cx = param.camera_matrix[2] / imageOptimizationHelper.downscaleRatio;
cy = param.camera_matrix[5] / imageOptimizationHelper.downscaleRatio;
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, fx);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, cx);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, fy);
camMatrix.put(1, 2, cy);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
distCoeffs = new MatOfDouble(param.GetDistortionCoefficients());
Debug.Log("Loaded CameraParameters from a stored XML file.");
Debug.Log("loadPath: " + loadPath);
}
else
{
fx = this.fx;
fy = this.fy;
cx = this.cx / imageOptimizationHelper.downscaleRatio;
cy = this.cy / imageOptimizationHelper.downscaleRatio;
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, fx);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, cx);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, fy);
camMatrix.put(1, 2, cy);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
distCoeffs = new MatOfDouble(this.distCoeffs1, this.distCoeffs2, this.distCoeffs3, this.distCoeffs4, this.distCoeffs5);
Debug.Log("Created a dummy CameraParameters.");
}
Debug.Log("camMatrix " + camMatrix.dump());
Debug.Log("distCoeffs " + distCoeffs.dump());
//Calibration camera
Size imageSize = new Size(width, height);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point(0, 0);
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx [0]);
Debug.Log("fovy " + fovy [0]);
Debug.Log("focalLength " + focalLength [0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio [0]);
// Display objects near the camera.
arCamera.nearClipPlane = 0.01f;
grayMat = new Mat();
ids = new Mat();
corners = new List <Mat> ();
rejectedCorners = new List <Mat> ();
rvecs = new Mat();
tvecs = new Mat();
rotMat = new Mat(3, 3, CvType.CV_64FC1);
transformationM = new Matrix4x4();
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
detectorParams = DetectorParameters.create();
dictionary = Aruco.getPredefinedDictionary(Aruco.DICT_6X6_250);
//If WebCamera is frontFaceing,flip Mat.
if (webCamTextureToMatHelper.GetWebCamDevice().isFrontFacing)
{
webCamTextureToMatHelper.flipHorizontal = true;
}
downScaleFrameMat = new Mat((int)height, (int)width, CvType.CV_8UC4);
rgbMat4preview = new Mat();
}
/// <summary>
/// Raises the web cam texture to mat helper initialized event.
/// </summary>
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat grayMat = webCamTextureToMatHelper.GetMat();
float rawFrameWidth = grayMat.width();
float rawFrameHeight = grayMat.height();
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
float width = downScaleMat.width();
float height = downScaleMat.height();
texture = new Texture2D((int)width, (int)height, TextureFormat.RGB24, false);
previewQuad.GetComponent <MeshRenderer>().material.mainTexture = texture;
previewQuad.transform.localScale = new Vector3(0.2f * width / height, 0.2f, 1);
previewQuad.SetActive(displayCameraPreview);
//Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
DebugUtils.AddDebugStr(webCamTextureToMatHelper.GetWidth() + " x " + webCamTextureToMatHelper.GetHeight() + " : " + webCamTextureToMatHelper.GetFPS());
if (enableDownScale)
{
DebugUtils.AddDebugStr("enableDownScale = true: " + DOWNSCALE_RATIO + " / " + width + " x " + height);
}
// create camera matrix and dist coeffs.
string loadDirectoryPath = Path.Combine(Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample");
string calibratonDirectoryName = "camera_parameters" + rawFrameWidth + "x" + rawFrameWidth;
string loadCalibratonFileDirectoryPath = Path.Combine(loadDirectoryPath, calibratonDirectoryName);
string loadPath = Path.Combine(loadCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml");
if (useStoredCameraParameters && File.Exists(loadPath))
{
// If there is a camera parameters stored by HoloLensArUcoCameraCalibrationExample, use it
CameraParameters param;
XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters));
using (var stream = new FileStream(loadPath, FileMode.Open))
{
param = (CameraParameters)serializer.Deserialize(stream);
}
double fx = param.camera_matrix[0];
double fy = param.camera_matrix[4];
double cx = param.camera_matrix[2];
double cy = param.camera_matrix[5];
camMatrix = CreateCameraMatrix(fx, fy, cx / DOWNSCALE_RATIO, cy / DOWNSCALE_RATIO);
distCoeffs = new MatOfDouble(param.GetDistortionCoefficients());
Debug.Log("Loaded CameraParameters from a stored XML file.");
Debug.Log("loadPath: " + loadPath);
DebugUtils.AddDebugStr("Loaded CameraParameters from a stored XML file.");
DebugUtils.AddDebugStr("loadPath: " + loadPath);
}
else
{
if (useStoredCameraParameters && !File.Exists(loadPath))
{
DebugUtils.AddDebugStr("The CameraParameters XML file (" + loadPath + ") does not exist.");
}
#if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API
CameraIntrinsics cameraIntrinsics = webCamTextureToMatHelper.GetCameraIntrinsics();
camMatrix = CreateCameraMatrix(cameraIntrinsics.FocalLengthX, cameraIntrinsics.FocalLengthY, cameraIntrinsics.PrincipalPointX / DOWNSCALE_RATIO, cameraIntrinsics.PrincipalPointY / DOWNSCALE_RATIO);
distCoeffs = new MatOfDouble(cameraIntrinsics.RadialDistK1, cameraIntrinsics.RadialDistK2, cameraIntrinsics.RadialDistK3, cameraIntrinsics.TangentialDistP1, cameraIntrinsics.TangentialDistP2);
Debug.Log("Created CameraParameters from VideoMediaFrame.CameraIntrinsics on device.");
DebugUtils.AddDebugStr("Created CameraParameters from VideoMediaFrame.CameraIntrinsics on device.");
#else
// The camera matrix value of Hololens camera 896x504 size.
// For details on the camera matrix, please refer to this page. (http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html)
// These values are unique to my device, obtained from the "Windows.Media.Devices.Core.CameraIntrinsics" class. (https://docs.microsoft.com/en-us/uwp/api/windows.media.devices.core.cameraintrinsics)
// Can get these values by using this helper script. (https://github.com/EnoxSoftware/HoloLensWithOpenCVForUnityExample/tree/master/Assets/HololensCameraIntrinsicsChecker/CameraIntrinsicsCheckerHelper)
double fx = 1035.149; //focal length x.
double fy = 1034.633; //focal length y.
double cx = 404.9134; //principal point x.
double cy = 236.2834; //principal point y.
double distCoeffs1 = 0.2036923; //radial distortion coefficient k1.
double distCoeffs2 = -0.2035773; //radial distortion coefficient k2.
double distCoeffs3 = 0.0; //tangential distortion coefficient p1.
double distCoeffs4 = 0.0; //tangential distortion coefficient p2.
double distCoeffs5 = -0.2388065; //radial distortion coefficient k3.
camMatrix = CreateCameraMatrix(fx, fy, cx / DOWNSCALE_RATIO, cy / DOWNSCALE_RATIO);
distCoeffs = new MatOfDouble(distCoeffs1, distCoeffs2, distCoeffs3, distCoeffs4, distCoeffs5);
Debug.Log("Created a dummy CameraParameters (896x504).");
DebugUtils.AddDebugStr("Created a dummy CameraParameters (896x504).");
#endif
}
Debug.Log("camMatrix " + camMatrix.dump());
Debug.Log("distCoeffs " + distCoeffs.dump());
//DebugUtils.AddDebugStr("camMatrix " + camMatrix.dump());
//DebugUtils.AddDebugStr("distCoeffs " + distCoeffs.dump());
//Calibration camera
Size imageSize = new Size(width, height);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point(0, 0);
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx[0]);
Debug.Log("fovy " + fovy[0]);
Debug.Log("focalLength " + focalLength[0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio[0]);
// Display objects near the camera.
arCamera.nearClipPlane = 0.01f;
ids = new Mat();
corners = new List <Mat>();
rejectedCorners = new List <Mat>();
rvecs = new Mat();
tvecs = new Mat();
rotMat = new Mat(3, 3, CvType.CV_64FC1);
transformationM = new Matrix4x4();
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
detectorParams = DetectorParameters.create();
dictionary = Aruco.getPredefinedDictionary(Aruco.DICT_6X6_250);
//If WebCamera is frontFaceing, flip Mat.
webCamTextureToMatHelper.flipHorizontal = webCamTextureToMatHelper.IsFrontFacing();
rgbMat4preview = new Mat();
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(bgraMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = bgraMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.cvtColor(downScaleMat, grayMat, CVTCOLOR_CODE);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread);
isThreadRunning = false;
OnDetectionDone();
});
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
}
if (enableDownScale)
{
int len = rects.Length;
for (int i = 0; i < len; i++)
{
Rect rect = rects[i];
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
}
}
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
DrawRects(rects, bgraMat.width(), bgraMat.height());
bgraMat.Dispose();
Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance));
Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance));
//position
Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos);
gameObject.transform.position = position;
//scale
Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1);
gameObject.transform.localScale = scale;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.rotation = rotation;
rectOverlay.UpdateOverlayTransform(gameObject.transform);
});
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(bgraMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = bgraMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.cvtColor(downScaleMat, grayMat, CVTCOLOR_CODE);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread);
isThreadRunning = false;
OnDetectionDone();
});
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
int len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
if (enableDownScale)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
}
// draw face rect
Imgproc.rectangle(bgraMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), COLOR_RED, 2);
}
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_BLUE, 1);
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GREEN, 1);
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
if (enableDownScale)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
}
// draw face rect
Imgproc.rectangle(bgraMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), COLOR_RED, 2);
}
}
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(bgraMat, texture);
bgraMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
}
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics)
{
isDetectingInFrameArrivedThread = true;
DebugUtils.VideoTick();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.equalizeHist(downScaleMat, downScaleMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread);
isThreadRunning = false;
OnDetectionDone();
});
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
// draw face rect.
DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6);
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//});
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
}
else
{
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions from previous positions");
//});
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
// draw previous rect.
DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1);
// draw face rect.
DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6);
}
DebugUtils.TrackTick();
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(grayMat, texture);
grayMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
isDetectingInFrameArrivedThread = false;
}