// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
DebugUtils.VideoTick();
Mat grayMat = webCamTextureToMatHelper.GetMat();
comicFilter.Process(grayMat, grayMat, false);
//Imgproc.putText(grayMat, "W:" + grayMat.width() + " H:" + grayMat.height() + " SO:" + Screen.orientation, new Point(5, grayMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255), 2, Imgproc.LINE_AA, false);
DebugUtils.TrackTick();
Utils.fastMatToTexture2D(grayMat, texture);
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = webCamTextureToMatHelper.GetCameraToWorldMatrix();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
private void OnDetectionDone()
{
DebugUtils.TrackTick();
if (displayCameraPreview)
{
Imgproc.cvtColor(downScaleMat, rgbMat4preview, Imgproc.COLOR_GRAY2RGB);
if (ids.total() > 0)
{
Aruco.drawDetectedMarkers(rgbMat4preview, corners, ids, new Scalar(0, 255, 0));
if (applyEstimationPose)
{
for (int i = 0; i < ids.total(); i++)
{
using (Mat rvec = new Mat(rvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1)))
using (Mat tvec = new Mat(tvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1)))
{
// In this example we are processing with RGB color image, so Axis-color correspondences are X: blue, Y: green, Z: red. (Usually X: red, Y: green, Z: blue)
Calib3d.drawFrameAxes(rgbMat4preview, camMatrix, distCoeffs, rvec, tvec, markerLength * 0.5f);
}
}
}
}
Utils.fastMatToTexture2D(rgbMat4preview, texture);
}
if (applyEstimationPose)
{
if (hasUpdatedARTransformMatrix)
{
hasUpdatedARTransformMatrix = false;
// Apply the cameraToWorld matrix with the Z-axis inverted.
ARM = arCamera.cameraToWorldMatrix * invertZM * ARM;
if (enableLerpFilter)
{
arGameObject.SetMatrix4x4(ARM);
}
else
{
ARUtils.SetTransformFromMatrix(arGameObject.transform, ref ARM);
}
}
}
isDetecting = false;
}
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics)
{
isDetectingInFrameArrivedThread = true;
DebugUtils.VideoTick();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
Mat camMatrix = null;
MatOfDouble distCoeffs = null;
if (useStoredCameraParameters)
{
camMatrix = this.camMatrix;
distCoeffs = this.distCoeffs;
}
else
{
camMatrix = CreateCameraMatrix(cameraIntrinsics.FocalLengthX, cameraIntrinsics.FocalLengthY, cameraIntrinsics.PrincipalPointX / DOWNSCALE_RATIO, cameraIntrinsics.PrincipalPointY / DOWNSCALE_RATIO);
distCoeffs = new MatOfDouble(cameraIntrinsics.RadialDistK1, cameraIntrinsics.RadialDistK2, cameraIntrinsics.RadialDistK3, cameraIntrinsics.TangentialDistP1, cameraIntrinsics.TangentialDistP2);
}
if (enableDetection)
{
// Detect markers and estimate Pose
Aruco.detectMarkers(downScaleMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs);
if (applyEstimationPose && ids.total() > 0)
{
Aruco.estimatePoseSingleMarkers(corners, markerLength, camMatrix, distCoeffs, rvecs, tvecs);
for (int i = 0; i < ids.total(); i++)
{
//This example can display ARObject on only first detected marker.
if (i == 0)
{
// Convert to unity pose data.
double[] rvecArr = new double[3];
rvecs.get(0, 0, rvecArr);
double[] tvecArr = new double[3];
tvecs.get(0, 0, tvecArr);
tvecArr[2] /= DOWNSCALE_RATIO;
PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr);
// Create transform matrix.
transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one);
lock (sync)
{
// Right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARM = invertYM * transformationM;
// Apply Z-axis inverted matrix.
ARM = ARM * invertZM;
}
hasUpdatedARTransformMatrix = true;
break;
}
}
}
}
Mat rgbMat4preview = null;
if (displayCameraPreview)
{
rgbMat4preview = new Mat();
Imgproc.cvtColor(downScaleMat, rgbMat4preview, Imgproc.COLOR_GRAY2RGB);
if (ids.total() > 0)
{
Aruco.drawDetectedMarkers(rgbMat4preview, corners, ids, new Scalar(0, 255, 0));
if (applyEstimationPose)
{
for (int i = 0; i < ids.total(); i++)
{
using (Mat rvec = new Mat(rvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1)))
using (Mat tvec = new Mat(tvecs, new OpenCVForUnity.CoreModule.Rect(0, i, 1, 1)))
{
// In this example we are processing with RGB color image, so Axis-color correspondences are X: blue, Y: green, Z: red. (Usually X: red, Y: green, Z: blue)
Calib3d.drawFrameAxes(rgbMat4preview, camMatrix, distCoeffs, rvec, tvec, markerLength * 0.5f);
}
}
}
}
}
DebugUtils.TrackTick();
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
if (displayCameraPreview && rgbMat4preview != null)
{
Utils.fastMatToTexture2D(rgbMat4preview, texture);
rgbMat4preview.Dispose();
}
if (applyEstimationPose)
{
if (hasUpdatedARTransformMatrix)
{
hasUpdatedARTransformMatrix = false;
lock (sync)
{
// Apply camera transform matrix.
ARM = cameraToWorldMatrix * invertZM * ARM;
if (enableLerpFilter)
{
arGameObject.SetMatrix4x4(ARM);
}
else
{
ARUtils.SetTransformFromMatrix(arGameObject.transform, ref ARM);
}
}
}
}
grayMat.Dispose();
});
isDetectingInFrameArrivedThread = false;
}
// Update is called once per frame
void Update()
{
lock (ExecuteOnMainThread)
{
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
DebugUtils.VideoTick();
Mat grayMat = webCamTextureToMatHelper.GetMat();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.equalizeHist(downScaleMat, downScaleMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
rectangleTracker.GetObjects(resultObjects, true);
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
// draw face rect.
DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6);
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.ToArray();
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
// draw previous rect.
DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1);
// draw face rect.
DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6);
}
DebugUtils.TrackTick();
Utils.fastMatToTexture2D(grayMat, texture);
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;;
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics)
{
isDetectingInFrameArrivedThread = true;
DebugUtils.VideoTick();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.equalizeHist(downScaleMat, downScaleMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread);
isThreadRunning = false;
OnDetectionDone();
});
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
// draw face rect.
DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6);
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//});
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
}
else
{
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions from previous positions");
//});
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
// draw previous rect.
DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1);
// draw face rect.
DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6);
}
DebugUtils.TrackTick();
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(grayMat, texture);
grayMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
isDetectingInFrameArrivedThread = false;
}
// Update is called once per frame
void Update()
{
lock (ExecuteOnMainThread)
{
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
DebugUtils.VideoTick();
Mat grayMat = webCamTextureToMatHelper.GetMat();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.equalizeHist(downScaleMat, downScaleMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
rectangleTracker.GetObjects(resultObjects, true);
rects = resultObjects.ToArray();
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.ToArray();
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
rects = resultObjects.ToArray();
}
if (enableDownScale)
{
int len = rects.Length;
for (int i = 0; i < len; i++)
{
Rect rect = rects[i];
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
}
}
// draw face rect.
DrawRects(rects, grayMat.width(), grayMat.height());
DebugUtils.TrackTick();
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;;
Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance));
Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance));
//position
Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos);
gameObject.transform.position = position;
//scale
Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1);
gameObject.transform.localScale = scale;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.rotation = rotation;
rectOverlay.UpdateOverlayTransform(gameObject.transform);
}
}
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics)
{
isDetectingInFrameArrivedThread = true;
DebugUtils.VideoTick();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.equalizeHist(downScaleMat, downScaleMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread);
isThreadRunning = false;
OnDetectionDone();
});
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//});
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
}
else
{
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions from previous positions");
//});
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
}
if (enableDownScale)
{
int len = rects.Length;
for (int i = 0; i < len; i++)
{
Rect rect = rects[i];
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
}
}
DebugUtils.TrackTick();
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
// draw face rect.
DrawRects(rects, grayMat.width(), grayMat.height());
grayMat.Dispose();
Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance));
Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance));
//position
Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos);
gameObject.transform.position = position;
//scale
Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1);
gameObject.transform.localScale = scale;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.rotation = rotation;
rectOverlay.UpdateOverlayTransform(gameObject.transform);
});
isDetectingInFrameArrivedThread = false;
}
