private void DetectInRegion(Mat img, Rect region, List <Rect> detectedObjectsInRegions, FaceLandmarkDetector landmarkDetector)
{
Rect r0 = new Rect(new Point(), img.size());
Rect r1 = new Rect(region.x, region.y, region.width, region.height);
Rect.inflate(r1, (int)((r1.width * coeffTrackingWindowSize) - r1.width) / 2,
(int)((r1.height * coeffTrackingWindowSize) - r1.height) / 2);
r1 = Rect.intersect(r0, r1);
if ((r1.width <= 0) || (r1.height <= 0))
{
Debug.Log("detectInRegion: Empty intersection");
return;
}
using (Mat img1_roi = new Mat(img, r1))
using (Mat img1 = new Mat(r1.size(), img.type()))
{
img1_roi.copyTo(img1);
OpenCVForUnityUtils.SetImage(landmarkDetector, img1);
List <UnityEngine.Rect> detectResult = landmarkDetector.Detect();
int len = detectResult.Count;
for (int i = 0; i < len; i++)
{
UnityEngine.Rect tmp = detectResult[i];
Rect r = new Rect((int)(tmp.x + r1.x), (int)(tmp.y + r1.y), (int)tmp.width, (int)tmp.height);
detectedObjectsInRegions.Add(r);
}
}
}
private void DetectObject(Mat img, out List <Rect> detectedObjects, FaceLandmarkDetector landmarkDetector)
{
OpenCVForUnityUtils.SetImage(landmarkDetector, img);
List <UnityEngine.Rect> detectResult = landmarkDetector.Detect();
detectedObjects = new List <Rect>();
int len = detectResult.Count;
for (int i = 0; i < len; i++)
{
UnityEngine.Rect r = detectResult[i];
detectedObjects.Add(new Rect((int)r.x, (int)r.y, (int)r.width, (int)r.height));
}
}
void OnPhotoCaptured(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
Matrix4x4 cameraToWorldMatrix;
photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
Matrix4x4 projectionMatrix;
photoCaptureFrame.TryGetProjectionMatrix(out projectionMatrix);
photoCaptureFrame.UploadImageDataToTexture(m_Texture);
OpenCVForUnity.Utils.texture2DToMat(m_Texture, rgbaMat);
// fill all black.
// Imgproc.rectangle (rgbaMat, new Point (0, 0), new Point (rgbaMat.width (), rgbaMat.height ()), new Scalar (0, 0, 0, 0), -1);
// draw an edge lines.
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(255, 0, 0, 255), 2);
// draw a diagonal line.
// Imgproc.line (rgbaMat, new Point (0, 0), new Point (rgbaMat.cols (), rgbaMat.rows ()), new Scalar (255, 0, 0, 255));
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect face
List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection();
foreach (var r in detectResult)
{
Debug.Log("rect : " + r.rect);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(r.rect);
Debug.Log("face points count : " + points.Count);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, r.rect, new Scalar(255, 0, 0, 255), 2);
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.5, new Scalar(0, 255, 0, 255), 2, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, m_Texture, colors);
m_Texture.wrapMode = TextureWrapMode.Clamp;
m_CanvasRenderer.enabled = true;
m_CanvasRenderer.sharedMaterial.SetTexture("_MainTex", m_Texture);
m_CanvasRenderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
m_CanvasRenderer.sharedMaterial.SetMatrix("_CameraProjectionMatrix", projectionMatrix);
m_CanvasRenderer.sharedMaterial.SetFloat("_VignetteScale", 0.0f);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
m_Canvas.transform.position = position;
m_Canvas.transform.rotation = rotation;
Debug.Log("Took picture!");
Debug.Log("projectionMatrix:\n" + projectionMatrix.ToString());
m_CapturingPhoto = false;
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Mat rgbaMatClipROI = new Mat(rgbaMat, processingAreaRect);
rgbaMatClipROI.copyTo(processingAreaMat);
// fill all black.
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, processingAreaMat);
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (isUsingDlibFaceDetecter)
{
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(processingAreaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
cascade.detectMultiScale(grayMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(grayMat.cols() * 0.15, grayMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
foreach (var rect in detectResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMatClipROI, points, new Scalar(0, 255, 0, 255), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMatClipROI, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
}
Imgproc.putText(rgbaMatClipROI, "W:" + rgbaMatClipROI.width() + " H:" + rgbaMatClipROI.height() + " SO:" + Screen.orientation, new Point(5, rgbaMatClipROI.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false);
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(255, 0, 0, 255), 2);
// Draw prosessing area rectangle.
Imgproc.rectangle(rgbaMat, processingAreaRect.tl(), processingAreaRect.br(), new Scalar(255, 255, 0, 255), 2);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture);
rgbaMatClipROI.Dispose();
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
;
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
texture.wrapMode = TextureWrapMode.Clamp;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
// Update is called once per frame
void Update()
{
lock (sync) {
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = imageOptimizationHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
rectangleTracker.GetObjects(resultObjects, true);
rects = rectangleTracker.CreateCorrectionBySpeedOfRects();
if (rects.Length > 0)
{
OpenCVForUnity.Rect rect = rects [0];
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
if (enableOpticalFlowFilter)
{
opticalFlowFilter.Process(rgbaMat, points, points, false);
}
UpdateARHeadTransform(points, arCamera.cameraToWorldMatrix);
if (displayCameraPreview)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
}
}
else
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.toArray();
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
if (resultObjects.Count > 0)
{
OpenCVForUnity.Rect rect = resultObjects [0];
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
if (enableOpticalFlowFilter)
{
opticalFlowFilter.Process(rgbaMat, points, points, false);
}
UpdateARHeadTransform(points, arCamera.cameraToWorldMatrix);
if (displayCameraPreview)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
}
}
if (displayCameraPreview)
{
OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Mat downScaleFrameMat = imageOptimizationHelper.GetDownScaleMat(bgraMat);
Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() => {
isThreadRunning = true;
DetectObject();
isThreadRunning = false;
OnDetectionDone();
});
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
Mat bgraMat4preview = null;
if (displayCameraPreview)
{
bgraMat4preview = new Mat();
downScaleFrameMat.copyTo(bgraMat4preview);
}
List <Vector2> points = null;
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
}
lock (rectangleTracker) {
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
if (rects.Length > 0)
{
OpenCVForUnity.Rect rect = rects [0];
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
if (enableOpticalFlowFilter)
{
opticalFlowFilter.Process(bgraMat, points, points, false);
}
if (displayCameraPreview && bgraMat4preview != null)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2);
}
}
}
else
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = detectionResult.toArray();
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
if (resultObjects.Count > 0)
{
OpenCVForUnity.Rect rect = resultObjects [0];
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
if (enableOpticalFlowFilter)
{
opticalFlowFilter.Process(bgraMat, points, points, false);
}
if (displayCameraPreview && bgraMat4preview != null)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2);
}
}
}
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
if (displayCameraPreview && bgraMat4preview != null)
{
OpenCVForUnity.Utils.fastMatToTexture2D(bgraMat4preview, texture);
}
if (points != null)
{
UpdateARHeadTransform(points, cameraToWorldMatrix);
}
bgraMat.Dispose();
if (bgraMat4preview != null)
{
bgraMat4preview.Dispose();
}
}, false);
}
// Update is called once per frame
void Update()
{
lock (sync) {
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enable && !detecting)
{
detecting = true;
grayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
Rect[] rects;
if (!isUsingSeparateDetection)
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
rectangleTracker.GetObjects(resultObjects, true);
rects = rectangleTracker.CreateCorrectionBySpeedOfRects();
if (rects.Length > 0)
{
OpenCVForUnity.Rect rect = rects [0];
// Adjust to Dilb's result.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
UpdateARHeadTransform(points);
}
}
else
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.toArray();
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
if (resultObjects.Count > 0)
{
OpenCVForUnity.Rect rect = resultObjects [0];
// Adjust to Dilb's result.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
UpdateARHeadTransform(points);
}
}
}
}
// Update is called once per frame
void Update()
{
lock (ExecuteOnMainThread)
{
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, CVTCOLOR_CODE);
Mat downScaleGrayMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
if (useOpenCVDetector)
{
Imgproc.equalizeHist(downScaleGrayMat, downScaleGrayMat);
}
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleGrayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
if (!displayCameraImage)
{
// fill all black
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
rectangleTracker.GetObjects(resultObjects, true);
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
int len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
// restore to original size rect
float downscaleRatio = imageOptimizationHelper.downscaleRatio;
rect.x = (int)(rect.x * downscaleRatio);
rect.y = (int)(rect.y * downscaleRatio);
rect.width = (int)(rect.width * downscaleRatio);
rect.height = (int)(rect.height * downscaleRatio);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
// draw face landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
if (displayDetectedFaceRect)
{
// draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
}
}
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.ToArray();
if (displayDetectedFaceRect)
{
DrawDownScaleFaceRects(rgbaMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_BLUE, 1);
}
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
if (useOpenCVDetector)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
else
{
rectsWhereRegions = rectangleTracker.CreateRawRects();
}
if (displayDetectedFaceRect)
{
DrawDownScaleFaceRects(rgbaMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_GREEN, 1);
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
if (useOpenCVDetector)
{
DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade, true);
}
else
{
DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, faceLandmarkDetector);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, false);
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
// restore to original size rect
float downscaleRatio = imageOptimizationHelper.downscaleRatio;
rect.x = (int)(rect.x * downscaleRatio);
rect.y = (int)(rect.y * downscaleRatio);
rect.width = (int)(rect.width * downscaleRatio);
rect.height = (int)(rect.height * downscaleRatio);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
// draw face landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
if (displayDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
}
}
}
Utils.fastMatToTexture2D(rgbaMat, texture);
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Imgproc.cvtColor(bgraMat, grayMat, CVTCOLOR_CODE);
Mat downScaleGrayMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
if (useOpenCVDetector)
{
Imgproc.equalizeHist(downScaleGrayMat, downScaleGrayMat);
}
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleGrayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
if (useOpenCVDetector)
{
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true);
}
else
{
DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread);
}
isThreadRunning = false;
OnDetectionDone();
});
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
int len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
// restore to original size rect
float downscaleRatio = imageOptimizationHelper.downscaleRatio;
rect.x = (int)(rect.x * downscaleRatio);
rect.y = (int)(rect.y * downscaleRatio);
rect.width = (int)(rect.width * downscaleRatio);
rect.height = (int)(rect.height * downscaleRatio);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
// draw face landmark points
OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2);
if (displayDetectedFaceRect)
{
// draw face rect
OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
}
}
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
if (displayDetectedFaceRect)
{
DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_BLUE, 1);
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
if (useOpenCVDetector)
{
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
else
{
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateRawRects();
}
}
if (displayDetectedFaceRect)
{
DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, imageOptimizationHelper.downscaleRatio, COLOR_GREEN, 1);
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
if (useOpenCVDetector)
{
DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade, true);
}
else
{
DetectInRegion(downScaleGrayMat, rectsWhereRegions[i], detectedObjectsInRegions, faceLandmarkDetector);
}
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, false);
}
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
// restore to original size rect
float downscaleRatio = imageOptimizationHelper.downscaleRatio;
rect.x = (int)(rect.x * downscaleRatio);
rect.y = (int)(rect.y * downscaleRatio);
rect.width = (int)(rect.width * downscaleRatio);
rect.height = (int)(rect.height * downscaleRatio);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
// draw face landmark points
OpenCVForUnityUtils.DrawFaceLandmark(bgraMat, points, COLOR_GREEN, 2);
if (displayDetectedFaceRect)
{
// draw face rect
OpenCVForUnityUtils.DrawFaceRect(bgraMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_RED, 2);
}
}
}
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(bgraMat, texture);
bgraMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
}
// Update is called once per frame
void Update()
{
lock (sync) {
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enable && !detecting)
{
detecting = true;
grayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
if (!isShowingWebCamImage)
{
// fill all black.
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
}
Rect[] rects;
if (!isUsingSeparateDetection)
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
rectangleTracker.GetObjects(resultObjects, true);
// rects = resultObjects.ToArray ();
rects = rectangleTracker.CreateCorrectionBySpeedOfRects();
if (isShowingDetectedFaceRect)
{
for (int i = 0; i < rects.Length; i++)
{
//Debug.Log ("detected face[" + i + "] " + rects [i]);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
}
}
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in resultObjects)
{
r.y += (int)(r.height * 0.1f);
}
foreach (var rect in resultObjects)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
if (isShowingDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
}
}
}
else
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.toArray();
if (isShowingDetectedFaceRect)
{
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 1);
}
}
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
if (isShowingDetectedFaceRect)
{
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1);
}
}
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
if (isShowingDetectedFaceRect)
{
rects = resultObjects.ToArray();
for (int i = 0; i < rects.Length; i++)
{
//Debug.Log ("detected face[" + i + "] " + rects [i]);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
}
}
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in resultObjects)
{
r.y += (int)(r.height * 0.1f);
}
foreach (var rect in resultObjects)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
if (isShowingDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
}
}
}
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
texture.wrapMode = TextureWrapMode.Clamp;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
// Update is called once per frame
void Update()
{
lock (ExecuteOnMainThread)
{
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
DebugUtils.VideoTick();
Mat grayMat = webCamTextureToMatHelper.GetMat();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
if (useOpenCVDetector)
{
Imgproc.equalizeHist(downScaleMat, downScaleMat);
}
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
rectangleTracker.GetObjects(resultObjects, true);
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
resultFaceLandmarkPoints.Clear();
foreach (Rect rect in resultObjects)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
resultFaceLandmarkPoints.Add(points);
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (displayDetectedFaceRect)
{
// draw face rects
foreach (Rect rect in resultObjects)
{
OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2);
}
}
// draw face landmark points
foreach (List <Vector2> points in resultFaceLandmarkPoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4);
}
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.ToArray();
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
if (useOpenCVDetector)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
else
{
rectsWhereRegions = rectangleTracker.CreateRawRects();
}
}
detectedObjectsInRegions.Clear();
foreach (Rect rect in rectsWhereRegions)
{
if (useOpenCVDetector)
{
DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, cascade, true);
}
else
{
DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, faceLandmarkDetector);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, false);
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
resultFaceLandmarkPoints.Clear();
foreach (Rect rect in resultObjects)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
resultFaceLandmarkPoints.Add(points);
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (displayDetectedFaceRect)
{
// draw previous rects
DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1);
// draw face rects
foreach (Rect rect in resultObjects)
{
OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2);
}
}
// draw face landmark points
foreach (List <Vector2> points in resultFaceLandmarkPoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4);
}
}
DebugUtils.TrackTick();
Utils.fastMatToTexture2D(grayMat, texture);
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Mat downScaleFrameMat = bgraMat;
if (imageOptimizationHelper.downscaleRatio != 1)
{
downScaleFrameMat = new Mat();
Imgproc.resize(bgraMat, downScaleFrameMat, new Size(), 1.0 / imageOptimizationHelper.downscaleRatio, 1.0 / imageOptimizationHelper.downscaleRatio, Imgproc.INTER_LINEAR);
}
Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() => {
isThreadRunning = true;
DetectObject();
isThreadRunning = false;
OnDetectionDone();
});
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(downScaleFrameMat, new Point(0, 0), new Point(downScaleFrameMat.width(), downScaleFrameMat.height()), new Scalar(0, 0, 0, 0), -1);
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
}
lock (rectangleTracker) {
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
if (displayDetectedFaceRect)
{
for (int i = 0; i < rects.Length; i++)
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log ("detected face[" + i + "] " + rects [i]);
//}, true);
Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3);
}
}
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in resultObjects)
{
r.y += (int)(r.height * 0.1f);
}
foreach (var rect in resultObjects)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2);
if (displayDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
else
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = detectionResult.toArray();
}
if (displayDetectedFaceRect)
{
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
}
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
if (displayDetectedFaceRect)
{
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1);
}
}
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
if (displayDetectedFaceRect)
{
rects = resultObjects.ToArray();
for (int i = 0; i < rects.Length; i++)
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log ("detected face[" + i + "] " + rects [i]);
//}, true);
Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3);
}
}
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in resultObjects)
{
r.y += (int)(r.height * 0.1f);
}
foreach (var rect in resultObjects)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2);
if (displayDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
OpenCVForUnity.Utils.fastMatToTexture2D(downScaleFrameMat, texture);
bgraMat.Dispose();
if (imageOptimizationHelper.downscaleRatio != 1)
{
downScaleFrameMat.Dispose();
}
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}, false);
}