// Update is called once per frame
void Update()
{
lock (sync) {
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enable && !detecting)
{
detecting = true;
grayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
Rect[] rects;
if (!isUsingSeparateDetection)
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
rectangleTracker.GetObjects(resultObjects, true);
rects = rectangleTracker.CreateCorrectionBySpeedOfRects();
if (rects.Length > 0)
{
OpenCVForUnity.Rect rect = rects [0];
// Adjust to Dilb's result.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
UpdateARHeadTransform(points);
}
}
else
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.toArray();
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
if (resultObjects.Count > 0)
{
OpenCVForUnity.Rect rect = resultObjects [0];
// Adjust to Dilb's result.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
UpdateARHeadTransform(points);
}
}
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Mat rgbaMatClipROI = new Mat(rgbaMat, processingAreaRect);
rgbaMatClipROI.copyTo(processingAreaMat);
// fill all black.
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, processingAreaMat);
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (isUsingDlibFaceDetecter)
{
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(processingAreaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
cascade.detectMultiScale(grayMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(grayMat.cols() * 0.15, grayMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
foreach (var rect in detectResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMatClipROI, points, new Scalar(0, 255, 0, 255), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMatClipROI, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
}
Imgproc.putText(rgbaMatClipROI, "W:" + rgbaMatClipROI.width() + " H:" + rgbaMatClipROI.height() + " SO:" + Screen.orientation, new Point(5, rgbaMatClipROI.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0, 255), 1, Imgproc.LINE_AA, false);
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(255, 0, 0, 255), 2);
// Draw prosessing area rectangle.
Imgproc.rectangle(rgbaMat, processingAreaRect.tl(), processingAreaRect.br(), new Scalar(255, 255, 0, 255), 2);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture);
rgbaMatClipROI.Dispose();
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
;
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
texture.wrapMode = TextureWrapMode.Clamp;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
// Update is called once per frame
void Update()
{
lock (sync) {
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enable && !detecting)
{
detecting = true;
grayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
if (!isShowingWebCamImage)
{
// fill all black.
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
}
Rect[] rects;
if (!isUsingSeparateDetection)
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
rectangleTracker.GetObjects(resultObjects, true);
// rects = resultObjects.ToArray ();
rects = rectangleTracker.CreateCorrectionBySpeedOfRects();
if (isShowingDetectedFaceRect)
{
for (int i = 0; i < rects.Length; i++)
{
//Debug.Log ("detected face[" + i + "] " + rects [i]);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
}
}
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in resultObjects)
{
r.y += (int)(r.height * 0.1f);
}
foreach (var rect in resultObjects)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
if (isShowingDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
}
}
}
else
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.toArray();
if (isShowingDetectedFaceRect)
{
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 1);
}
}
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
if (isShowingDetectedFaceRect)
{
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1);
}
}
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
if (isShowingDetectedFaceRect)
{
rects = resultObjects.ToArray();
for (int i = 0; i < rects.Length; i++)
{
//Debug.Log ("detected face[" + i + "] " + rects [i]);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
}
}
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in resultObjects)
{
r.y += (int)(r.height * 0.1f);
}
foreach (var rect in resultObjects)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
if (isShowingDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(255, 0, 0, 255), 2);
}
}
}
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
texture.wrapMode = TextureWrapMode.Clamp;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
