// Update is called once per frame
void Update()
{
// loop play.
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
if (capture.grab())
{
capture.retrieve(rgbMat, 0);
Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB);
//Debug.Log ("Mat toString " + rgbMat.ToString ());
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect()) {
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
}
// face traking.
rectangleTracker.UpdateTrackedObjects(detectResult);
List <TrackedRect> trackedRects = new List <TrackedRect> ();
rectangleTracker.GetObjects(trackedRects, true);
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> > ();
for (int i = 0; i < trackedRects.Count; i++)
{
TrackedRect tr = trackedRects [i];
UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
landmarkPoints.Add(points);
}
// face masking.
if (faceMaskTexture != null && landmarkPoints.Count >= 1)
{
OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
float imageWidth = meshOverlay.width;
float imageHeight = meshOverlay.height;
float maskImageWidth = faceMaskTexture.width;
float maskImageHeight = faceMaskTexture.height;
TrackedRect tr;
TrackedMesh tm;
for (int i = 0; i < trackedRects.Count; i++)
{
tr = trackedRects [i];
if (tr.state == TrackedState.NEW)
{
meshOverlay.CreateObject(tr.id, faceMaskTexture);
}
if (tr.state < TrackedState.DELETED)
{
tm = meshOverlay.GetObjectById(tr.id);
Vector3[] vertices = tm.meshFilter.mesh.vertices;
if (vertices.Length == landmarkPoints [i].Count)
{
for (int j = 0; j < vertices.Length; j++)
{
vertices [j].x = landmarkPoints [i] [j].x / imageWidth - 0.5f;
vertices [j].y = 0.5f - landmarkPoints [i] [j].y / imageHeight;
}
}
Vector2[] uv = tm.meshFilter.mesh.uv;
if (uv.Length == faceLandmarkPointsInMask.Count)
{
for (int jj = 0; jj < uv.Length; jj++)
{
uv [jj].x = faceLandmarkPointsInMask [jj].x / maskImageWidth;
uv [jj].y = (maskImageHeight - faceLandmarkPointsInMask [jj].y) / maskImageHeight;
}
}
meshOverlay.UpdateObject(tr.id, vertices, null, uv);
if (tr.numFramesNotDetected > 3)
{
tm.material.SetFloat(shader_FadeID, 1f);
}
else if (tr.numFramesNotDetected > 0 && tr.numFramesNotDetected <= 3)
{
tm.material.SetFloat(shader_FadeID, 0.3f + (0.7f / 4f) * tr.numFramesNotDetected);
}
else
{
tm.material.SetFloat(shader_FadeID, 0.3f);
}
// filter non frontal faces.
if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit)
{
tm.material.SetFloat(shader_FadeID, 1f);
}
}
else if (tr.state == TrackedState.DELETED)
{
meshOverlay.DeleteObject(tr.id);
}
}
}
else if (landmarkPoints.Count >= 1)
{
float imageWidth = meshOverlay.width;
float imageHeight = meshOverlay.height;
float maskImageWidth = texture.width;
float maskImageHeight = texture.height;
TrackedRect tr;
TrackedMesh tm;
for (int i = 0; i < trackedRects.Count; i++)
{
tr = trackedRects [i];
if (tr.state == TrackedState.NEW)
{
meshOverlay.CreateObject(tr.id, texture);
}
if (tr.state < TrackedState.DELETED)
{
tm = meshOverlay.GetObjectById(tr.id);
Vector3[] vertices = tm.meshFilter.mesh.vertices;
if (vertices.Length == landmarkPoints [i].Count)
{
for (int j = 0; j < vertices.Length; j++)
{
vertices [j].x = landmarkPoints[i][j].x / imageWidth - 0.5f;
vertices [j].y = 0.5f - landmarkPoints[i][j].y / imageHeight;
}
}
Vector2[] uv = tm.meshFilter.mesh.uv;
if (uv.Length == landmarkPoints [0].Count)
{
for (int jj = 0; jj < uv.Length; jj++)
{
uv [jj].x = landmarkPoints[0][jj].x / maskImageWidth;
uv [jj].y = (maskImageHeight - landmarkPoints[0][jj].y) / maskImageHeight;
}
}
meshOverlay.UpdateObject(tr.id, vertices, null, uv);
if (tr.numFramesNotDetected > 3)
{
tm.material.SetFloat(shader_FadeID, 1f);
}
else if (tr.numFramesNotDetected > 0 && tr.numFramesNotDetected <= 3)
{
tm.material.SetFloat(shader_FadeID, 0.3f + (0.7f / 4f) * tr.numFramesNotDetected);
}
else
{
tm.material.SetFloat(shader_FadeID, 0.3f);
}
// filter non frontal faces.
if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit)
{
tm.material.SetFloat(shader_FadeID, 1f);
}
}
else if (tr.state == TrackedState.DELETED)
{
meshOverlay.DeleteObject(tr.id);
}
}
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < detectResult.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2);
}
for (int i = 0; i < trackedRects.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects [i].x, trackedRects [i].y, trackedRects [i].width, trackedRects [i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 255, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
// draw face points.
if (displayDebugFacePoints)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2);
}
}
// display face mask image.
if (faceMaskTexture != null && faceMaskMat != null)
{
if (displayFaceRects)
{
OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2);
}
if (displayDebugFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2);
}
float scale = (rgbMat.width() / 4f) / faceMaskMat.width();
float tx = rgbMat.width() - faceMaskMat.width() * scale;
float ty = 0.0f;
Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty);
trans.put(0, 0, scale);
trans.put(0, 1, 0.0f);
trans.put(0, 2, tx);
trans.put(1, 0, 0.0f);
trans.put(1, 1, scale);
trans.put(1, 2, ty);
Imgproc.warpAffine(faceMaskMat, rgbMat, trans, rgbMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0));
}
Imgproc.putText(rgbMat, "W:" + rgbMat.width() + " H:" + rgbMat.height() + " SO:" + Screen.orientation, new Point(5, rgbMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbMat, texture);
}
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Imgproc.cvtColor(bgraMat, grayMat, Imgproc.COLOR_BGRA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() => {
isThreadRunning = true;
DetectObject();
isThreadRunning = false;
OnDetectionDone();
});
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1);
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
}
lock (rectangleTracker) {
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
for (int i = 0; i < rects.Length; i++)
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log ("detected face[" + i + "] " + rects [i]);
//}, true);
Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3);
}
}
else
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = detectionResult.toArray();
}
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1);
}
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
for (int i = 0; i < rects.Length; i++)
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log ("detected face[" + i + "] " + rects [i]);
//}, true);
Imgproc.rectangle(bgraMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3);
}
}
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(bgraMat, texture);
bgraMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
position *= 1.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}, false);
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(bgraMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = bgraMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.cvtColor(downScaleMat, grayMat, CVTCOLOR_CODE);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread);
isThreadRunning = false;
OnDetectionDone();
});
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(bgraMat, new Point(0, 0), new Point(bgraMat.width(), bgraMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
int len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
if (enableDownScale)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
}
// draw face rect
Imgproc.rectangle(bgraMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), COLOR_RED, 2);
}
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_BLUE, 1);
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
DrawDownScaleFaceRects(bgraMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GREEN, 1);
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
len = resultObjects.Count;
for (int i = 0; i < len; i++)
{
Rect rect = resultObjects[i];
if (enableDownScale)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
}
// draw face rect
Imgproc.rectangle(bgraMat, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), COLOR_RED, 2);
}
}
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(bgraMat, texture);
bgraMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
}
// Update is called once per frame
void Update()
{
if (sourceToMatHelper.IsPlaying() && sourceToMatHelper.DidUpdateThisFrame())
{
Mat rgbMat = sourceToMatHelper.GetMat();
// detect faces.
List <Rect> detectResult = new List <Rect>();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
foreach (Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
}
// face tracking.
List <TrackedRect> trackedRects = new List <TrackedRect>();
rectangleTracker.UpdateTrackedObjects(detectResult);
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict[openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict[openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
opticalFlowFilterDict[openCVRect.id].Process(rgbMat, points, points);
lowPassFilterDict[openCVRect.id].Process(rgbMat, points, points);
}
landmarkPoints.Add(points);
}
}
// filter non frontal faces.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit)
{
trackedRects.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// face swapping.
if (landmarkPoints.Count >= 2)
{
int ann = 0, bob = 1;
for (int i = 0; i < landmarkPoints.Count - 1; i += 2)
{
ann = i;
bob = i + 1;
faceSwapper.SwapFaces(rgbMat, landmarkPoints[ann], landmarkPoints[bob], 1);
}
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < trackedRects.Count; i++)
{
Rect openCVRect = trackedRects[i];
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2);
//Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
//Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbMat, texture);
}
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Mat downScaleFrameMat = imageOptimizationHelper.GetDownScaleMat(bgraMat);
Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() => {
isThreadRunning = true;
DetectObject();
isThreadRunning = false;
OnDetectionDone();
});
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
Mat bgraMat4preview = null;
if (displayCameraPreview)
{
bgraMat4preview = new Mat();
downScaleFrameMat.copyTo(bgraMat4preview);
}
List <Vector2> points = null;
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
}
lock (rectangleTracker) {
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
if (rects.Length > 0)
{
OpenCVForUnity.Rect rect = rects [0];
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
if (enableOpticalFlowFilter)
{
opticalFlowFilter.Process(bgraMat, points, points, false);
}
if (displayCameraPreview && bgraMat4preview != null)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2);
}
}
}
else
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = detectionResult.toArray();
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
if (resultObjects.Count > 0)
{
OpenCVForUnity.Rect rect = resultObjects [0];
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
if (enableOpticalFlowFilter)
{
opticalFlowFilter.Process(bgraMat, points, points, false);
}
if (displayCameraPreview && bgraMat4preview != null)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(bgraMat4preview, points, new Scalar(0, 255, 0, 255), 2);
}
}
}
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
if (displayCameraPreview && bgraMat4preview != null)
{
OpenCVForUnity.Utils.fastMatToTexture2D(bgraMat4preview, texture);
}
if (points != null)
{
UpdateARHeadTransform(points, cameraToWorldMatrix);
}
bgraMat.Dispose();
if (bgraMat4preview != null)
{
bgraMat4preview.Dispose();
}
}, false);
}
// Update is called once per frame
void Update()
{
lock (sync) {
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enable && !detecting)
{
detecting = true;
grayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
if (!isShowingWebCamImage)
{
// fill all black.
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
}
Rect[] rects;
if (!isUsingSeparateDetection)
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
rectangleTracker.GetObjects(resultObjects, true);
rects = resultObjects.ToArray();
// rects = rectangleTracker.CreateCorrectionBySpeedOfRects ();
for (int i = 0; i < rects.Length; i++)
{
//Debug.Log ("detected face[" + i + "] " + rects [i]);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2);
}
}
else
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.toArray();
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 1);
}
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1);
}
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
rects = resultObjects.ToArray();
for (int i = 0; i < rects.Length; i++)
{
//Debug.Log ("detected face[" + i + "] " + rects [i]);
Imgproc.rectangle(rgbaMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 2);
}
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;;
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
texture.wrapMode = TextureWrapMode.Clamp;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// detect faces.
List <Rect> detectResult;
DetectFaces(rgbaMat, out detectResult, useDlibFaceDetecter);
// face tracking.
List <TrackedRect> trackedRects = new List <TrackedRect>();
rectangleTracker.UpdateTrackedObjects(detectResult);
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict[openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict[openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
opticalFlowFilterDict[openCVRect.id].Process(rgbaMat, points, points);
lowPassFilterDict[openCVRect.id].Process(rgbaMat, points, points);
}
landmarkPoints.Add(points);
}
}
// filter non frontal faces.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit)
{
trackedRects.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// face changing.
if (faceMaskTexture != null && landmarkPoints.Count >= 1)
{ // Apply face changing between detected faces and a mask image.
if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f)
{
DetectFaces(faceMaskMat, out detectResult, useDlibFaceDetecter);
if (detectResult.Count >= 1)
{
detectedFaceRect = new UnityEngine.Rect(detectResult[0].x, detectResult[0].y, detectResult[0].width, detectResult[0].height);
}
}
if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat);
List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect);
faceChanger.SetTargetImage(rgbaMat);
for (int i = 0; i < landmarkPoints.Count; i++)
{
faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints[i], 1);
}
faceChanger.ChangeFace();
if (displayFaceRects)
{
OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2);
}
}
}
else if (landmarkPoints.Count >= 2)
{ // Apply face changing between detected faces.
faceChanger.SetTargetImage(rgbaMat);
for (int i = 1; i < landmarkPoints.Count; i++)
{
faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints[0], landmarkPoints[i], 1);
}
faceChanger.ChangeFace();
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < trackedRects.Count; i++)
{
Rect openCVRect = trackedRects[i];
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
// display face mask image.
if (faceMaskMat != null)
{
float scale = (rgbaMat.width() / 4f) / faceMaskMat.width();
float tx = rgbaMat.width() - faceMaskMat.width() * scale;
float ty = 0.0f;
Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty);
trans.put(0, 0, scale);
trans.put(0, 1, 0.0f);
trans.put(0, 2, tx);
trans.put(1, 0, 0.0f);
trans.put(1, 1, scale);
trans.put(1, 2, ty);
Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0));
if (displayFaceRects || displayDebugFacePoints)
{
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics)
{
isDetectingInFrameArrivedThread = true;
DebugUtils.VideoTick();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
if (useOpenCVDetector)
{
Imgproc.equalizeHist(downScaleMat, downScaleMat);
}
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
if (useOpenCVDetector)
{
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread, true);
}
else
{
DetectObject(grayMat4Thread, out detectionResult, faceLandmarkDetector4Thread);
}
isThreadRunning = false;
OnDetectionDone();
});
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
resultFaceLandmarkPoints.Clear();
foreach (Rect rect in resultObjects)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
resultFaceLandmarkPoints.Add(points);
}
if (!displayCameraImage)
{
// fill all black
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (displayDetectedFaceRect)
{
// draw face rects
foreach (Rect rect in resultObjects)
{
OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2);
}
}
// draw face landmark points
foreach (List <Vector2> points in resultFaceLandmarkPoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4);
}
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//});
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
}
else
{
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions from previous positions");
//});
if (useOpenCVDetector)
{
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
else
{
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateRawRects();
}
}
}
detectedObjectsInRegions.Clear();
foreach (Rect rect in rectsWhereRegions)
{
if (useOpenCVDetector)
{
DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, cascade, true);
}
else
{
DetectInRegion(downScaleMat, rect, detectedObjectsInRegions, faceLandmarkDetector);
}
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, false);
}
// set original size image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
resultFaceLandmarkPoints.Clear();
foreach (Rect rect in resultObjects)
{
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
// detect face landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
resultFaceLandmarkPoints.Add(points);
}
if (!displayCameraImage)
{
// fill all black
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
if (displayDetectedFaceRect)
{
// draw previous rects
DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1);
// draw face rects
foreach (Rect rect in resultObjects)
{
OpenCVForUnityUtils.DrawFaceRect(grayMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), COLOR_GRAY, 2);
}
}
// draw face landmark points
foreach (List <Vector2> points in resultFaceLandmarkPoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(grayMat, points, COLOR_WHITE, 4);
}
}
DebugUtils.TrackTick();
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(grayMat, texture);
grayMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
isDetectingInFrameArrivedThread = false;
}
// Update is called once per frame
void Update()
{
lock (sync) {
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enable && !detecting)
{
detecting = true;
grayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
Rect[] rects;
if (!isUsingSeparateDetection)
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
rectangleTracker.GetObjects(resultObjects, true);
rects = rectangleTracker.CreateCorrectionBySpeedOfRects();
if (rects.Length > 0)
{
OpenCVForUnity.Rect rect = rects [0];
// Adjust to Dilb's result.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
UpdateARHeadTransform(points);
}
}
else
{
if (didUpdateTheDetectionResult)
{
didUpdateTheDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.toArray();
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
detectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
if (resultObjects.Count > 0)
{
OpenCVForUnity.Rect rect = resultObjects [0];
// Adjust to Dilb's result.
rect.y += (int)(rect.height * 0.1f);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
UpdateARHeadTransform(points);
}
}
}
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Imgproc.cvtColor(bgraMat, grayMat, Imgproc.COLOR_BGRA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() => {
isThreadRunning = true;
DetectObject();
isThreadRunning = false;
OnDetectionDone();
});
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
}
lock (rectangleTracker) {
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
}
else
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = detectionResult.toArray();
}
rects = rectsWhereRegions;
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
rects = rectsWhereRegions;
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
}
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
DrawRects(rects, bgraMat.width(), bgraMat.height());
bgraMat.Dispose();
Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance));
Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance));
//position
Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos);
gameObject.transform.position = position;
//scale
Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1);
gameObject.transform.localScale = scale;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.rotation = rotation;
rectOverlay.UpdateOverlayTransform(gameObject.transform);
}, false);
}
// Update is called once per frame
void Update()
{
lock (sync) {
while (ExecuteOnMainThread.Count > 0)
{
ExecuteOnMainThread.Dequeue().Invoke();
}
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetDownScaleMat(webCamTextureToMatHelper.GetMat());
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
StartThread(ThreadWorker);
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
rectangleTracker.GetObjects(resultObjects, true);
rects = resultObjects.ToArray();
// rects = rectangleTracker.CreateCorrectionBySpeedOfRects ();
DrawRects(rects, grayMat.width(), grayMat.height());
}
else
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Debug.Log("process: get rectsWhereRegions were got from detectionResult");
rectsWhereRegions = detectionResult.toArray();
// rects = rectsWhereRegions;
// for (int i = 0; i < rects.Length; i++) {
// Imgproc.rectangle (rgbaMat, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (0, 0, 255, 255), 1);
// }
}
else
{
//Debug.Log("process: get rectsWhereRegions from previous positions");
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
// rects = rectsWhereRegions;
// for (int i = 0; i < rects.Length; i++) {
// Imgproc.rectangle (rgbaMat, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (0, 255, 0, 255), 1);
// }
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
rects = resultObjects.ToArray();
DrawRects(rects, grayMat.width(), grayMat.height());
}
}
if (webCamTextureToMatHelper.IsPlaying())
{
Matrix4x4 cameraToWorldMatrix = Camera.main.cameraToWorldMatrix;;
Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance));
Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance));
//position
Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos);
gameObject.transform.position = position;
//scale
Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1);
gameObject.transform.localScale = scale;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.rotation = rotation;
rectOverlay.UpdateOverlayTransform(gameObject.transform);
}
}
public void OnFrameMatAcquired(Mat grayMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix, CameraIntrinsics cameraIntrinsics)
{
isDetectingInFrameArrivedThread = true;
DebugUtils.VideoTick();
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(grayMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = grayMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.equalizeHist(downScaleMat, downScaleMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
downScaleMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread);
isThreadRunning = false;
OnDetectionDone();
});
}
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
// draw face rect.
DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6);
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//});
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
}
else
{
//Enqueue(() =>
//{
// Debug.Log("process: get rectsWhereRegions from previous positions");
//});
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(downScaleMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(grayMat, new Point(0, 0), new Point(grayMat.width(), grayMat.height()), new Scalar(0, 0, 0, 0), -1);
}
// draw previous rect.
DrawDownScaleFaceRects(grayMat, rectsWhereRegions, DOWNSCALE_RATIO, COLOR_GRAY, 1);
// draw face rect.
DrawDownScaleFaceRects(grayMat, resultObjects.ToArray(), DOWNSCALE_RATIO, COLOR_WHITE, 6);
}
DebugUtils.TrackTick();
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
Utils.fastMatToTexture2D(grayMat, texture);
grayMat.Dispose();
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2) * 2.2f;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
});
isDetectingInFrameArrivedThread = false;
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect()) {
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
}
// face traking.
if (enableTracking)
{
rectangleTracker.UpdateTrackedObjects(detectResult);
detectResult = new List <OpenCVForUnity.Rect> ();
rectangleTracker.GetObjects(detectResult, true);
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> > ();
foreach (var openCVRect in detectResult)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
landmarkPoints.Add(points);
}
// filter non frontal faces.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit)
{
detectResult.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// face changeing.
if (faceMaskTexture != null && landmarkPoints.Count >= 1)
{
OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f)
{
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
if (result.Count >= 1)
{
detectedFaceRect = result [0];
}
}
else
{
using (Mat grayMat = new Mat())
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect()) {
//convert image to greyscale
Imgproc.cvtColor(faceMaskMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
//detectResult = faces.toList ();
List <OpenCVForUnity.Rect> faceList = faces.toList();
if (faceList.Count >= 1)
{
detectedFaceRect = new UnityEngine.Rect(faceList [0].x, faceList [0].y, faceList [0].width, faceList [0].height);
// Adjust to Dilb's result.
detectedFaceRect.y += detectedFaceRect.height * 0.1f;
}
}
}
}
if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat);
List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect);
faceChanger.SetTargetImage(rgbaMat);
for (int i = 0; i < landmarkPoints.Count; i++)
{
faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints [i], 1);
}
faceChanger.ChangeFace();
if (displayFaceRects)
{
OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2);
}
}
}
else if (landmarkPoints.Count >= 2)
{
faceChanger.SetTargetImage(rgbaMat);
for (int i = 1; i < landmarkPoints.Count; i++)
{
faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints [0], landmarkPoints [i], 1);
}
faceChanger.ChangeFace();
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < detectResult.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceParam.getAngleOfFrontalFace (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceParam.getFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
// display face mask image.
if (faceMaskMat != null)
{
float scale = (rgbaMat.width() / 4f) / faceMaskMat.width();
float tx = rgbaMat.width() - faceMaskMat.width() * scale;
float ty = 0.0f;
Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty);
trans.put(0, 0, scale);
trans.put(0, 1, 0.0f);
trans.put(0, 2, tx);
trans.put(1, 0, 0.0f);
trans.put(1, 1, scale);
trans.put(1, 2, ty);
Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0));
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect()) {
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
}
// face traking.
if (enableTracking)
{
rectangleTracker.UpdateTrackedObjects(detectResult);
detectResult = new List <OpenCVForUnity.Rect> ();
rectangleTracker.GetObjects(detectResult, true);
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> > ();
foreach (var openCVRect in detectResult)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
landmarkPoints.Add(points);
}
// filter non frontal faces.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit)
{
detectResult.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// face swapping.
if (landmarkPoints.Count >= 2)
{
int ann = 0, bob = 1;
for (int i = 0; i < landmarkPoints.Count - 1; i += 2)
{
ann = i;
bob = i + 1;
faceSwapper.SwapFaces(rgbaMat, landmarkPoints [ann], landmarkPoints [bob], 1);
}
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < detectResult.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceParam.getAngleOfFrontalFace (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceParam.getFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors);
}
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Mat downScaleMat = null;
float DOWNSCALE_RATIO;
if (enableDownScale)
{
downScaleMat = imageOptimizationHelper.GetDownScaleMat(bgraMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleMat = bgraMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.cvtColor(downScaleMat, grayMat, CVTCOLOR_CODE);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() =>
{
isThreadRunning = true;
DetectObject(grayMat4Thread, out detectionResult, cascade4Thread);
isThreadRunning = false;
OnDetectionDone();
});
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectionResult);
}
}
lock (rectangleTracker)
{
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
}
else
{
Rect[] rectsWhereRegions;
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = detectionResult.ToArray();
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker)
{
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
}
detectedObjectsInRegions.Clear();
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions[i], detectedObjectsInRegions, cascade);
}
lock (rectangleTracker)
{
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
}
if (enableDownScale)
{
int len = rects.Length;
for (int i = 0; i < len; i++)
{
Rect rect = rects[i];
// restore to original size rect
rect.x = (int)(rect.x * DOWNSCALE_RATIO);
rect.y = (int)(rect.y * DOWNSCALE_RATIO);
rect.width = (int)(rect.width * DOWNSCALE_RATIO);
rect.height = (int)(rect.height * DOWNSCALE_RATIO);
}
}
Enqueue(() =>
{
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
DrawRects(rects, bgraMat.width(), bgraMat.height());
bgraMat.Dispose();
Vector3 ccCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(0.0f, 0.0f, overlayDistance));
Vector3 tlCameraSpacePos = UnProjectVector(projectionMatrix, new Vector3(-overlayDistance, overlayDistance, overlayDistance));
//position
Vector3 position = cameraToWorldMatrix.MultiplyPoint3x4(ccCameraSpacePos);
gameObject.transform.position = position;
//scale
Vector3 scale = new Vector3(Mathf.Abs(tlCameraSpacePos.x - ccCameraSpacePos.x) * 2, Mathf.Abs(tlCameraSpacePos.y - ccCameraSpacePos.y) * 2, 1);
gameObject.transform.localScale = scale;
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.rotation = rotation;
rectOverlay.UpdateOverlayTransform(gameObject.transform);
});
}
// Update is called once per frame
void Update()
{
// loop play.
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
if (capture.grab())
{
capture.retrieve(rgbMat, 0);
Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB);
//Debug.Log ("Mat toString " + rgbMat.ToString ());
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect()) {
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
}
// face tracking.
rectangleTracker.UpdateTrackedObjects(detectResult);
List <TrackedRect> trackedRects = new List <TrackedRect> ();
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict [openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict [openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// create LUT texture.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
faceMaskColorCorrector.CreateLUTTex(openCVRect.id);
}
else if (openCVRect.state == TrackedState.DELETED)
{
faceMaskColorCorrector.DeleteLUTTex(openCVRect.id);
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> > ();
for (int i = 0; i < trackedRects.Count; i++)
{
TrackedRect tr = trackedRects [i];
UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
if (tr.state > TrackedState.NEW && tr.state < TrackedState.DELETED)
{
opticalFlowFilterDict [tr.id].Process(rgbMat, points, points);
lowPassFilterDict [tr.id].Process(rgbMat, points, points);
}
}
landmarkPoints.Add(points);
}
// face masking.
if (faceMaskTexture != null && landmarkPoints.Count >= 1) // Apply face masking between detected faces and a face mask image.
{
float maskImageWidth = faceMaskTexture.width;
float maskImageHeight = faceMaskTexture.height;
TrackedRect tr;
for (int i = 0; i < trackedRects.Count; i++)
{
tr = trackedRects [i];
if (tr.state == TrackedState.NEW)
{
meshOverlay.CreateObject(tr.id, faceMaskTexture);
}
if (tr.state < TrackedState.DELETED)
{
MaskFace(meshOverlay, tr, landmarkPoints [i], faceLandmarkPointsInMask, maskImageWidth, maskImageHeight);
if (enableColorCorrection)
{
CorrectFaceMaskColor(tr.id, faceMaskMat, rgbMat, faceLandmarkPointsInMask, landmarkPoints [i]);
}
}
else if (tr.state == TrackedState.DELETED)
{
meshOverlay.DeleteObject(tr.id);
}
}
}
else if (landmarkPoints.Count >= 1) // Apply face masking between detected faces.
{
float maskImageWidth = texture.width;
float maskImageHeight = texture.height;
TrackedRect tr;
for (int i = 0; i < trackedRects.Count; i++)
{
tr = trackedRects [i];
if (tr.state == TrackedState.NEW)
{
meshOverlay.CreateObject(tr.id, texture);
}
if (tr.state < TrackedState.DELETED)
{
MaskFace(meshOverlay, tr, landmarkPoints [i], landmarkPoints [0], maskImageWidth, maskImageHeight);
if (enableColorCorrection)
{
CorrectFaceMaskColor(tr.id, rgbMat, rgbMat, landmarkPoints [0], landmarkPoints [i]);
}
}
else if (tr.state == TrackedState.DELETED)
{
meshOverlay.DeleteObject(tr.id);
}
}
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < detectResult.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2);
}
for (int i = 0; i < trackedRects.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects [i].x, trackedRects [i].y, trackedRects [i].width, trackedRects [i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 255, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
// draw face points.
if (displayDebugFacePoints)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2);
}
}
// display face mask image.
if (faceMaskTexture != null && faceMaskMat != null)
{
if (displayFaceRects)
{
OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2);
}
if (displayDebugFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2);
}
float scale = (rgbMat.width() / 4f) / faceMaskMat.width();
float tx = rgbMat.width() - faceMaskMat.width() * scale;
float ty = 0.0f;
Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty);
trans.put(0, 0, scale);
trans.put(0, 1, 0.0f);
trans.put(0, 2, tx);
trans.put(1, 0, 0.0f);
trans.put(1, 1, scale);
trans.put(1, 2, ty);
Imgproc.warpAffine(faceMaskMat, rgbMat, trans, rgbMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0));
if (displayFaceRects || displayDebugFacePointsToggle)
{
OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
}
}
// Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbMat, texture);
}
}
public void OnFrameMatAcquired(Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4x4 cameraToWorldMatrix)
{
Mat downScaleFrameMat = bgraMat;
if (imageOptimizationHelper.downscaleRatio != 1)
{
downScaleFrameMat = new Mat();
Imgproc.resize(bgraMat, downScaleFrameMat, new Size(), 1.0 / imageOptimizationHelper.downscaleRatio, 1.0 / imageOptimizationHelper.downscaleRatio, Imgproc.INTER_LINEAR);
}
Imgproc.cvtColor(downScaleFrameMat, grayMat, Imgproc.COLOR_BGRA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (enableDetection && !isDetecting)
{
isDetecting = true;
grayMat.copyTo(grayMat4Thread);
System.Threading.Tasks.Task.Run(() => {
isThreadRunning = true;
DetectObject();
isThreadRunning = false;
OnDetectionDone();
});
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, grayMat);
if (!displayCameraImage)
{
// fill all black.
Imgproc.rectangle(downScaleFrameMat, new Point(0, 0), new Point(downScaleFrameMat.width(), downScaleFrameMat.height()), new Scalar(0, 0, 0, 0), -1);
}
Rect[] rects;
if (!useSeparateDetection)
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectionResult.toList());
}
}
lock (rectangleTracker) {
rectangleTracker.GetObjects(resultObjects, true);
}
rects = resultObjects.ToArray();
if (displayDetectedFaceRect)
{
for (int i = 0; i < rects.Length; i++)
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log ("detected face[" + i + "] " + rects [i]);
//}, true);
Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3);
}
}
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in resultObjects)
{
r.y += (int)(r.height * 0.1f);
}
foreach (var rect in resultObjects)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2);
if (displayDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
else
{
if (hasUpdatedDetectionResult)
{
hasUpdatedDetectionResult = false;
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions were got from detectionResult");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = detectionResult.toArray();
}
if (displayDetectedFaceRect)
{
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(255, 0, 0, 255), 1);
}
}
}
else
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log("process: get rectsWhereRegions from previous positions");
//}, true);
lock (rectangleTracker) {
rectsWhereRegions = rectangleTracker.CreateCorrectionBySpeedOfRects();
}
if (displayDetectedFaceRect)
{
rects = rectsWhereRegions;
for (int i = 0; i < rects.Length; i++)
{
Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 255, 0, 255), 1);
}
}
}
detectedObjectsInRegions.Clear();
if (rectsWhereRegions.Length > 0)
{
int len = rectsWhereRegions.Length;
for (int i = 0; i < len; i++)
{
DetectInRegion(grayMat, rectsWhereRegions [i], detectedObjectsInRegions);
}
}
lock (rectangleTracker) {
rectangleTracker.UpdateTrackedObjects(detectedObjectsInRegions);
rectangleTracker.GetObjects(resultObjects, true);
}
if (displayDetectedFaceRect)
{
rects = resultObjects.ToArray();
for (int i = 0; i < rects.Length; i++)
{
//UnityEngine.WSA.Application.InvokeOnAppThread (() => {
// Debug.Log ("detected face[" + i + "] " + rects [i]);
//}, true);
Imgproc.rectangle(downScaleFrameMat, new Point(rects [i].x, rects [i].y), new Point(rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar(0, 0, 255, 255), 3);
}
}
// Adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in resultObjects)
{
r.y += (int)(r.height * 0.1f);
}
foreach (var rect in resultObjects)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height));
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(downScaleFrameMat, points, new Scalar(0, 255, 0, 255), 2);
if (displayDetectedFaceRect)
{
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(downScaleFrameMat, new UnityEngine.Rect(rect.x, rect.y, rect.width, rect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
if (!webCamTextureToMatHelper.IsPlaying())
{
return;
}
OpenCVForUnity.Utils.fastMatToTexture2D(downScaleFrameMat, texture);
bgraMat.Dispose();
if (imageOptimizationHelper.downscaleRatio != 1)
{
downScaleFrameMat.Dispose();
}
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
quad_renderer.sharedMaterial.SetMatrix("_WorldToCameraMatrix", worldToCameraMatrix);
// Position the canvas object slightly in front
// of the real world web camera.
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
gameObject.transform.position = position;
gameObject.transform.rotation = rotation;
}, false);
}