// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (capture == null)
{
return;
}
//Loop play
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
//error PlayerLoop called recursively! on iOS.reccomend WebCamTexture.
if (capture.grab())
{
capture.retrieve(rgbMat, 0);
Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB);
//Debug.Log ("Mat toString " + rgbMat.ToString ());
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, points, new Scalar(0, 255, 0), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0), 2);
}
Imgproc.putText(rgbMat, "W:" + rgbMat.width() + " H:" + rgbMat.height() + " SO:" + Screen.orientation, new Point(5, rgbMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbMat, texture, colors);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Mat downScaleRgbaMat = webCamTextureToMatHelper.GetDownScaleMat(rgbaMat);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, downScaleRgbaMat);
// Detect faces on resize image
if (!webCamTextureToMatHelper.IsSkipFrame())
{
//detect face rects
detectResult = faceLandmarkDetector.Detect();
}
float DOWNSCALE_RATIO = webCamTextureToMatHelper.DOWNSCALE_RATIO;
foreach (var rect in detectResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
List <Vector2> originalPoints = new List <Vector2> (points.Count);
foreach (var point in points)
{
originalPoints.Add(new Vector2(point.x * DOWNSCALE_RATIO, point.y * DOWNSCALE_RATIO));
}
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, originalPoints, new Scalar(0, 255, 0, 255), 2);
UnityEngine.Rect originalRect = new UnityEngine.Rect(rect.x * DOWNSCALE_RATIO, rect.y * DOWNSCALE_RATIO, rect.width * DOWNSCALE_RATIO, rect.height * DOWNSCALE_RATIO);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, originalRect, new Scalar(255, 0, 0, 255), 2);
}
Imgproc.putText(rgbaMat, "Original: (" + rgbaMat.width() + "," + rgbaMat.height() + ") DownScale; (" + downScaleRgbaMat.width() + "," + downScaleRgbaMat.height() + ") SkipFrames: " + webCamTextureToMatHelper.SKIP_FRAMES, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Mat downScaleRgbaMat = null;
float DOWNSCALE_RATIO = 1.0f;
if (enableDownScale)
{
downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(rgbaMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleRgbaMat = rgbaMat;
DOWNSCALE_RATIO = 1.0f;
}
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, downScaleRgbaMat);
// Detect faces on resize image
if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped())
{
//detect face rects
if (useOpenCVFaceDetector)
{
// convert image to greyscale.
Imgproc.cvtColor(downScaleRgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect()) {
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
List <OpenCVForUnity.Rect> opencvDetectResult = faces.toList();
// adjust to Dilb's result.
detectionResult.Clear();
foreach (var opencvRect in opencvDetectResult)
{
detectionResult.Add(new UnityEngine.Rect((float)opencvRect.x, (float)opencvRect.y + (float)(opencvRect.height * 0.1f), (float)opencvRect.width, (float)opencvRect.height));
}
}
}
else
{
detectionResult = faceLandmarkDetector.Detect();
}
}
foreach (var rect in detectionResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
List <Vector2> originalPoints = new List <Vector2> (points.Count);
foreach (var point in points)
{
originalPoints.Add(new Vector2(point.x * DOWNSCALE_RATIO, point.y * DOWNSCALE_RATIO));
}
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, originalPoints, new Scalar(0, 255, 0, 255), 2);
UnityEngine.Rect originalRect = new UnityEngine.Rect(rect.x * DOWNSCALE_RATIO, rect.y * DOWNSCALE_RATIO, rect.width * DOWNSCALE_RATIO, rect.height * DOWNSCALE_RATIO);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, originalRect, new Scalar(255, 0, 0, 255), 2);
}
//Imgproc.putText (rgbaMat, "Original:(" + rgbaMat.width () + "," + rgbaMat.height () + ") DownScale:(" + downScaleRgbaMat.width () + "," + downScaleRgbaMat.height () + ") FrameSkipping: " + imageOptimizationHelper.frameSkippingRatio, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
if (detectResult.Count > 0)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult [0]);
if (isShowingFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
imagePoints.fromArray(
new Point((points [38].x + points [41].x) / 2, (points [38].y + points [41].y) / 2), //l eye
new Point((points [43].x + points [46].x) / 2, (points [43].y + points [46].y) / 2), //r eye
new Point(points [33].x, points [33].y), //nose
new Point(points [48].x, points [48].y), //l mouth
new Point(points [54].x, points [54].y) //r mouth
,
new Point(points [0].x, points [0].y), //l ear
new Point(points [16].x, points [16].y) //r ear
);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
if (tvec.get(2, 0) [0] > 0)
{
if (Mathf.Abs((float)(points [43].y - points [46].y)) > Mathf.Abs((float)(points [42].x - points [45].x)) / 6.0)
{
if (isShowingEffects)
{
rightEye.SetActive(true);
}
}
if (Mathf.Abs((float)(points [38].y - points [41].y)) > Mathf.Abs((float)(points [39].x - points [36].x)) / 6.0)
{
if (isShowingEffects)
{
leftEye.SetActive(true);
}
}
if (isShowingHead)
{
head.SetActive(true);
}
if (isShowingAxes)
{
axes.SetActive(true);
}
float noseDistance = Mathf.Abs((float)(points [27].y - points [33].y));
float mouseDistance = Mathf.Abs((float)(points [62].y - points [66].y));
if (mouseDistance > noseDistance / 5.0)
{
if (isShowingEffects)
{
mouth.SetActive(true);
foreach (ParticleSystem ps in mouthParticleSystem)
{
ps.enableEmission = true;
ps.startSize = 40 * (mouseDistance / noseDistance);
}
}
}
else
{
if (isShowingEffects)
{
foreach (ParticleSystem ps in mouthParticleSystem)
{
ps.enableEmission = false;
}
}
}
// position
Vector4 pos = new Vector4((float)tvec.get(0, 0) [0], (float)tvec.get(1, 0) [0], (float)tvec.get(2, 0) [0], 0); // from OpenCV
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARGameObject.transform.localPosition = new Vector3(pos.x, -pos.y, pos.z);
// rotation
Calib3d.Rodrigues(rvec, rotMat);
Vector3 forward = new Vector3((float)rotMat.get(0, 2) [0], (float)rotMat.get(1, 2) [0], (float)rotMat.get(2, 2) [0]); // from OpenCV
Vector3 up = new Vector3((float)rotMat.get(0, 1) [0], (float)rotMat.get(1, 1) [0], (float)rotMat.get(2, 1) [0]); // from OpenCV
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
Quaternion rot = Quaternion.LookRotation(new Vector3(forward.x, -forward.y, forward.z), new Vector3(up.x, -up.y, up.z));
ARGameObject.transform.localRotation = rot;
// Apply Z axis inverted matrix.
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
transformationM = ARGameObject.transform.localToWorldMatrix * invertZM;
// Apply camera transform matrix.
transformationM = ARCamera.transform.localToWorldMatrix * transformationM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref transformationM);
}
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
UnityEngine.Rect rect = new UnityEngine.Rect();
List <Vector2> points = null;
bool shouldResetfilter = false;
if (detectResult.Count > 0)
{
rect = detectResult [0];
//detect landmark points
points = faceLandmarkDetector.DetectLandmark(rect);
skippedFrames = 0;
}
else
{
skippedFrames++;
if (skippedFrames == maximumAllowedSkippedFrames)
{
shouldResetfilter = true;
}
}
switch (filterMode)
{
default:
case FilterMode.None:
break;
case FilterMode.LowPassFilter:
if (shouldResetfilter)
{
lowPassFilter.Reset();
}
lowPassFilter.Process(rgbaMat, points, lowPassFilteredPoints, isDebugMode);
break;
case FilterMode.KalmanFilter:
if (shouldResetfilter)
{
kalmanFilter.Reset();
}
kalmanFilter.Process(rgbaMat, points, kalmanFilteredPoints, isDebugMode);
break;
case FilterMode.OpticalFlowFilter:
if (shouldResetfilter)
{
opticalFlowFilter.Reset();
}
opticalFlowFilter.Process(rgbaMat, points, opticalFlowFilteredPoints, isDebugMode);
break;
case FilterMode.OFAndLPFilter:
if (shouldResetfilter)
{
opticalFlowFilter.Reset();
lowPassFilter.Reset();
}
opticalFlowFilter.Process(rgbaMat, points, points, false);
lowPassFilter.Process(rgbaMat, points, ofAndLPFilteredPoints, isDebugMode);
break;
}
if (points != null && !isDebugMode)
{
// draw raw landmark points.
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
// draw face rect.
// OpenCVForUnityUtils.DrawFaceRect (rgbaMat, rect, new Scalar (255, 0, 0, 255), 2);
// draw filtered lam points.
if (points != null && !isDebugMode)
{
switch (filterMode)
{
default:
case FilterMode.None:
break;
case FilterMode.LowPassFilter:
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, lowPassFilteredPoints, new Scalar(0, 255, 255, 255), 2);
break;
case FilterMode.KalmanFilter:
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, kalmanFilteredPoints, new Scalar(0, 0, 255, 255), 2);
break;
case FilterMode.OpticalFlowFilter:
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, opticalFlowFilteredPoints, new Scalar(255, 0, 0, 255), 2);
break;
case FilterMode.OFAndLPFilter:
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, ofAndLPFilteredPoints, new Scalar(255, 0, 255, 255), 2);
break;
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
// Convert Unity Texture2D to OpenCV Mat.
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(imgTexture, imgMat);
Debug.Log("imgMat dst ToString " + imgMat.ToString());
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = imgMat.width();
float height = imgMat.height();
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
FaceLandmarkDetector faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, imgMat);
//detect face rectdetecton
List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection();
foreach (var result in detectResult)
{
Debug.Log("rect : " + result.rect);
Debug.Log("detection_confidence : " + result.detection_confidence);
Debug.Log("weight_index : " + result.weight_index);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(result.rect);
Debug.Log("face points count : " + points.Count);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2, true);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(imgMat, result, new Scalar(255, 0, 0, 255), 2);
}
faceLandmarkDetector.Dispose();
Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false);
// Convert OpenCV Mat to Unity Texture2D.
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(imgMat, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName);
fpsMonitor.Add("width", width.ToString());
fpsMonitor.Add("height", height.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// detect faces on the downscale image
if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped())
{
Mat downScaleRgbaMat = null;
float DOWNSCALE_RATIO = 1.0f;
if (enableDownScale)
{
downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(rgbaMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleRgbaMat = rgbaMat;
DOWNSCALE_RATIO = 1.0f;
}
// set the downscale mat
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, downScaleRgbaMat);
//detect face rects
detectionResult = faceLandmarkDetector.Detect();
if (enableDownScale)
{
for (int i = 0; i < detectionResult.Count; ++i)
{
var rect = detectionResult[i];
detectionResult[i] = new UnityEngine.Rect(
rect.x * DOWNSCALE_RATIO,
rect.y * DOWNSCALE_RATIO,
rect.width * DOWNSCALE_RATIO,
rect.height * DOWNSCALE_RATIO);
}
}
}
List <Vector2> points = null;
if (detectionResult != null && detectionResult.Count > 0)
{
// set the original scale image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect landmark points
points = faceLandmarkDetector.DetectLandmark(detectionResult[0]);
}
if (points != null)
{
if (displayFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
MatOfPoint3f objectPoints = null;
bool isRightEyeOpen = false;
bool isLeftEyeOpen = false;
bool isMouthOpen = false;
if (points.Count == 68)
{
objectPoints = objectPoints68;
imagePoints.fromArray(
new Point((points[38].x + points[41].x) / 2, (points[38].y + points[41].y) / 2), //l eye (Interpupillary breadth)
new Point((points[43].x + points[46].x) / 2, (points[43].y + points[46].y) / 2), //r eye (Interpupillary breadth)
new Point(points[30].x, points[30].y), //nose (Tip)
new Point(points[33].x, points[33].y), //nose (Subnasale)
new Point(points[0].x, points[0].y), //l ear (Bitragion breadth)
new Point(points[16].x, points[16].y) //r ear (Bitragion breadth)
);
if (Mathf.Abs((float)(points[43].y - points[46].y)) > Mathf.Abs((float)(points[42].x - points[45].x)) / 5.0)
{
isRightEyeOpen = true;
}
if (Mathf.Abs((float)(points[38].y - points[41].y)) > Mathf.Abs((float)(points[39].x - points[36].x)) / 5.0)
{
isLeftEyeOpen = true;
}
float noseDistance = Mathf.Abs((float)(points[27].y - points[33].y));
float mouseDistance = Mathf.Abs((float)(points[62].y - points[66].y));
if (mouseDistance > noseDistance / 5.0)
{
isMouthOpen = true;
}
else
{
isMouthOpen = false;
}
}
else if (points.Count == 17)
{
objectPoints = objectPoints17;
imagePoints.fromArray(
new Point((points[2].x + points[3].x) / 2, (points[2].y + points[3].y) / 2), //l eye (Interpupillary breadth)
new Point((points[4].x + points[5].x) / 2, (points[4].y + points[5].y) / 2), //r eye (Interpupillary breadth)
new Point(points[0].x, points[0].y), //nose (Tip)
new Point(points[1].x, points[1].y), //nose (Subnasale)
new Point(points[6].x, points[6].y), //l ear (Bitragion breadth)
new Point(points[8].x, points[8].y) //r ear (Bitragion breadth)
);
if (Mathf.Abs((float)(points[11].y - points[12].y)) > Mathf.Abs((float)(points[4].x - points[5].x)) / 5.0)
{
isRightEyeOpen = true;
}
if (Mathf.Abs((float)(points[9].y - points[10].y)) > Mathf.Abs((float)(points[2].x - points[3].x)) / 5.0)
{
isLeftEyeOpen = true;
}
float noseDistance = Mathf.Abs((float)(points[3].y - points[1].y));
float mouseDistance = Mathf.Abs((float)(points[14].y - points[16].y));
if (mouseDistance > noseDistance / 2.0)
{
isMouthOpen = true;
}
else
{
isMouthOpen = false;
}
}
else if (points.Count == 6)
{
objectPoints = objectPoints6;
imagePoints.fromArray(
new Point((points[2].x + points[3].x) / 2, (points[2].y + points[3].y) / 2), //l eye (Interpupillary breadth)
new Point((points[4].x + points[5].x) / 2, (points[4].y + points[5].y) / 2), //r eye (Interpupillary breadth)
new Point(points[0].x, points[0].y), //nose (Tip)
new Point(points[1].x, points[1].y) //nose (Subnasale)
);
}
else if (points.Count == 5)
{
objectPoints = objectPoints5;
imagePoints.fromArray(
new Point(points[3].x, points[3].y), //l eye (Inner corner of the eye)
new Point(points[1].x, points[1].y), //r eye (Inner corner of the eye)
new Point(points[2].x, points[2].y), //l eye (Tail of the eye)
new Point(points[0].x, points[0].y), //r eye (Tail of the eye)
new Point(points[4].x, points[4].y) //nose (Subnasale)
);
if (fpsMonitor != null)
{
fpsMonitor.consoleText = "This example supports mainly the face landmark points of 68/17/6 points.";
}
}
// estimate head pose
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_x = tvec.get(0, 0)[0], tvec_y = tvec.get(1, 0)[0], tvec_z = tvec.get(2, 0)[0];
bool isNotInViewport = false;
Vector4 pos = VP * new Vector4((float)tvec_x, (float)tvec_y, (float)tvec_z, 1.0f);
if (pos.w != 0)
{
float x = pos.x / pos.w, y = pos.y / pos.w, z = pos.z / pos.w;
if (x < -1.0f || x > 1.0f || y < -1.0f || y > 1.0f || z < -1.0f || z > 1.0f)
{
isNotInViewport = true;
}
}
if (double.IsNaN(tvec_z) || isNotInViewport)
{ // if tvec is wrong data, do not use extrinsic guesses. (the estimated object is not in the camera field of view)
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
//Debug.Log (tvec.dump());
if (!isNotInViewport)
{
if (displayHead)
{
head.SetActive(true);
}
if (displayAxes)
{
axes.SetActive(true);
}
if (displayEffects)
{
rightEye.SetActive(isRightEyeOpen);
leftEye.SetActive(isLeftEyeOpen);
if (isMouthOpen)
{
mouth.SetActive(true);
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = true;
#if UNITY_5_5_OR_NEWER
var main = ps.main;
main.startSizeMultiplier = 20;
#else
ps.startSize = 20;
#endif
}
}
else
{
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = false;
}
}
}
// Convert to unity pose data.
double[] rvecArr = new double[3];
rvec.get(0, 0, rvecArr);
double[] tvecArr = new double[3];
tvec.get(0, 0, tvecArr);
PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr);
// Changes in pos/rot below these thresholds are ignored.
if (enableLowPassFilter)
{
ARUtils.LowpassPoseData(ref oldPoseData, ref poseData, positionLowPass, rotationLowPass);
}
oldPoseData = poseData;
// Create transform matrix.
transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one);
}
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
// https://stackoverflow.com/questions/30234945/change-handedness-of-a-row-major-4x4-transformation-matrix
ARM = invertYM * transformationM * invertYM;
// Apply Y-axis and Z-axis refletion matrix. (Adjust the posture of the AR object)
ARM = ARM * invertYM * invertZM;
if (shouldMoveARCamera)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
if (detectResult.Count > 0)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult [0]);
if (displayFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
imagePoints.fromArray(
new Point((points [38].x + points [41].x) / 2, (points [38].y + points [41].y) / 2), //l eye (Interpupillary breadth)
new Point((points [43].x + points [46].x) / 2, (points [43].y + points [46].y) / 2), //r eye (Interpupillary breadth)
new Point(points [30].x, points [30].y), //nose (Nose top)
new Point(points [48].x, points [48].y), //l mouth (Mouth breadth)
new Point(points [54].x, points [54].y), //r mouth (Mouth breadth)
new Point(points [0].x, points [0].y), //l ear (Bitragion breadth)
new Point(points [16].x, points [16].y) //r ear (Bitragion breadth)
);
// Estimate head pose.
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_z = tvec.get(2, 0) [0];
if (double.IsNaN(tvec_z) || tvec_z < 0) // if tvec is wrong data, do not use extrinsic guesses.
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
// Debug.Log (tvec.dump());
if (!double.IsNaN(tvec_z))
{
if (Mathf.Abs((float)(points [43].y - points [46].y)) > Mathf.Abs((float)(points [42].x - points [45].x)) / 6.0)
{
if (displayEffects)
{
rightEye.SetActive(true);
}
}
if (Mathf.Abs((float)(points [38].y - points [41].y)) > Mathf.Abs((float)(points [39].x - points [36].x)) / 6.0)
{
if (displayEffects)
{
leftEye.SetActive(true);
}
}
if (displayHead)
{
head.SetActive(true);
}
if (displayAxes)
{
axes.SetActive(true);
}
float noseDistance = Mathf.Abs((float)(points [27].y - points [33].y));
float mouseDistance = Mathf.Abs((float)(points [62].y - points [66].y));
if (mouseDistance > noseDistance / 5.0)
{
if (displayEffects)
{
mouth.SetActive(true);
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = true;
ps.startSize = 40 * (mouseDistance / noseDistance);
}
}
}
else
{
if (displayEffects)
{
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = false;
}
}
}
Calib3d.Rodrigues(rvec, rotMat);
transformationM.SetRow(0, new Vector4((float)rotMat.get(0, 0) [0], (float)rotMat.get(0, 1) [0], (float)rotMat.get(0, 2) [0], (float)tvec.get(0, 0) [0]));
transformationM.SetRow(1, new Vector4((float)rotMat.get(1, 0) [0], (float)rotMat.get(1, 1) [0], (float)rotMat.get(1, 2) [0], (float)tvec.get(1, 0) [0]));
transformationM.SetRow(2, new Vector4((float)rotMat.get(2, 0) [0], (float)rotMat.get(2, 1) [0], (float)rotMat.get(2, 2) [0], (float)tvec.get(2, 0) [0]));
transformationM.SetRow(3, new Vector4(0, 0, 0, 1));
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARM = invertYM * transformationM;
// Apply Z axis inverted matrix.
ARM = ARM * invertZM;
if (shouldMoveARCamera)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
}
}
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
// Update is called once per frame
void Update()
{
if (capture == null)
{
return;
}
//Loop play
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
if (capture.grab())
{
capture.retrieve(rgbMat, 0);
Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB);
//Debug.Log ("Mat toString " + rgbMat.ToString ());
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
if (detectResult.Count > 0)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult [0]);
if (displayFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, points, new Scalar(0, 255, 0), 2);
}
MatOfPoint3f objectPoints = null;
bool isRightEyeOpen = false;
bool isLeftEyeOpen = false;
bool isMouthOpen = false;
if (points.Count == 68)
{
objectPoints = objectPoints68;
imagePoints.fromArray(
new Point((points [38].x + points [41].x) / 2, (points [38].y + points [41].y) / 2), //l eye (Interpupillary breadth)
new Point((points [43].x + points [46].x) / 2, (points [43].y + points [46].y) / 2), //r eye (Interpupillary breadth)
new Point(points [30].x, points [30].y), //nose (Nose top)
new Point(points [48].x, points [48].y), //l mouth (Mouth breadth)
new Point(points [54].x, points [54].y), //r mouth (Mouth breadth)
new Point(points [0].x, points [0].y), //l ear (Bitragion breadth)
new Point(points [16].x, points [16].y) //r ear (Bitragion breadth)
);
if (Mathf.Abs((float)(points [43].y - points [46].y)) > Mathf.Abs((float)(points [42].x - points [45].x)) / 5.0)
{
isRightEyeOpen = true;
}
if (Mathf.Abs((float)(points [38].y - points [41].y)) > Mathf.Abs((float)(points [39].x - points [36].x)) / 5.0)
{
isLeftEyeOpen = true;
}
float noseDistance = Mathf.Abs((float)(points [27].y - points [33].y));
float mouseDistance = Mathf.Abs((float)(points [62].y - points [66].y));
if (mouseDistance > noseDistance / 5.0)
{
isMouthOpen = true;
}
else
{
isMouthOpen = false;
}
}
else if (points.Count == 5)
{
objectPoints = objectPoints5;
imagePoints.fromArray(
new Point(points [3].x, points [3].y), //l eye (Inner corner of the eye)
new Point(points [1].x, points [1].y), //r eye (Inner corner of the eye)
new Point(points [2].x, points [2].y), //l eye (Tail of the eye)
new Point(points [0].x, points [0].y), //r eye (Tail of the eye)
new Point(points [4].x, points [4].y) //nose (Nose top)
);
if (fpsMonitor != null)
{
fpsMonitor.consoleText = "This example supports mainly the face landmark points of 68 points.";
}
}
// Estimate head pose.
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_z = tvec.get(2, 0) [0];
if (double.IsNaN(tvec_z) || tvec_z < 0) // if tvec is wrong data, do not use extrinsic guesses.
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
//Debug.Log (tvec.dump());
if (!double.IsNaN(tvec_z))
{
// Display effects.
if (displayHead)
{
head.SetActive(true);
}
if (displayAxes)
{
axes.SetActive(true);
}
if (displayEffects)
{
rightEye.SetActive(isRightEyeOpen);
leftEye.SetActive(isLeftEyeOpen);
if (isMouthOpen)
{
mouth.SetActive(true);
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = true;
#if UNITY_5_5_OR_NEWER
var main = ps.main;
main.startSizeMultiplier = 20;
#else
ps.startSize = 20;
#endif
}
}
else
{
foreach (ParticleSystem ps in mouthParticleSystem)
{
var em = ps.emission;
em.enabled = false;
}
}
}
// Convert to unity pose data.
double[] rvecArr = new double[3];
rvec.get(0, 0, rvecArr);
double[] tvecArr = new double[3];
tvec.get(0, 0, tvecArr);
PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr);
// Changes in pos/rot below these thresholds are ignored.
if (enableLowPassFilter)
{
ARUtils.LowpassPoseData(ref oldPoseData, ref poseData, positionLowPass, rotationLowPass);
}
oldPoseData = poseData;
// Create transform matrix.
transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one);
}
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARM = invertYM * transformationM;
// Apply Z-axis inverted matrix.
ARM = ARM * invertZM;
if (shouldMoveARCamera)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
}
}
}
else
{
rightEye.SetActive(false);
leftEye.SetActive(false);
head.SetActive(false);
mouth.SetActive(false);
axes.SetActive(false);
}
//Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbMat, texture);
}
private void Run()
{
Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
// Convert Unity Texture2D to OpenCV Mat.
OpenCVForUnity.Utils.texture2DToMat(imgTexture, imgMat);
Debug.Log("imgMat dst ToString " + imgMat.ToString());
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = imgMat.width();
float height = imgMat.height();
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
FaceLandmarkDetector faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, imgMat);
//detect face rectdetecton
List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection();
foreach (var result in detectResult)
{
Debug.Log("rect : " + result.rect);
Debug.Log("detection_confidence : " + result.detection_confidence);
Debug.Log("weight_index : " + result.weight_index);
// Debug.Log ("face : " + rect);
Imgproc.putText(imgMat, "" + result.detection_confidence, new Point(result.rect.xMin, result.rect.yMin - 20), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
Imgproc.putText(imgMat, "" + result.weight_index, new Point(result.rect.xMin, result.rect.yMin - 5), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(result.rect);
Debug.Log("face points count : " + points.Count);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(imgMat, result.rect, new Scalar(255, 0, 0, 255), 2);
}
faceLandmarkDetector.Dispose();
Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false);
// Convert OpenCV Mat to Unity Texture2D.
OpenCVForUnity.Utils.matToTexture2D(imgMat, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
if (detectResult.Count > 0)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(detectResult [0]);
if (displayFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
MatOfPoint3f objectPoints = null;
if (points.Count == 68)
{
objectPoints = objectPoints68;
imagePoints.fromArray(
new Point((points [38].x + points [41].x) / 2, (points [38].y + points [41].y) / 2), //l eye (Interpupillary breadth)
new Point((points [43].x + points [46].x) / 2, (points [43].y + points [46].y) / 2), //r eye (Interpupillary breadth)
new Point(points [30].x, points [30].y), //nose (Tip)
new Point(points [33].x, points [33].y), //nose (Subnasale)
new Point(points [0].x, points [0].y), //l ear (Bitragion breadth)
new Point(points [16].x, points [16].y) //r ear (Bitragion breadth)
);
float noseDistance = Mathf.Abs((float)(points [27].y - points [33].y));
float mouseDistance = Mathf.Abs((float)(points [62].y - points [66].y));
}
else if (points.Count == 17)
{
objectPoints = objectPoints17;
imagePoints.fromArray(
new Point((points [2].x + points [3].x) / 2, (points [2].y + points [3].y) / 2), //l eye (Interpupillary breadth)
new Point((points [4].x + points [5].x) / 2, (points [4].y + points [5].y) / 2), //r eye (Interpupillary breadth)
new Point(points [0].x, points [0].y), //nose (Tip)
new Point(points [1].x, points [1].y), //nose (Subnasale)
new Point(points [6].x, points [6].y), //l ear (Bitragion breadth)
new Point(points [8].x, points [8].y) //r ear (Bitragion breadth)
);
float noseDistance = Mathf.Abs((float)(points [3].y - points [1].y));
float mouseDistance = Mathf.Abs((float)(points [14].y - points [16].y));
}
else if (points.Count == 6)
{
objectPoints = objectPoints6;
imagePoints.fromArray(
new Point((points [2].x + points [3].x) / 2, (points [2].y + points [3].y) / 2), //l eye (Interpupillary breadth)
new Point((points [4].x + points [5].x) / 2, (points [4].y + points [5].y) / 2), //r eye (Interpupillary breadth)
new Point(points [0].x, points [0].y), //nose (Tip)
new Point(points [1].x, points [1].y) //nose (Subnasale)
);
}
else if (points.Count == 5)
{
objectPoints = objectPoints5;
imagePoints.fromArray(
new Point(points [3].x, points [3].y), //l eye (Inner corner of the eye)
new Point(points [1].x, points [1].y), //r eye (Inner corner of the eye)
new Point(points [2].x, points [2].y), //l eye (Tail of the eye)
new Point(points [0].x, points [0].y), //r eye (Tail of the eye)
new Point(points [4].x, points [4].y) //nose (Subnasale)
);
if (fpsMonitor != null)
{
fpsMonitor.consoleText = "This example supports mainly the face landmark points of 68/17/6 points.";
}
}
// estimate head pose
if (rvec == null || tvec == null)
{
rvec = new Mat(3, 1, CvType.CV_64FC1);
tvec = new Mat(3, 1, CvType.CV_64FC1);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
double tvec_x = tvec.get(0, 0) [0], tvec_y = tvec.get(1, 0) [0], tvec_z = tvec.get(2, 0) [0];
bool isNotInViewport = false;
Vector4 pos = VP * new Vector4((float)tvec_x, (float)tvec_y, (float)tvec_z, 1.0f);
if (pos.w != 0)
{
float x = pos.x / pos.w, y = pos.y / pos.w, z = pos.z / pos.w;
if (x < -1.0f || x > 1.0f || y < -1.0f || y > 1.0f || z < -1.0f || z > 1.0f)
{
isNotInViewport = true;
}
}
if (double.IsNaN(tvec_z) || isNotInViewport) // if tvec is wrong data, do not use extrinsic guesses. (the estimated object is not in the camera field of view)
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
}
else
{
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
}
if (!isNotInViewport)
{
if (displayHead)
{
head.SetActive(true);
}
if (displayAxes)
{
axes.SetActive(true);
}
// Convert to unity pose data.
double[] rvecArr = new double[3];
rvec.get(0, 0, rvecArr);
double[] tvecArr = new double[3];
tvec.get(0, 0, tvecArr);
PoseData poseData = ARUtils.ConvertRvecTvecToPoseData(rvecArr, tvecArr);
// Changes in pos/rot below these thresholds are ignored.
if (enableLowPassFilter)
{
ARUtils.LowpassPoseData(ref oldPoseData, ref poseData, positionLowPass, rotationLowPass);
}
oldPoseData = poseData;
// Create transform matrix.
transformationM = Matrix4x4.TRS(poseData.pos, poseData.rot, Vector3.one);
//move joystick
float t = 0.50f;
int xpos;
if ((poseData.rot.eulerAngles.x - oldPoseData.rot.eulerAngles.x) < 10 && (poseData.rot.eulerAngles.x - oldPoseData.rot.eulerAngles.x) > 0)
{
xpos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.x, oldPoseData.rot.eulerAngles.x, t));
}
else
{
xpos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.x, oldPoseData.rot.eulerAngles.x, 1));
}
int ypos;
if ((poseData.rot.eulerAngles.y - oldPoseData.rot.eulerAngles.y) < 10 && (poseData.rot.eulerAngles.y - oldPoseData.rot.eulerAngles.y) > 0)
{
ypos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.y, oldPoseData.rot.eulerAngles.y, t));
}
else
{
ypos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.y, oldPoseData.rot.eulerAngles.y, 1));
}
int rpos;
if ((poseData.rot.eulerAngles.z - oldPoseData.rot.eulerAngles.z) < 10 && (poseData.rot.eulerAngles.z - oldPoseData.rot.eulerAngles.z) > 0)
{
rpos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.z, oldPoseData.rot.eulerAngles.z, t));
}
else
{
rpos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.z, oldPoseData.rot.eulerAngles.z, 1));
}
//int ypos = Convert.ToInt16(Mathf.Lerp(poseData.rot.eulerAngles.y, oldPoseData.rot.eulerAngles.y,t));
int zpos = Convert.ToInt16(poseData.pos.z);
//zoom
int scZ = Convert.ToInt32(Scale(zpos, 400, 800, 0, 32000));
//up down
int scX;
if (xpos >= 0 && xpos < 20)
{
scX = Convert.ToInt32(Scale(xpos, 0, 10, 29000, 32000));
}
else
{
scX = Convert.ToInt32(Scale(xpos, 340, 360, 0, 29000));
}
//left right
int scY = Convert.ToInt32(Scale(ypos, 160, 200, 0, 32000));
//roll
int scR = Convert.ToInt32(Scale(rpos, 160, 200, 0, 32000));
iReport.AxisX = scX;
iReport.AxisY = scY;
iReport.AxisZ = scZ;
iReport.Slider = scR;
bool upd = joystick.UpdateVJD(id, ref iReport);
string message = "x:" + scX + " y:" + scY + " z:" + scZ + " r:" + scR;
Debug.Log(message);
}
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARM = invertYM * transformationM;
// Apply Z-axis inverted matrix.
ARM = ARM * invertZM;
if (shouldMoveARCamera)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
}
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (capture == null)
{
return;
}
//Loop play
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
//error PlayerLoop called recursively! on iOS.reccomend WebCamTexture.
if (capture.grab())
{
capture.retrieve(rgbMat, 0);
Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB);
//Debug.Log ("Mat toString " + rgbMat.ToString ());
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
UnityEngine.Rect rect = new UnityEngine.Rect();
List <Vector2> points = null;
if (detectResult.Count > 0)
{
rect = detectResult [0];
//detect landmark points
points = faceLandmarkDetector.DetectLandmark(rect);
skippedFrames = 0;
}
else
{
skippedFrames++;
if (skippedFrames == maximumAllowedSkippedFrames)
{
if (drawLowPassFilter)
{
lowPassFilter.Reset();
}
if (drawKalmanFilter)
{
kalmanFilter.Reset();
}
if (drawOpticalFlowFilter)
{
opticalFlowFilter.Reset();
}
if (drawOFAndLPFilter)
{
opticalFlowFilter.Reset();
}
lowPassFilter.Reset();
}
}
if (drawLowPassFilter)
{
lowPassFilter.Process(rgbMat, points, lowPassFilteredPoints, isDebugMode);
}
if (drawKalmanFilter)
{
kalmanFilter.Process(rgbMat, points, kalmanFilteredPoints, isDebugMode);
}
if (drawOpticalFlowFilter)
{
opticalFlowFilter.Process(rgbMat, points, opticalFlowFilteredPoints, isDebugMode);
}
if (drawOFAndLPFilter)
{
opticalFlowFilter.Process(rgbMat, points, points, false);
lowPassFilter.Process(rgbMat, points, ofAndLPFilteredPoints, isDebugMode);
}
if (points != null && !isDebugMode)
{
// draw raw landmark points.
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, points, new Scalar(0, 255, 0), 2);
}
// draw face rect.
//OpenCVForUnityUtils.DrawFaceRect (rgbMat, rect, new Scalar (255, 0, 0), 2);
// draw filtered lam points.
if (points != null && !isDebugMode)
{
if (drawLowPassFilter)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, lowPassFilteredPoints, new Scalar(0, 255, 255), 2);
}
if (drawKalmanFilter)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, kalmanFilteredPoints, new Scalar(0, 0, 255), 2);
}
if (drawOpticalFlowFilter)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, opticalFlowFilteredPoints, new Scalar(255, 0, 0), 2);
}
if (drawOFAndLPFilter)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, ofAndLPFilteredPoints, new Scalar(255, 0, 255), 2);
}
}
//Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbMat, texture);
}
}
