// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//convert image to greyscale
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
if (isAutoResetMode || faceTracker.getPoints().Count <= 0)
{
// Debug.Log ("detectFace");
//convert image to greyscale
using (Mat equalizeHistMat = new Mat()) using (MatOfRect faces = new MatOfRect()) {
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0
| Objdetect.CASCADE_FIND_BIGGEST_OBJECT
| Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
if (faces.rows() > 0)
{
// Debug.Log ("faces " + faces.dump ());
List <OpenCVForUnity.CoreModule.Rect> rectsList = faces.toList();
List <Point[]> pointsList = faceTracker.getPoints();
if (isAutoResetMode)
{
//add initial face points from MatOfRect
if (pointsList.Count <= 0)
{
faceTracker.addPoints(faces);
// Debug.Log ("reset faces ");
}
else
{
for (int i = 0; i < rectsList.Count; i++)
{
OpenCVForUnity.CoreModule.Rect trackRect = new OpenCVForUnity.CoreModule.Rect(rectsList [i].x + rectsList [i].width / 3, rectsList [i].y + rectsList [i].height / 2, rectsList [i].width / 3, rectsList [i].height / 3);
//It determines whether nose point has been included in trackRect.
if (i < pointsList.Count && !trackRect.contains(pointsList [i] [67]))
{
rectsList.RemoveAt(i);
pointsList.RemoveAt(i);
// Debug.Log ("remove " + i);
}
Imgproc.rectangle(rgbaMat, new Point(trackRect.x, trackRect.y), new Point(trackRect.x + trackRect.width, trackRect.y + trackRect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
else
{
faceTracker.addPoints(faces);
}
//draw face rect
for (int i = 0; i < rectsList.Count; i++)
{
Imgproc.rectangle(rgbaMat, new Point(rectsList [i].x, rectsList [i].y), new Point(rectsList [i].x + rectsList [i].width, rectsList [i].y + rectsList [i].height), new Scalar(255, 0, 0, 255), 2);
}
}
else
{
if (isAutoResetMode)
{
faceTracker.reset();
rightEye.SetActive(false);
leftEye.SetActive(false);
head.SetActive(false);
mouth.SetActive(false);
axes.SetActive(false);
}
}
}
}
//track face points.if face points <= 0, always return false.
if (faceTracker.track(grayMat, faceTrackerParams))
{
if (isShowingFacePoints)
{
faceTracker.draw(rgbaMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255));
}
Imgproc.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Imgproc.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Point[] points = faceTracker.getPoints() [0];
if (points.Length > 0)
{
// for (int i = 0; i < points.Length; i++)
// {
// Imgproc.putText(rgbaMat, "" + i, new Point(points [i].x, points [i].y), Imgproc.FONT_HERSHEY_SIMPLEX, 0.3, new Scalar(0, 0, 255, 255), 2, Imgproc.LINE_AA, false);
// }
imagePoints.fromArray(
points [31], //l eye
points [36], //r eye
points [67], //nose
points [48], //l mouth
points [54] //r mouth
// ,
// points [0],//l ear
// points [14]//r ear
);
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
bool isRefresh = false;
if (tvec.get(2, 0) [0] > 0 && tvec.get(2, 0) [0] < 1200 * ((float)rgbaMat.cols() / (float)webCamTextureToMatHelper.requestedWidth))
{
isRefresh = true;
if (oldRvec == null)
{
oldRvec = new Mat();
rvec.copyTo(oldRvec);
}
if (oldTvec == null)
{
oldTvec = new Mat();
tvec.copyTo(oldTvec);
}
//filter Rvec Noise.
using (Mat absDiffRvec = new Mat()) {
Core.absdiff(rvec, oldRvec, absDiffRvec);
// Debug.Log ("absDiffRvec " + absDiffRvec.dump());
using (Mat cmpRvec = new Mat()) {
Core.compare(absDiffRvec, new Scalar(rvecNoiseFilterRange), cmpRvec, Core.CMP_GT);
if (Core.countNonZero(cmpRvec) > 0)
{
isRefresh = false;
}
}
}
//filter Tvec Noise.
using (Mat absDiffTvec = new Mat()) {
Core.absdiff(tvec, oldTvec, absDiffTvec);
// Debug.Log ("absDiffRvec " + absDiffRvec.dump());
using (Mat cmpTvec = new Mat()) {
Core.compare(absDiffTvec, new Scalar(tvecNoiseFilterRange), cmpTvec, Core.CMP_GT);
if (Core.countNonZero(cmpTvec) > 0)
{
isRefresh = false;
}
}
}
}
if (isRefresh)
{
if (isShowingEffects)
{
rightEye.SetActive(true);
}
if (isShowingEffects)
{
leftEye.SetActive(true);
}
if (isShowingHead)
{
head.SetActive(true);
}
if (isShowingAxes)
{
axes.SetActive(true);
}
if ((Mathf.Abs((float)(points [48].x - points [56].x)) < Mathf.Abs((float)(points [31].x - points [36].x)) / 2.2 &&
Mathf.Abs((float)(points [51].y - points [57].y)) > Mathf.Abs((float)(points [31].x - points [36].x)) / 2.9) ||
Mathf.Abs((float)(points [51].y - points [57].y)) > Mathf.Abs((float)(points [31].x - points [36].x)) / 2.7)
{
if (isShowingEffects)
{
mouth.SetActive(true);
}
}
else
{
if (isShowingEffects)
{
mouth.SetActive(false);
}
}
rvec.copyTo(oldRvec);
tvec.copyTo(oldTvec);
Calib3d.Rodrigues(rvec, rotM);
transformationM.SetRow(0, new Vector4((float)rotM.get(0, 0) [0], (float)rotM.get(0, 1) [0], (float)rotM.get(0, 2) [0], (float)tvec.get(0, 0) [0]));
transformationM.SetRow(1, new Vector4((float)rotM.get(1, 0) [0], (float)rotM.get(1, 1) [0], (float)rotM.get(1, 2) [0], (float)tvec.get(1, 0) [0]));
transformationM.SetRow(2, new Vector4((float)rotM.get(2, 0) [0], (float)rotM.get(2, 1) [0], (float)rotM.get(2, 2) [0], (float)tvec.get(2, 0) [0]));
transformationM.SetRow(3, new Vector4(0, 0, 0, 1));
// right-handed coordinates system (OpenCV) to left-handed one (Unity)
ARM = invertYM * transformationM;
// Apply Z-axis inverted matrix.
ARM = ARM * invertZM;
if (shouldMoveARCamera)
{
if (ARGameObject != null)
{
ARM = ARGameObject.transform.localToWorldMatrix * ARM.inverse;
ARUtils.SetTransformFromMatrix(ARCamera.transform, ref ARM);
ARGameObject.SetActive(true);
}
}
else
{
ARM = ARCamera.transform.localToWorldMatrix * ARM;
if (ARGameObject != null)
{
ARUtils.SetTransformFromMatrix(ARGameObject.transform, ref ARM);
ARGameObject.SetActive(true);
}
}
}
}
}
// Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Utils.fastMatToTexture2D(rgbaMat, texture);
}
if (Input.GetKeyUp(KeyCode.Space) || Input.touchCount > 0)
{
faceTracker.reset();
if (oldRvec != null)
{
oldRvec.Dispose();
oldRvec = null;
}
if (oldTvec != null)
{
oldTvec.Dispose();
oldTvec = null;
}
rightEye.SetActive(false);
leftEye.SetActive(false);
head.SetActive(false);
mouth.SetActive(false);
axes.SetActive(false);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//convert image to greyscale
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
if (isAutoResetMode || faceTracker.getPoints().Count <= 0)
{
// Debug.Log ("detectFace");
//convert image to greyscale
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0
// | Objdetect.CASCADE_FIND_BIGGEST_OBJECT
| Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
if (faces.rows() > 0)
{
// Debug.Log ("faces " + faces.dump ());
List <OpenCVForUnity.Rect> rectsList = faces.toList();
List <Point[]> pointsList = faceTracker.getPoints();
if (isAutoResetMode)
{
//add initial face points from MatOfRect
if (pointsList.Count <= 0)
{
faceTracker.addPoints(faces);
// Debug.Log ("reset faces ");
}
else
{
for (int i = 0; i < rectsList.Count; i++)
{
OpenCVForUnity.Rect trackRect = new OpenCVForUnity.Rect(rectsList [i].x + rectsList [i].width / 3, rectsList [i].y + rectsList [i].height / 2, rectsList [i].width / 3, rectsList [i].height / 3);
//It determines whether nose point has been included in trackRect.
if (i < pointsList.Count && !trackRect.contains(pointsList [i] [67]))
{
rectsList.RemoveAt(i);
pointsList.RemoveAt(i);
// Debug.Log ("remove " + i);
}
Imgproc.rectangle(rgbaMat, new Point(trackRect.x, trackRect.y), new Point(trackRect.x + trackRect.width, trackRect.y + trackRect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
else
{
faceTracker.addPoints(faces);
}
//draw face rect
for (int i = 0; i < rectsList.Count; i++)
{
#if OPENCV_2
Core.rectangle(rgbaMat, new Point(rectsList [i].x, rectsList [i].y), new Point(rectsList [i].x + rectsLIst [i].width, rectsList [i].y + rectsList [i].height), new Scalar(255, 0, 0, 255), 2);
#else
Imgproc.rectangle(rgbaMat, new Point(rectsList [i].x, rectsList [i].y), new Point(rectsList [i].x + rectsList [i].width, rectsList [i].y + rectsList [i].height), new Scalar(255, 0, 0, 255), 2);
#endif
}
}
else
{
if (isAutoResetMode)
{
faceTracker.reset();
}
}
}
}
//track face points.if face points <= 0, always return false.
if (faceTracker.track(grayMat, faceTrackerParams))
{
faceTracker.draw(rgbaMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255));
}
#if OPENCV_2
Core.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Core.LINE_AA, false);
#else
Imgproc.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
#endif
// Core.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Core.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
if (Input.GetKeyUp(KeyCode.Space) || Input.touchCount > 0)
{
faceTracker.reset();
}
}
/// <summary>
/// Raises the webcam texture to mat helper error occurred event.
/// </summary>
/// <param name="errorCode">Error code.</param>
// public void OnWebCamTextureToMatHelperErrorOccurred(WebCamTextureToMatHelper.ErrorCode errorCode)
// {
// Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode);
// }
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//convert image to greyscale
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
if (isAutoResetMode || faceTracker.getPoints().Count <= 0)
{
// Debug.Log ("detectFace");
//convert image to greyscale
using (var equalizeHistMat = new Mat())
using (var faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat,
faces,
1.1f,
2,
0 | Objdetect.CASCADE_SCALE_IMAGE,
new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15,
equalizeHistMat.cols() * 0.15),
new Size()
);
if (faces.rows() > 0)
{
// Debug.Log ("faces " + faces.dump ());
List <OpenCVForUnity.Rect> rectsList = faces.toList();
List <Point[]> pointsList = faceTracker.getPoints();
if (isAutoResetMode)
{
//add initial face points from MatOfRect
if (pointsList.Count <= 0)
{
faceTracker.addPoints(faces);
// Debug.Log ("reset faces ");
}
else
{
for (int i = 0; i < rectsList.Count; i++)
{
var trackRect = new OpenCVForUnity.Rect(rectsList [i].x + rectsList [i].width / 3, rectsList [i].y + rectsList [i].height / 2, rectsList [i].width / 3, rectsList [i].height / 3);
//It determines whether nose point has been included in trackRect.
if (i < pointsList.Count && !trackRect.contains(pointsList [i] [67]))
{
rectsList.RemoveAt(i);
pointsList.RemoveAt(i);
// Debug.Log ("remove " + i);
}
Imgproc.rectangle(rgbaMat, new Point(trackRect.x, trackRect.y), new Point(trackRect.x + trackRect.width, trackRect.y + trackRect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
else
{
faceTracker.addPoints(faces);
}
// ターゲットメッシュのリストを更新
{
while (targetMeshList.Count < rectsList.Count)
{
var obj = Instantiate(targetMeshPrefab).GetComponent <MeshRenderer>();
obj.transform.rotation = Quaternion.Euler(-90, 0, 0);
targetMeshList.Add(obj);
}
for (int i = targetMeshList.Count - 1; i >= 0; --i)
{
if (i >= rectsList.Count)
{
targetMeshList[i].material.color = Color.clear;
}
else
{
targetMeshList[i].material.color = Color.red;
}
}
}
//draw face rect
for (int i = 0; i < rectsList.Count; i++)
{
#if OPENCV_2
Core.rectangle(rgbaMat, new Point(rectsList [i].x, rectsList [i].y), new Point(rectsList [i].x + rectsLIst [i].width, rectsList [i].y + rectsList [i].height), new Scalar(255, 0, 0, 255), 2);
#else
Imgproc.rectangle(rgbaMat, new Point(rectsList [i].x, rectsList [i].y), new Point(rectsList [i].x + rectsList [i].width, rectsList [i].y + rectsList [i].height), new Scalar(255, 0, 0, 255), 2);
#endif
// 顔の中心にオブジェクトを移動
{
var rect = rectsList[i];
// Debug.Log( string.Format("Rect位置( {0}, {1}) Rectサイズ( {2}, {3})", rect.x, rect.y, rect.width, rect.height) );
var pos = new Vector2(
rect.x - (rect.width / 2)
,
// rect.y + ( rect.height / 2 )
rect.y
);
// オブジェクトを移動する
targetMeshList[i].transform.localPosition = Vector2ToVector3(pos);
}
}
// 顔の中心位置にオブジェクトを移動
if (false)
{
for (int i = 0; i < pointsList.Count; ++i)
{
Vector2 pos;
// 中心位置を求める
{
double sumX = 0, sumY = 0;
for (int j = 0; j < pointsList[i].Length; ++j)
{
var point = pointsList[i][j];
sumX += point.x;
sumY += point.y;
}
var averageX = sumX / pointsList[i].Length;
var averageY = sumY / pointsList[i].Length;
pos = new Vector2((float)averageX, (float)averageY);
}
{
double leftEnd = 0, topEnd = 0;
}
// オブジェクトを移動する
targetMeshList[i].transform.localPosition = Vector2ToVector3(pos);
}
}
}
else
{
if (isAutoResetMode)
{
faceTracker.reset();
}
}
}
}
//track face points.if face points <= 0, always return false.
if (faceTracker.track(grayMat, faceTrackerParams))
{
faceTracker.draw(rgbaMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255));
}
#if OPENCV_2
Core.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Core.LINE_AA, false);
#else
Imgproc.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
#endif
// Core.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Core.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
if (Input.GetKeyUp(KeyCode.Space) || Input.touchCount > 0)
{
faceTracker.reset();
}
}
/// <summary>
/// Raises the webcam texture to mat helper error occurred event.
/// </summary>
/// <param name="errorCode">Error code.</param>
// public void OnWebCamTextureToMatHelperErrorOccurred(WebCamTextureToMatHelper.ErrorCode errorCode)
// {
// Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode);
// }
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//convert image to greyscale
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
if (isAutoResetMode || faceTracker.getPoints().Count <= 0)
{
// Debug.Log ("detectFace");
//convert image to greyscale
using (var equalizeHistMat = new Mat())
using (var faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat,
faces,
1.1f,
2,
0 | Objdetect.CASCADE_SCALE_IMAGE,
new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15,
equalizeHistMat.cols() * 0.15),
new Size()
);
if (faces.rows() > 0)
{
// Debug.Log ("faces " + faces.dump ());
List <OpenCVForUnity.Rect> rectsList = faces.toList();
List <Point[]> pointsList = faceTracker.getPoints();
if (isAutoResetMode)
{
//add initial face points from MatOfRect
if (pointsList.Count <= 0)
{
faceTracker.addPoints(faces);
// Debug.Log ("reset faces ");
}
else
{
for (int i = 0; i < rectsList.Count; i++)
{
var trackRect = new OpenCVForUnity.Rect(rectsList [i].x + rectsList [i].width / 3, rectsList [i].y + rectsList [i].height / 2, rectsList [i].width / 3, rectsList [i].height / 3);
//It determines whether nose point has been included in trackRect.
if (i < pointsList.Count && !trackRect.contains(pointsList [i] [67]))
{
rectsList.RemoveAt(i);
pointsList.RemoveAt(i);
// Debug.Log ("remove " + i);
}
Imgproc.rectangle(rgbaMat, new Point(trackRect.x, trackRect.y), new Point(trackRect.x + trackRect.width, trackRect.y + trackRect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
else
{
faceTracker.addPoints(faces);
}
// ターゲットメッシュのリストを更新
{
while (targetMeshList.Count < rectsList.Count)
{
var obj = Instantiate(targetMeshPrefab).GetComponent <MeshRenderer>();
obj.transform.rotation = Quaternion.Euler(-90, 0, 0);
targetMeshList.Add(obj);
}
for (int i = targetMeshList.Count - 1; i >= 0; --i)
{
if (i >= rectsList.Count)
{
targetMeshList[i].material.color = Color.clear;
}
else
{
targetMeshList[i].material.color = Color.red;
}
}
}
//draw face rect
for (int i = 0; i < rectsList.Count; i++)
{
#if OPENCV_2
Core.rectangle(rgbaMat, new Point(rectsList [i].x, rectsList [i].y), new Point(rectsList [i].x + rectsLIst [i].width, rectsList [i].y + rectsList [i].height), new Scalar(255, 0, 0, 255), 2);
#else
Imgproc.rectangle(rgbaMat, new Point(rectsList [i].x, rectsList [i].y), new Point(rectsList [i].x + rectsList [i].width, rectsList [i].y + rectsList [i].height), new Scalar(255, 0, 0, 255), 2);
#endif
if (i < targetMeshList.Count)
{
var rectPos = new Vector2(rectsList[i].x, rectsList[i].y);
// targetMeshList[i].transform.position = new Vector3( rectsList[i].x, rectsList[i].y, targetMeshPosZ );
var ray = RectTransformUtility.ScreenPointToRay(Camera.main, rectPos);
RaycastHit hit;
if (Physics.Raycast(ray, out hit))
{
targetMeshList[i].transform.position = hit.point + hit.normal * 0.5f;
}
targetImage.transform.localPosition = new Vector2(rectsList[i].x, rectsList[i].y);
}
}
}
else
{
if (isAutoResetMode)
{
faceTracker.reset();
}
}
}
}
//track face points.if face points <= 0, always return false.
if (faceTracker.track(grayMat, faceTrackerParams))
{
faceTracker.draw(rgbaMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255));
}
#if OPENCV_2
Core.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Core.LINE_AA, false);
#else
Imgproc.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
#endif
// Core.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Core.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
if (Input.GetKeyUp(KeyCode.Space) || Input.touchCount > 0)
{
faceTracker.reset();
}
}
// Update is called once per frame
void Update()
{
if (Input.GetMouseButtonDown(0))
{
Debug.Log(Input.mousePosition);
}
if (isCamera)
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//convert image to greyscale
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
if (isAutoResetMode || faceTracker.getPoints().Count <= 0)
{
// Debug.Log ("detectFace");
//convert image to greyscale
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0
// | Objdetect.CASCADE_FIND_BIGGEST_OBJECT
| Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
if (faces.rows() > 0)
{
// Debug.Log ("faces " + faces.dump ());
List <OpenCVForUnity.Rect> rectsList = faces.toList();
List <Point[]> pointsList = faceTracker.getPoints();
if (isAutoResetMode)
{
//add initial face points from MatOfRect
if (pointsList.Count <= 0)
{
faceTracker.addPoints(faces);
// Debug.Log ("reset faces ");
}
else
{
// 얼굴 부분을 찍어야할 때
if (isShot)
{
for (int i = 0; i < rectsList.Count; i++)
{
OpenCVForUnity.Rect trackRect = new OpenCVForUnity.Rect(rectsList[i].x + rectsList[i].width / 3, rectsList[i].y + rectsList[i].height / 2, rectsList[i].width / 3, rectsList[i].height / 3);
//It determines whether nose point has been included in trackRect.
if (i < pointsList.Count && !trackRect.contains(pointsList[i][67]))
{
rectsList.RemoveAt(i);
pointsList.RemoveAt(i);
// Debug.Log ("remove " + i);
}
double left = 0;
double right = 0;
double up = 0;
double down = 0;
//track face points.if face points <= 0, always return false.
if (faceTracker.track(rgbaMat, faceTrackerParams))
{
// 얼굴 좌푤르 저장할 Point List
pointsList = faceTracker.getPoints();
// 상하좌우 기준점을 찾음
left = findMin(pointsList[0][0].x, pointsList[0][1].x, pointsList[0][2].x, pointsList[0][3].x);
right = findMax(pointsList[0][14].x, pointsList[0][13].x, pointsList[0][12].x, pointsList[0][11].x);
up = findMin(pointsList[0][16].y, pointsList[0][17].y, pointsList[0][22].y, pointsList[0][23].y);
down = findMax(pointsList[0][6].y, pointsList[0][7].y, pointsList[0][8].y, pointsList[0][9].y);
// 얼굴 좌표 출력
//for (int i = 0; i < pointsList[0].Length; i++)
//{
// Imgproc.putText(imgMat, i + "", pointsList[0][i], Core.FONT_HERSHEY_SIMPLEX, 0.2, new Scalar(255, 2, 10, 255), 1, Imgproc.LINE_AA, false);
//}
//faceTracker.draw(imgMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255));
}
// 가로폭, 세로폭을 정함
double widthLR = (right - left) / 10;
double heightU = (down - up) / 3;
double heightD = (down - up) / 18;
// 스크린 상의 빨간색 네모 왼쪽 위와 오른쪽 아래의 좌표 찾음
Vector2 screenPos1 = Camera.main.WorldToScreenPoint(new Vector2((float)(left - widthLR), (float)(up - heightU)));
Vector2 screenPos2 = Camera.main.WorldToScreenPoint(new Vector2((float)(right + widthLR), (float)(down + heightD)));
// 스크린 상의 그림의 왼쪽 위와 오른쪽 위 좌표
Vector2 leftUp = Camera.main.WorldToScreenPoint(new Vector2(0f, 0f));
Vector2 rightUp = Camera.main.WorldToScreenPoint(new Vector2(texture.width / 2, texture.height / 2));
// 가로, 세로 여백의 크기 추출
float widthSpace = Screen.width - rightUp.x;
float heightSpace = Screen.height - rightUp.y;
// 중앙의 스크린 좌표
Vector2 mid = Camera.main.WorldToScreenPoint(new Vector2(0f, 0f));
// 그림과 빨간색 상자 사이의 여백
float leftSpace = screenPos1.x - leftUp.x;
float upSpace = screenPos1.y - leftUp.y;
// 빨간색 상자의 가로, 세로를 정함
float widthImg = screenPos2.x - screenPos1.x;
float heightImg = screenPos2.y - screenPos1.y;
//Debug.Log("width: " + widthImg + ", height: " + heightImg);
// 저장을 할 땐 사각형 표시 X
if (!isTake)
{
// 기준점과 폭을 토대로 사각형을 그림
Imgproc.rectangle(rgbaMat, new Point(left - widthLR, up - heightU), new Point(right + widthLR, down + heightD), new Scalar(255, 0, 0, 255), 2);
}
if (isTake)
{
// image를 정사각형으로 저장
// 가로가 크냐 세로가 크냐에 따라 따로 저장
if (widthImg > heightImg)
{
int picX = (int)(widthSpace + leftSpace);
int picY = (int)(Screen.height - upSpace - heightImg - (widthImg - heightImg) / 2 - heightSpace);
// 얼굴 크기가 화면 크기 1/4 보다 커야하고 얼굴이 중앙
if (widthImg > Screen.width / 4 && picX < mid.x && picX + widthImg > mid.x && picY < mid.y && picY + heightImg > mid.y)
{
StartCoroutine(downImage((int)widthImg, picX, picY));
}
else
{
StartCoroutine(reShot());
}
}
else
{
int picX = (int)(widthSpace + leftSpace - (heightImg - widthImg) / 2);
int picY = (int)(Screen.height - upSpace - heightImg - heightSpace);
//Debug.Log("picX : " + picX);
//Debug.Log("picY : " + picY);
//Debug.Log("wid : " + widthImg);
//Debug.Log("hei : " + heightImg);
//Debug.Log("midX : " + mid.x);
//Debug.Log("midY : " + mid.y);
if (heightImg > Screen.width / 4 && picX < mid.x && picX + widthImg > mid.x && picY < mid.y && picY + heightImg > mid.y)
{
StartCoroutine(downImage((int)heightImg, picX, picY));
}
else
{
StartCoroutine(reShot());
}
}
// 안쪽 파란 사각형으로 얼굴 표현
//Imgproc.rectangle(rgbaMat, new Point(trackRect.x, trackRect.y), new Point(trackRect.x + trackRect.width, trackRect.y + trackRect.height), new Scalar(0, 0, 255, 255), 2);
}
}
}
}
}
else
{
faceTracker.addPoints(faces);
}
}
else
{
if (isTake)
{
StartCoroutine(reShot());
}
if (isAutoResetMode)
{
faceTracker.reset();
}
}
}
}
// 얼굴 부분을 점으로 표현
//track face points.if face points <= 0, always return false.
//if (faceTracker.track(grayMat, faceTrackerParams))
// faceTracker.draw(rgbaMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255));
#if OPENCV_2
Core.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Core.LINE_AA, false);
#else
//Imgproc.putText(rgbaMat, "'Tap' or 'Space Key' to Reset", new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
#endif
// Core.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Core.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
//convert image to greyscale
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
if (isAutoResetMode || faceTracker.getPoints().Count <= 0)
{
// Debug.Log ("detectFace");
//convert image to greyscale
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0
// | Objdetect.CASCADE_FIND_BIGGEST_OBJECT
| Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
if (faces.rows() > 0)
{
// Debug.Log ("faces " + faces.dump ());
List <OpenCVForUnity.Rect> rectsList = faces.toList();
List <Point[]> pointsList = faceTracker.getPoints();
if (isAutoResetMode)
{
//add initial face points from MatOfRect
if (pointsList.Count <= 0)
{
faceTracker.addPoints(faces);
// Debug.Log ("reset faces ");
}
else
{
// Disable nose rect display
//for (int i = 0; i < rectsList.Count; i++)
//{
// OpenCVForUnity.Rect trackRect = new OpenCVForUnity.Rect(rectsList [i].x + rectsList [i].width / 3, rectsList [i].y + rectsList [i].height / 2, rectsList [i].width / 3, rectsList [i].height / 3);
// //It determines whether nose point has been included in trackRect.
// if (i < pointsList.Count && !trackRect.contains(pointsList [i] [67]))
// {
// rectsList.RemoveAt(i);
// pointsList.RemoveAt(i);
// // Debug.Log ("remove " + i);
// }
// Imgproc.rectangle(rgbaMat, new Point(trackRect.x, trackRect.y), new Point(trackRect.x + trackRect.width, trackRect.y + trackRect.height), new Scalar(0, 0, 255, 255), 2);
//}
}
}
else
{
faceTracker.addPoints(faces);
}
//draw face rect with largest rect
int l = 0;
for (int i = 0; i < rectsList.Count; i++)
{
if (rectsList [i].area() > rectsList [l].area())
{
l = i;
}
#if OPENCV_2
Core.rectangle(rgbaMat, new Point(rectsList [l].x, rectsList [l].y), new Point(rectsList [l].x + rectsList [l].width, rectsList [l].y + rectsList [l].height), new Scalar(255, 0, 0, 255), 2);
#else
Imgproc.rectangle(rgbaMat, new Point(rectsList [l].x, rectsList [l].y), new Point(rectsList [l].x + rectsList [l].width, rectsList [l].y + rectsList [l].height), new Scalar(255, 0, 0, 255), 2);
#endif
Imgproc.line(rgbaMat,
new Point(rectsList[l].x + rectsList[l].width / 2, rectsList[l].y),
new Point(rectsList[l].x + rectsList[l].width / 2, rectsList[l].y - rectsList[l].height / 6),
new Scalar(255, 0, 0, 255), 2);
Imgproc.line(rgbaMat,
new Point(rectsList[l].x + rectsList[l].width / 2, rectsList[l].y + rectsList[l].height),
new Point(rectsList[l].x + rectsList[l].width / 2, rectsList[l].y + rectsList[l].height * 7 / 6),
new Scalar(255, 0, 0, 255), 2);
Imgproc.line(rgbaMat,
new Point(rectsList[l].x - rectsList[l].width / 6, rectsList[l].y + rectsList[l].height / 2),
new Point(rectsList[l].x, rectsList[l].y + rectsList[l].height / 2),
new Scalar(255, 0, 0, 255), 2);
Imgproc.line(rgbaMat,
new Point(rectsList[l].x + rectsList[l].width * 7 / 6, rectsList[l].y + rectsList[l].height / 2),
new Point(rectsList[l].x + rectsList[l].width, rectsList[l].y + rectsList[l].height / 2),
new Scalar(255, 0, 0, 255), 2);
}
//grayscale or rgba
//Mat croppedImage = new Mat(grayMat, rectsList[l]);
int x = rectsList[l].x;
int y = rectsList[l].y;
int w = rectsList[l].width;
int h = rectsList[l].height;
Color[] c = texture.GetPixels(x, y, w, h);
m2Texture = new Texture2D(w, h);
m2Texture.SetPixels(c);
m2Texture.Apply();
byte[] imageBytes = m2Texture.EncodeToJPG();
Destroy(m2Texture);
//Debug
//File.WriteAllBytes(Application.dataPath + "/image.jpg", imageBytes);
//StartCoroutine(PostRequest("http://localhost:5000/face_detection", imageBytes));
if (isRequestCompleted)
{
StartCoroutine(PostRequest("http://" + apiserverip + ":5000/face_detection", imageBytes));
}
// Display rect on texture
}
else
{
if (isAutoResetMode)
{
faceTracker.reset();
}
}
}
}
//track face points.if face points <= 0, always return false.
//if (faceTracker.track(grayMat, faceTrackerParams))
// faceTracker.draw(rgbaMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255));
//#if OPENCV_2
//Core.putText (rgbaMat, "Contribution " + contribution + " repos " + publicRepos + " followers " + followers + " gists " + publicGists, new Point (5, rgbaMat.rows () - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar (255, 255, 255, 255), 2, Core.LINE_AA, false);
//#else
//Imgproc.putText(rgbaMat, "Contribution " + contribution + " repos " + publicRepos + " followers " + followers + " gists " + publicGists, new Point(5, rgbaMat.rows() - 5), Core.FONT_HERSHEY_SIMPLEX, 0.8, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//#endif
//Core.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Core.LINE_AA, false);
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
if (Input.GetKeyUp(KeyCode.Space) || Input.touchCount > 0)
{
faceTracker.reset();
}
}
