/// <summary> /// Raises the destroy event. /// </summary> void OnDestroy() { if (webCamTextureToMatHelper != null) { webCamTextureToMatHelper.Dispose(); } if (cascade != null) { cascade.Dispose(); } if (rectangleTracker != null) { rectangleTracker.Dispose(); } if (faceLandmarkDetector != null) { faceLandmarkDetector.Dispose(); } foreach (var key in lowPassFilterDict.Keys) { lowPassFilterDict[key].Dispose(); } lowPassFilterDict.Clear(); foreach (var key in opticalFlowFilterDict.Keys) { opticalFlowFilterDict[key].Dispose(); } opticalFlowFilterDict.Clear(); if (faceChanger != null) { faceChanger.Dispose(); } if (faceMaskMat != null) { faceMaskMat.Dispose(); } #if UNITY_WEBGL if (getFilePath_Coroutine != null) { StopCoroutine(getFilePath_Coroutine); ((IDisposable)getFilePath_Coroutine).Dispose(); } #endif }
/// <summary> /// Raises the destroy event. /// </summary> void OnDestroy() { webCamTextureToMatHelper.Dispose(); if (cascade != null) { cascade.Dispose(); } if (rectangleTracker != null) { rectangleTracker.Dispose(); } if (faceLandmarkDetector != null) { faceLandmarkDetector.Dispose(); } if (faceChanger != null) { faceChanger.Dispose(); } if (faceMaskMat != null) { faceMaskMat.Dispose(); } #if UNITY_WEBGL && !UNITY_EDITOR foreach (var coroutine in coroutines) { StopCoroutine(coroutine); ((IDisposable)coroutine).Dispose(); } #endif }
void Run() { Mat rgbaMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4); OpenCVForUnity.Utils.texture2DToMat(imgTexture, rgbaMat); //Mat dstMat = Imgcodecs.imread(Application.dataPath + "/Resources/changer.jpg"); //Imgproc.cvtColor(dstMat, dstMat, Imgproc.COLOR_BGR2RGB); FrontalFaceChecker frontalFaceChecker = new FrontalFaceChecker(imgTexture.width, imgTexture.height); //检测正面脸,范围整个t2d //1、人脸检测。两种检测人脸rect方法,toggle切换 List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect>(); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (UnityEngine.Rect unityRect in result) { detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath); Mat gray = new Mat(); Imgproc.cvtColor(rgbaMat, gray, Imgproc.COLOR_RGBA2GRAY); MatOfRect faces = new MatOfRect(); Imgproc.equalizeHist(gray, gray); cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(gray.cols() * 0.05, gray.cols() * 0.05), new Size()); detectResult = faces.toList(); // adjust to Dilb's result. foreach (OpenCVForUnity.Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } gray.Dispose(); } //0-拷贝到->1,靠list中排序,改变顺序 detectResult.Sort(delegate(OpenCVForUnity.Rect x, OpenCVForUnity.Rect y) { return(x.x.CompareTo(y.x)); //顺序与delegate中一致,升序排列 //return y.x.CompareTo(x.x); //顺序与delegate中不同,降序排列 }); //Debug.Log("[0]" + detectResult[0].x + ", [1]" + detectResult[1].x); //2、关键点定位。通过上一步的rect,检测人脸标记点landmark OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); //TODO...只检测一张脸,另一张改成UV图,手动输入68个points /* * foreach (OpenCVForUnity.Rect openCVRect in detectResult) * { * UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); * //Debug.Log("face : " + rect); * * OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //画框rect * Imgproc.circle(rgbaMat, new Point(rect.x, rect.y), 0, new Scalar(0, 255, 0, 255), 5, Imgproc.LINE_8, 0); * Imgproc.putText(rgbaMat, rect.x + "x" + rect.y, new Point(rect.x + 5, rect.y -5), 1, 1, new Scalar(0, 255, 0, 255)); * * List<Vector2> points = faceLandmarkDetector.DetectLandmark(rect); * //OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); //画框landmark * landmarkPoints.Add(points); * } */ //检测一张照片的脸的landmark点 OpenCVForUnity.Rect openCVRect = detectResult[0]; UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); //OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2); //画框rect //Imgproc.circle(rgbaMat, new Point(rect.x, rect.y), 0, new Scalar(0, 255, 0, 255), 5, Imgproc.LINE_8, 0); //Imgproc.putText(rgbaMat, rect.x + "x" + rect.y, new Point(rect.x + 5, rect.y - 5), 1, 1, new Scalar(0, 255, 0, 255)); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); //通过检测器从rect中提取point //OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2); //画框landmark landmarkPoints.Add(points); string log = ""; for (int i = 0; i < points.Count; i++) { //Debug.Log("[" + i + "] " + points[i]); log += "new Vector2(" + ((int)points[i].x + 500) + "," + (int)points[i].y + "),\n"; //Imgproc.circle(rgbaMat, new Point(10, 10), 0, new Scalar(255, 0, 0, 255), 4, Imgproc.LINE_8, 0); //检查原点 //Imgproc.putText(rgbaMat, i.ToString(), new Point(landmarkPoints[0][i].x, landmarkPoints[0][i].y), 1, 1, new Scalar(0, 255, 0, 255)); //Imgproc.circle(rgbaMat, new Point(landmarkPoints[0][i].x, landmarkPoints[0][i].y), 0, new Scalar(0, 255, 0, 255), 2, Imgproc.LINE_8, 0); //绘制68点 //Imgproc.putText(rgbaMat, rect.x + "x" + rect.y, new Point(rect.x + 5, rect.y - 5), 1, 1, new Scalar(0, 255, 0, 255)); //Imgproc.putText(rgbaMat, rect.x + "x" + rect.y, new Point(rect.x + 5, rect.y - 5), 1, 1, new Scalar(0, 255, 0, 255)); /* * * * * * * * * * * * * * jaw; // [0-16] * * rightBrow; // [17-21] * * leftBrow; // [22-26] * * nose; // [27-35] * * rightEye; // [36-41] * * leftEye; // [42-47] * * mouth; // [48-59] * * mouth2; // [60-67] * * * * * * * * * * * * * */ /* * if (i >= 0 && i <= 16) * { * Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 0, new Scalar(255, 0, 0, 255), 2, Imgproc.LINE_8, 0); //红 * } * else if (i >= 17 && i <= 21) * { * Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 0, new Scalar(255, 127, 0, 255), 2, Imgproc.LINE_8, 0); //橙 * } * else if (i >= 22 && i <= 26) * { * Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 0, new Scalar(255, 255, 0, 255), 2, Imgproc.LINE_8, 0); //黄 * } * else if (i >= 27 && i <= 35) * { * Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 0, new Scalar(0, 255, 0, 255), 2, Imgproc.LINE_8, 0); //绿 * } * else if (i >= 36 && i <= 21) * { * Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 0, new Scalar(0, 255, 170, 255), 2, Imgproc.LINE_8, 0); //青 * } * else if (i >= 42 && i <= 47) * { * Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 0, new Scalar(0, 0, 255, 255), 2, Imgproc.LINE_8, 0); //蓝 * } * else if (i >= 48 && i <= 59) * { * Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 0, new Scalar(128, 255, 0, 255), 2, Imgproc.LINE_8, 0); //紫 * } * else if (i >= 60 && i <= 67) * { * Imgproc.circle(rgbaMat, new Point(points[i].x, points[i].y), 0, new Scalar(255, 180, 255, 255), 2, Imgproc.LINE_8, 0); //粉红 * } */ } //Debug.Log(log); //手写一张UV的landmark点 List <Vector2> uvs = new List <Vector2>() { //0-7 new Vector2(583, 210), //0 new Vector2(584, 250), //1 new Vector2(585, 305), //2 new Vector2(600, 360), //3 new Vector2(630, 410), //4 new Vector2(670, 425), //5 new Vector2(710, 440), //6 new Vector2(740, 445), //7 //8 new Vector2(770, 450), //8+轴 //9-16 new Vector2(800, 445), //9 new Vector2(830, 440), //10 new Vector2(870, 425), //11 new Vector2(910, 410), //12 new Vector2(940, 360), //13 new Vector2(955, 305), //14 new Vector2(956, 250), //15 new Vector2(957, 210), //16 //17-21 leftBrow new Vector2(655, 165), //17 new Vector2(680, 155), //18 new Vector2(710, 160), //19 new Vector2(730, 170), //20 new Vector2(750, 190), //21 //22-26 rightBrow new Vector2(790, 190), //22 new Vector2(810, 170), //23 new Vector2(830, 160), //24 new Vector2(860, 155), //25 new Vector2(885, 165), //26 //27-30 nose竖 new Vector2(770, 220), //27 new Vector2(770, 250), //28 new Vector2(770, 275), //29 new Vector2(770, 300), //30 //31-35 nose横 new Vector2(740, 312), //31 new Vector2(755, 316), //32 new Vector2(770, 320), //33 new Vector2(785, 316), //34 new Vector2(800, 312), //35 //36-41 leftEye new Vector2(670, 215), //36 new Vector2(690, 200), //37 new Vector2(715, 205), //38 new Vector2(730, 225), //39 new Vector2(710, 230), //40 new Vector2(690, 227), //41 //42-47 rightEye new Vector2(810, 225), //42 new Vector2(825, 205), //43 new Vector2(850, 200), //44 new Vector2(870, 215), //45 new Vector2(855, 227), //46 new Vector2(830, 230), //47 //48-59 mouth new Vector2(720, 360), //48-l new Vector2(735, 355), //49 new Vector2(750, 350), //50 new Vector2(770, 352), //51+u new Vector2(790, 350), //52 new Vector2(805, 355), //53 new Vector2(820, 360), //54-r new Vector2(805, 375), //55 new Vector2(790, 382), //56 new Vector2(770, 380), //57+d new Vector2(750, 382), //58 new Vector2(735, 375), //59 //60-67 mouth2 new Vector2(730, 365), //60-l new Vector2(750, 357), //61 new Vector2(770, 354), //62+u new Vector2(790, 357), //63 new Vector2(810, 365), //64-r new Vector2(790, 370), //65 new Vector2(770, 375), //66+d new Vector2(750, 370), //67 }; landmarkPoints.Add(uvs); for (int i = 0; i < landmarkPoints[1].Count; i++) { //Imgproc.putText(rgbaMat, i.ToString(), new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 1, 1, new Scalar(255, 0, 0, 255)); /* * * * * * * * * * * * * * jaw; // [0-16] * * rightBrow; // [17-21] * * leftBrow; // [22-26] * * nose; // [27-35] * * rightEye; // [36-41] * * leftEye; // [42-47] * * mouth; // [48-59] * * mouth2; // [60-67] * * * * * * * * * * * * * */ /* * if (i >= 0 && i <= 16) * { * Imgproc.circle(rgbaMat, new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 0, new Scalar(255, 0, 0, 255), 5, Imgproc.LINE_8, 0); //红 * } * else if (i >= 17 && i <= 21) * { * Imgproc.circle(rgbaMat, new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 0, new Scalar(255, 0, 0, 255), 5, Imgproc.LINE_8, 0); //橙 * } * else if (i >= 22 && i <= 26) * { * Imgproc.circle(rgbaMat, new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 0, new Scalar(255, 0, 0, 255), 5, Imgproc.LINE_8, 0); //黄 * } * else if (i >= 27 && i <= 35) * { * Imgproc.circle(rgbaMat, new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 0, new Scalar(255, 0, 0, 255), 5, Imgproc.LINE_8, 0); //绿 * } * else if (i >= 36 && i <= 21) * { * Imgproc.circle(rgbaMat, new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 0, new Scalar(255, 0, 0, 255), 5, Imgproc.LINE_8, 0); //青 * } * else if (i >= 42 && i <= 47) * { * Imgproc.circle(rgbaMat, new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 0, new Scalar(255, 0, 0, 255), 5, Imgproc.LINE_8, 0); //蓝 * } * else if (i >= 48 && i <= 59) * { * Imgproc.circle(rgbaMat, new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 0, new Scalar(255, 0, 0, 255), 5, Imgproc.LINE_8, 0); //紫 * } * else if (i >= 60 && i <= 67) * { * //Imgproc.circle(rgbaMat, new Point(landmarkPoints[1][i].x, landmarkPoints[1][i].y), 0, new Scalar(255, 0, 0, 255), 5, Imgproc.LINE_8, 0); //粉红 * } */ } //过滤非正面的脸 if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit) //阈值0~1 { detectResult.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } //1张脸时只提取关键点,2张脸才换脸 if (landmarkPoints.Count == 2) { //开始换脸 DlibFaceChanger faceChanger = new DlibFaceChanger(); faceChanger.isShowingDebugFacePoints = displayDebugFacePoints; faceChanger.SetTargetImage(rgbaMat); //目标Mat faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints[0], landmarkPoints[1], 0.9f); //源Mat,从0拷贝到1 faceChanger.ChangeFace(); faceChanger.Dispose(); } frontalFaceChecker.Dispose(); //背景图 Texture2D t2d = new Texture2D(rgbaMat.width(), rgbaMat.height(), TextureFormat.RGBA32, false); OpenCVForUnity.Utils.matToTexture2D(rgbaMat, t2d); Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero); srcImage.sprite = sp; srcImage.preserveAspect = true; Mat dstMat = new Mat(rgbaMat, new OpenCVForUnity.Rect(rgbaMat.width() / 2, 0, rgbaMat.width() / 2, rgbaMat.height())); Texture2D dst_t2d = new Texture2D(dstMat.width(), dstMat.height(), TextureFormat.RGBA32, false); OpenCVForUnity.Utils.matToTexture2D(dstMat, dst_t2d); mesh.materials[1].mainTexture = dst_t2d; rgbaMat.Dispose(); }
private void Run() { displayFaceRectsToggle.isOn = displayFaceRects; useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter; filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces; displayDebugFacePointsToggle.isOn = displayDebugFacePoints; if (imgTexture == null) { imgTexture = Resources.Load("family") as Texture2D; } gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1); Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); float width = 0; float height = 0; width = gameObject.transform.localScale.x; height = gameObject.transform.localScale.y; float widthScale = (float)Screen.width / width; float heightScale = (float)Screen.height / height; if (widthScale < heightScale) { Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2; } else { Camera.main.orthographicSize = height / 2; } Mat rgbaMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4); OpenCVForUnity.UnityUtils.Utils.texture2DToMat(imgTexture, rgbaMat); Debug.Log("rgbaMat ToString " + rgbaMat.ToString()); if (faceLandmarkDetector == null) { faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath); } FrontalFaceChecker frontalFaceChecker = new FrontalFaceChecker(width, height); // detect faces. List <Rect> detectResult = new List <Rect>(); if (useDlibFaceDetecter) { OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <UnityEngine.Rect> result = faceLandmarkDetector.Detect(); foreach (var unityRect in result) { detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height)); } } else { if (cascade == null) { cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath); } //if (cascade.empty ()) { // Debug.LogError ("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); //} // convert image to greyscale. Mat gray = new Mat(); Imgproc.cvtColor(rgbaMat, gray, Imgproc.COLOR_RGBA2GRAY); MatOfRect faces = new MatOfRect(); Imgproc.equalizeHist(gray, gray); cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(gray.cols() * 0.05, gray.cols() * 0.05), new Size()); //Debug.Log ("faces " + faces.dump ()); detectResult = faces.toList(); // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib. foreach (Rect r in detectResult) { r.y += (int)(r.height * 0.1f); } gray.Dispose(); } // detect face landmark points. OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat); List <List <Vector2> > landmarkPoints = new List <List <Vector2> >(); foreach (var openCVRect in detectResult) { UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height); Debug.Log("face : " + rect); //OpenCVForUnityUtils.DrawFaceRect(imgMat, rect, new Scalar(255, 0, 0, 255), 2); List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect); //OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2); landmarkPoints.Add(points); } // filter non frontal faces. if (filterNonFrontalFaces) { for (int i = 0; i < landmarkPoints.Count; i++) { if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit) { detectResult.RemoveAt(i); landmarkPoints.RemoveAt(i); i--; } } } // change faces. int[] face_nums = new int[landmarkPoints.Count]; for (int i = 0; i < face_nums.Length; i++) { face_nums[i] = i; } face_nums = face_nums.OrderBy(i => System.Guid.NewGuid()).ToArray(); if (landmarkPoints.Count >= 2) { DlibFaceChanger faceChanger = new DlibFaceChanger(); faceChanger.isShowingDebugFacePoints = displayDebugFacePoints; faceChanger.SetTargetImage(rgbaMat); for (int i = 1; i < face_nums.Length; i++) { faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints[face_nums[0]], landmarkPoints[face_nums[i]], 1); } faceChanger.ChangeFace(); faceChanger.Dispose(); } // draw face rects. if (displayFaceRects && face_nums.Count() > 0) { int ann = face_nums[0]; UnityEngine.Rect rect_ann = new UnityEngine.Rect(detectResult[ann].x, detectResult[ann].y, detectResult[ann].width, detectResult[ann].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_ann, new Scalar(255, 255, 0, 255), 2); int bob = 0; for (int i = 1; i < face_nums.Length; i++) { bob = face_nums[i]; UnityEngine.Rect rect_bob = new UnityEngine.Rect(detectResult[bob].x, detectResult[bob].y, detectResult[bob].width, detectResult[bob].height); OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_bob, new Scalar(255, 0, 0, 255), 2); } } frontalFaceChecker.Dispose(); Texture2D texture = new Texture2D(rgbaMat.cols(), rgbaMat.rows(), TextureFormat.RGBA32, false); OpenCVForUnity.UnityUtils.Utils.matToTexture2D(rgbaMat, texture); gameObject.GetComponent <Renderer>().material.mainTexture = texture; rgbaMat.Dispose(); }