/// <summary>
/// Raises the web cam texture to mat helper initialized event.
/// </summary>
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat();
texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false);
gameObject.transform.localScale = new Vector3(webCamTextureMat.cols(), webCamTextureMat.rows(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
if (fpsMonitor != null)
{
fpsMonitor.Add("width", webCamTextureMat.width().ToString());
fpsMonitor.Add("height", webCamTextureMat.height().ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
float width = gameObject.transform.localScale.x;
float height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
grayMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC1);
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
//if (cascade.empty())
//{
// Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
//}
frontalFaceChecker = new FrontalFaceChecker(width, height);
meshOverlay.UpdateOverlayTransform(gameObject.transform);
OnChangeFaceMaskButtonClick();
}
private void Run()
{
meshOverlay = this.GetComponent <TrackedMeshOverlay> ();
shader_FadeID = Shader.PropertyToID("_Fade");
shader_ColorCorrectionID = Shader.PropertyToID("_ColorCorrection");
shader_LUTTexID = Shader.PropertyToID("_LUTTex");
rectangleTracker = new RectangleTracker();
frontalFaceChecker = new FrontalFaceChecker((float)frameWidth, (float)frameHeight);
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
lowPassFilterDict = new Dictionary <int, LowPassPointsFilter> ();
opticalFlowFilterDict = new Dictionary <int, OFPointsFilter> ();
faceMaskColorCorrector = new FaceMaskColorCorrector();
rgbMat = new Mat();
capture.open(couple_avi_filepath);
if (capture.isOpened())
{
Debug.Log("capture.isOpened() true");
}
else
{
Debug.Log("capture.isOpened() false");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
texture = new Texture2D((int)(frameWidth), (int)(frameHeight), TextureFormat.RGB24, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
float widthScale = (float)Screen.width / (float)frameWidth;
float heightScale = (float)Screen.height / (float)frameHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = (float)frameHeight / 2;
}
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
meshOverlay.UpdateOverlayTransform(gameObject.transform);
grayMat = new Mat((int)frameHeight, (int)frameWidth, CvType.CV_8UC1);
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
// if (cascade.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
enableNoiseFilterToggle.isOn = enableNoiseFilter;
enableColorCorrectionToggle.isOn = enableColorCorrection;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
OnChangeFaceMaskButtonClick();
}
private void Run()
{
meshOverlay = this.GetComponent <TrackedMeshOverlay> ();
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
enableColorCorrectionToggle.isOn = enableColorCorrection;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
if (imgTexture == null)
{
imgTexture = Resources.Load("family") as Texture2D;
}
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
meshOverlay.UpdateOverlayTransform(gameObject.transform);
meshOverlay.Reset();
float width = 0;
float height = 0;
width = gameObject.transform.localScale.x;
height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
Mat rgbaMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
OpenCVForUnity.Utils.texture2DToMat(imgTexture, rgbaMat);
Debug.Log("rgbaMat ToString " + rgbaMat.ToString());
if (faceLandmarkDetector == null)
{
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
}
faceMaskColorCorrector = faceMaskColorCorrector ?? new FaceMaskColorCorrector();
FrontalFaceChecker frontalFaceChecker = new FrontalFaceChecker(width, height);
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
if (cascade == null)
{
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
}
// if (cascade.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
// convert image to greyscale.
Mat gray = new Mat();
Imgproc.cvtColor(rgbaMat, gray, Imgproc.COLOR_RGBA2GRAY);
MatOfRect faces = new MatOfRect();
Imgproc.equalizeHist(gray, gray);
cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(gray.cols() * 0.05, gray.cols() * 0.05), new Size());
//Debug.Log ("faces " + faces.dump ());
detectResult = faces.toList();
// adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
gray.Dispose();
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> > ();
foreach (var openCVRect in detectResult)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
Debug.Log("face : " + rect);
//OpenCVForUnityUtils.DrawFaceRect(imgMat, rect, new Scalar(255, 0, 0, 255), 2);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2);
landmarkPoints.Add(points);
}
// mask faces.
int[] face_nums = new int[landmarkPoints.Count];
for (int i = 0; i < face_nums.Length; i++)
{
face_nums [i] = i;
}
face_nums = face_nums.OrderBy(i => System.Guid.NewGuid()).ToArray();
float imageWidth = meshOverlay.width;
float imageHeight = meshOverlay.height;
float maskImageWidth = imgTexture.width;
float maskImageHeight = imgTexture.height;
TrackedMesh tm;
for (int i = 0; i < face_nums.Length; i++)
{
meshOverlay.CreateObject(i, imgTexture);
tm = meshOverlay.GetObjectById(i);
Vector3[] vertices = tm.meshFilter.mesh.vertices;
if (vertices.Length == landmarkPoints [face_nums [i]].Count)
{
for (int j = 0; j < vertices.Length; j++)
{
vertices [j].x = landmarkPoints [face_nums [i]] [j].x / imageWidth - 0.5f;
vertices [j].y = 0.5f - landmarkPoints [face_nums [i]] [j].y / imageHeight;
}
}
Vector2[] uv = tm.meshFilter.mesh.uv;
if (uv.Length == landmarkPoints [face_nums [0]].Count)
{
for (int jj = 0; jj < uv.Length; jj++)
{
uv [jj].x = landmarkPoints [face_nums [0]] [jj].x / maskImageWidth;
uv [jj].y = (maskImageHeight - landmarkPoints [face_nums [0]] [jj].y) / maskImageHeight;
}
}
meshOverlay.UpdateObject(i, vertices, null, uv);
if (enableColorCorrection)
{
faceMaskColorCorrector.CreateLUTTex(i);
Texture2D LUTTex = faceMaskColorCorrector.UpdateLUTTex(i, rgbaMat, rgbaMat, landmarkPoints [face_nums [0]], landmarkPoints [face_nums [i]]);
tm.sharedMaterial.SetTexture("_LUTTex", LUTTex);
tm.sharedMaterial.SetFloat("_ColorCorrection", 1f);
}
else
{
tm.sharedMaterial.SetFloat("_ColorCorrection", 0f);
}
// filter non frontal faces.
if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit)
{
tm.sharedMaterial.SetFloat("_Fade", 1f);
}
else
{
tm.sharedMaterial.SetFloat("_Fade", 0.3f);
}
}
// draw face rects.
if (displayFaceRects)
{
int ann = face_nums[0];
UnityEngine.Rect rect_ann = new UnityEngine.Rect(detectResult [ann].x, detectResult [ann].y, detectResult [ann].width, detectResult [ann].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_ann, new Scalar(255, 255, 0, 255), 2);
int bob = 0;
for (int i = 1; i < face_nums.Length; i++)
{
bob = face_nums [i];
UnityEngine.Rect rect_bob = new UnityEngine.Rect(detectResult [bob].x, detectResult [bob].y, detectResult [bob].width, detectResult [bob].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_bob, new Scalar(255, 0, 0, 255), 2);
}
}
// draw face points.
if (displayDebugFacePoints)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2);
}
}
Texture2D texture = new Texture2D(rgbaMat.cols(), rgbaMat.rows(), TextureFormat.RGBA32, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture);
gameObject.transform.GetComponent <Renderer> ().material.mainTexture = texture;
frontalFaceChecker.Dispose();
rgbaMat.Dispose();
}