private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
#if !UNITY_WSA_10_0
if (cascade.empty())
{
Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
#endif
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
#if UNITY_ANDROID && !UNITY_EDITOR
// Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2).
webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true;
#endif
webCamTextureToMatHelper.Initialize();
}
private string Benchmark(Texture2D targetImg, FaceLandmarkDetector detector, int times = 100)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
string result = "sp_name: " + dlibShapePredictorFileName + "\n";
result += "times: " + times + " (size: " + targetImg.width + "*" + targetImg.height + ")" + "\n";
detector.SetImage(targetImg);
// FaceLandmarkDetector.Detect() benchmark.
sw.Start();
for (int i = 0; i < times; ++i)
{
detector.Detect();
}
sw.Stop();
result += " Detect(): " + sw.ElapsedMilliseconds + "ms" + " Avg:" + sw.ElapsedMilliseconds / times + "ms" + "\n";
// FaceLandmarkDetector.DetectLandmark() benchmark.
List <Rect> detectResult = detector.Detect();
sw.Reset();
sw.Start();
for (int i = 0; i < times; ++i)
{
foreach (var rect in detectResult)
{
detector.DetectLandmark(rect);
}
}
sw.Stop();
result += " DetectLandmark(): " + sw.ElapsedMilliseconds + "ms" + " Avg:" + sw.ElapsedMilliseconds / times + "ms";
return(result);
}
private void Run()
{
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
useSeamlessCloneToggle.isOn = useSeamlessClone;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
if (imgTexture == null)
{
imgTexture = Resources.Load("family") as Texture2D;
}
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = 0;
float height = 0;
width = gameObject.transform.localScale.x;
height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
Mat rgbaMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(imgTexture, rgbaMat);
Debug.Log("rgbaMat ToString " + rgbaMat.ToString());
if (faceLandmarkDetector == null)
{
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
}
FrontalFaceChecker frontalFaceChecker = new FrontalFaceChecker(width, height);
// detect faces.
List <Rect> detectResult = new List <Rect>();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
if (cascade == null)
{
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
}
//if (cascade.empty())
//{
// Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
//}
// convert image to greyscale.
Mat gray = new Mat();
Imgproc.cvtColor(rgbaMat, gray, Imgproc.COLOR_RGBA2GRAY);
MatOfRect faces = new MatOfRect();
Imgproc.equalizeHist(gray, gray);
cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(gray.cols() * 0.05, gray.cols() * 0.05), new Size());
//Debug.Log ("faces " + faces.dump ());
detectResult = faces.toList();
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
foreach (Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
gray.Dispose();
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
foreach (var openCVRect in detectResult)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
Debug.Log("face : " + rect);
//OpenCVForUnityUtils.DrawFaceRect(imgMat, rect, new Scalar(255, 0, 0, 255), 2);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2);
landmarkPoints.Add(points);
}
// filter non frontal facea.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit)
{
detectResult.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// swap faces.
int[] face_nums = new int[landmarkPoints.Count];
for (int i = 0; i < face_nums.Length; i++)
{
face_nums[i] = i;
}
face_nums = face_nums.OrderBy(i => System.Guid.NewGuid()).ToArray();
if (landmarkPoints.Count >= 2)
{
DlibFaceSwapper faceSwapper = new DlibFaceSwapper();
faceSwapper.useSeamlessCloneForPasteFaces = useSeamlessClone;
faceSwapper.isShowingDebugFacePoints = displayDebugFacePoints;
int ann = 0, bob = 0;
for (int i = 0; i < face_nums.Length - 1; i += 2)
{
ann = face_nums[i];
bob = face_nums[i + 1];
faceSwapper.SwapFaces(rgbaMat, landmarkPoints[ann], landmarkPoints[bob], 1);
}
faceSwapper.Dispose();
}
// draw face rects.
if (displayFaceRects)
{
int ann = 0, bob = 0;
for (int i = 0; i < face_nums.Length - 1; i += 2)
{
ann = face_nums[i];
bob = face_nums[i + 1];
UnityEngine.Rect rect_ann = new UnityEngine.Rect(detectResult[ann].x, detectResult[ann].y, detectResult[ann].width, detectResult[ann].height);
UnityEngine.Rect rect_bob = new UnityEngine.Rect(detectResult[bob].x, detectResult[bob].y, detectResult[bob].width, detectResult[bob].height);
Scalar color = new Scalar(UnityEngine.Random.Range(0, 256), UnityEngine.Random.Range(0, 256), UnityEngine.Random.Range(0, 256), 255);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_ann, color, 2);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_bob, color, 2);
//Imgproc.putText (rgbaMat, "" + i % 2, new Point (rect_ann.xMin, rect_ann.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, color, 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, "" + (i % 2 + 1), new Point (rect_bob.xMin, rect_bob.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, color, 2, Imgproc.LINE_AA, false);
}
}
frontalFaceChecker.Dispose();
Texture2D texture = new Texture2D(rgbaMat.cols(), rgbaMat.rows(), TextureFormat.RGBA32, false);
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(rgbaMat, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
rgbaMat.Dispose();
}
private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
//if true, The error log of the Native side Dlib will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Texture2D dstTexture2D = new Texture2D(texture2D.width, texture2D.height, texture2D.format, false);
Graphics.CopyTexture(texture2D, dstTexture2D);
gameObject.transform.localScale = new Vector3(texture2D.width, texture2D.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = gameObject.transform.localScale.x;
float height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
FaceLandmarkDetector faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
faceLandmarkDetector.SetImage(texture2D);
//detect face rects
List <Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
Debug.Log("face : " + rect);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
Debug.Log("face points count : " + points.Count);
foreach (var point in points)
{
Debug.Log("face point : x " + point.x + " y " + point.y);
}
//draw landmark points
faceLandmarkDetector.DrawDetectLandmarkResult(dstTexture2D, 0, 255, 0, 255);
}
//draw face rect
faceLandmarkDetector.DrawDetectResult(dstTexture2D, 255, 0, 0, 255, 2);
faceLandmarkDetector.Dispose();
gameObject.GetComponent <Renderer>().material.mainTexture = dstTexture2D;
Utils.setDebugMode(false);
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName);
fpsMonitor.Add("width", width.ToString());
fpsMonitor.Add("height", height.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
}
private void Run()
{
//set 3d face object points.
objectPoints68 = new MatOfPoint3f(
new Point3(-34, 90, 83), //l eye (Interpupillary breadth)
new Point3(34, 90, 83), //r eye (Interpupillary breadth)
new Point3(0.0, 50, 120), //nose (Nose top)
new Point3(-26, 15, 83), //l mouse (Mouth breadth)
new Point3(26, 15, 83), //r mouse (Mouth breadth)
new Point3(-79, 90, 0.0), //l ear (Bitragion breadth)
new Point3(79, 90, 0.0) //r ear (Bitragion breadth)
);
objectPoints5 = new MatOfPoint3f(
new Point3(-23, 90, 83), //l eye (Inner corner of the eye)
new Point3(23, 90, 83), //r eye (Inner corner of the eye)
new Point3(-50, 90, 80), //l eye (Tail of the eye)
new Point3(50, 90, 80), //r eye (Tail of the eye)
new Point3(0.0, 50, 120) //nose (Nose top)
);
imagePoints = new MatOfPoint2f();
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
rgbMat = new Mat();
capture = new VideoCapture();
capture.open(dance_avi_filepath);
if (capture.isOpened())
{
Debug.Log("capture.isOpened() true");
}
else
{
Debug.Log("capture.isOpened() false");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
capture.grab();
capture.retrieve(rgbMat, 0);
int frameWidth = rgbMat.cols();
int frameHeight = rgbMat.rows();
texture = new Texture2D(frameWidth, frameHeight, TextureFormat.RGB24, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName);
fpsMonitor.Add("width", frameWidth.ToString());
fpsMonitor.Add("height", frameHeight.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
float width = (float)frameWidth;
float height = (float)frameHeight;
float imageSizeScale = 1.0f;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
imageSizeScale = (float)Screen.height / (float)Screen.width;
}
else
{
Camera.main.orthographicSize = height / 2;
}
//set cameraparam
int max_d = (int)Mathf.Max(width, height);
double fx = max_d;
double fy = max_d;
double cx = width / 2.0f;
double cy = height / 2.0f;
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, fx);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, cx);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, fy);
camMatrix.put(1, 2, cy);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
Debug.Log("camMatrix " + camMatrix.dump());
distCoeffs = new MatOfDouble(0, 0, 0, 0);
Debug.Log("distCoeffs " + distCoeffs.dump());
//calibration camera
Size imageSize = new Size(width * imageSizeScale, height * imageSizeScale);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point(0, 0);
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx [0]);
Debug.Log("fovy " + fovy [0]);
Debug.Log("focalLength " + focalLength [0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio [0]);
//To convert the difference of the FOV value of the OpenCV and Unity.
double fovXScale = (2.0 * Mathf.Atan((float)(imageSize.width / (2.0 * fx)))) / (Mathf.Atan2((float)cx, (float)fx) + Mathf.Atan2((float)(imageSize.width - cx), (float)fx));
double fovYScale = (2.0 * Mathf.Atan((float)(imageSize.height / (2.0 * fy)))) / (Mathf.Atan2((float)cy, (float)fy) + Mathf.Atan2((float)(imageSize.height - cy), (float)fy));
Debug.Log("fovXScale " + fovXScale);
Debug.Log("fovYScale " + fovYScale);
//Adjust Unity Camera FOV https://github.com/opencv/opencv/commit/8ed1945ccd52501f5ab22bdec6aa1f91f1e2cfd4
if (widthScale < heightScale)
{
ARCamera.fieldOfView = (float)(fovx [0] * fovXScale);
}
else
{
ARCamera.fieldOfView = (float)(fovy [0] * fovYScale);
}
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
axes.SetActive(false);
head.SetActive(false);
rightEye.SetActive(false);
leftEye.SetActive(false);
mouth.SetActive(false);
mouthParticleSystem = mouth.GetComponentsInChildren <ParticleSystem> (true);
}
// Start is called before the first frame update
void Awake()
{
dlibShapePredictorFilePath = Utils.getFilePath(dlibShapePredictorMobileFileName);
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
}
private void Run()
{
if (string.IsNullOrEmpty(object_detector_filepath))
{
Debug.LogError("object detecter file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
if (string.IsNullOrEmpty(shape_predictor_filepath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
Texture2D dstTexture2D = new Texture2D(texture2D.width, texture2D.height, texture2D.format, false);
Graphics.CopyTexture(texture2D, dstTexture2D);
gameObject.transform.localScale = new Vector3(texture2D.width, texture2D.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = gameObject.transform.localScale.x;
float height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
FaceLandmarkDetector faceLandmarkDetector = new FaceLandmarkDetector(object_detector_filepath, shape_predictor_filepath);
faceLandmarkDetector.SetImage(texture2D);
//detect face rects
List <Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
Debug.Log("face : " + rect);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
Debug.Log("face points count : " + points.Count);
foreach (var point in points)
{
Debug.Log("face point : x " + point.x + " y " + point.y);
}
//draw landmark points
faceLandmarkDetector.DrawDetectLandmarkResult(dstTexture2D, 0, 255, 0, 255);
}
//draw face rects
faceLandmarkDetector.DrawDetectResult(dstTexture2D, 255, 0, 0, 255, 3);
faceLandmarkDetector.Dispose();
gameObject.GetComponent <Renderer> ().material.mainTexture = dstTexture2D;
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib object detector", "frontal_cat_face.svm");
fpsMonitor.Add("dlib shape predictor", "sp_cat_face_68.dat");
fpsMonitor.Add("width", width.ToString());
fpsMonitor.Add("height", height.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
}
void Start()
{
_detector = new FaceLandmarkDetector(_resources);
_material = new Material(_shader);
}
private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
rgbMat = new Mat();
capture = new VideoCapture();
capture.open(video_filepath);
if (!capture.isOpened())
{
Debug.LogError("capture.isOpened() is false. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
double ext = capture.get(Videoio.CAP_PROP_FOURCC);
Debug.Log("CAP_PROP_FOURCC: " + (char)((int)ext & 0XFF) + (char)(((int)ext & 0XFF00) >> 8) + (char)(((int)ext & 0XFF0000) >> 16) + (char)(((int)ext & 0XFF000000) >> 24));
capture.grab();
capture.retrieve(rgbMat, 0);
int frameWidth = rgbMat.cols();
int frameHeight = rgbMat.rows();
texture = new Texture2D(frameWidth, frameHeight, TextureFormat.RGB24, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
float widthScale = (float)Screen.width / (float)frameWidth;
float heightScale = (float)Screen.height / (float)frameHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = (float)frameHeight / 2;
}
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName);
fpsMonitor.Add("width", frameWidth.ToString());
fpsMonitor.Add("height", frameHeight.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
}
private void Run()
{
meshOverlay = this.GetComponent <TrackedMeshOverlay> ();
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
enableColorCorrectionToggle.isOn = enableColorCorrection;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
if (imgTexture == null)
{
imgTexture = Resources.Load("family") as Texture2D;
}
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
meshOverlay.UpdateOverlayTransform(gameObject.transform);
meshOverlay.Reset();
float width = 0;
float height = 0;
width = gameObject.transform.localScale.x;
height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
Mat rgbaMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
OpenCVForUnity.Utils.texture2DToMat(imgTexture, rgbaMat);
Debug.Log("rgbaMat ToString " + rgbaMat.ToString());
if (faceLandmarkDetector == null)
{
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
}
faceMaskColorCorrector = faceMaskColorCorrector ?? new FaceMaskColorCorrector();
FrontalFaceChecker frontalFaceChecker = new FrontalFaceChecker(width, height);
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
if (cascade == null)
{
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
}
// if (cascade.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
// convert image to greyscale.
Mat gray = new Mat();
Imgproc.cvtColor(rgbaMat, gray, Imgproc.COLOR_RGBA2GRAY);
MatOfRect faces = new MatOfRect();
Imgproc.equalizeHist(gray, gray);
cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(gray.cols() * 0.05, gray.cols() * 0.05), new Size());
//Debug.Log ("faces " + faces.dump ());
detectResult = faces.toList();
// adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
gray.Dispose();
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> > ();
foreach (var openCVRect in detectResult)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
Debug.Log("face : " + rect);
//OpenCVForUnityUtils.DrawFaceRect(imgMat, rect, new Scalar(255, 0, 0, 255), 2);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2);
landmarkPoints.Add(points);
}
// mask faces.
int[] face_nums = new int[landmarkPoints.Count];
for (int i = 0; i < face_nums.Length; i++)
{
face_nums [i] = i;
}
face_nums = face_nums.OrderBy(i => System.Guid.NewGuid()).ToArray();
float imageWidth = meshOverlay.width;
float imageHeight = meshOverlay.height;
float maskImageWidth = imgTexture.width;
float maskImageHeight = imgTexture.height;
TrackedMesh tm;
for (int i = 0; i < face_nums.Length; i++)
{
meshOverlay.CreateObject(i, imgTexture);
tm = meshOverlay.GetObjectById(i);
Vector3[] vertices = tm.meshFilter.mesh.vertices;
if (vertices.Length == landmarkPoints [face_nums [i]].Count)
{
for (int j = 0; j < vertices.Length; j++)
{
vertices [j].x = landmarkPoints [face_nums [i]] [j].x / imageWidth - 0.5f;
vertices [j].y = 0.5f - landmarkPoints [face_nums [i]] [j].y / imageHeight;
}
}
Vector2[] uv = tm.meshFilter.mesh.uv;
if (uv.Length == landmarkPoints [face_nums [0]].Count)
{
for (int jj = 0; jj < uv.Length; jj++)
{
uv [jj].x = landmarkPoints [face_nums [0]] [jj].x / maskImageWidth;
uv [jj].y = (maskImageHeight - landmarkPoints [face_nums [0]] [jj].y) / maskImageHeight;
}
}
meshOverlay.UpdateObject(i, vertices, null, uv);
if (enableColorCorrection)
{
faceMaskColorCorrector.CreateLUTTex(i);
Texture2D LUTTex = faceMaskColorCorrector.UpdateLUTTex(i, rgbaMat, rgbaMat, landmarkPoints [face_nums [0]], landmarkPoints [face_nums [i]]);
tm.sharedMaterial.SetTexture("_LUTTex", LUTTex);
tm.sharedMaterial.SetFloat("_ColorCorrection", 1f);
}
else
{
tm.sharedMaterial.SetFloat("_ColorCorrection", 0f);
}
// filter non frontal faces.
if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit)
{
tm.sharedMaterial.SetFloat("_Fade", 1f);
}
else
{
tm.sharedMaterial.SetFloat("_Fade", 0.3f);
}
}
// draw face rects.
if (displayFaceRects)
{
int ann = face_nums[0];
UnityEngine.Rect rect_ann = new UnityEngine.Rect(detectResult [ann].x, detectResult [ann].y, detectResult [ann].width, detectResult [ann].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_ann, new Scalar(255, 255, 0, 255), 2);
int bob = 0;
for (int i = 1; i < face_nums.Length; i++)
{
bob = face_nums [i];
UnityEngine.Rect rect_bob = new UnityEngine.Rect(detectResult [bob].x, detectResult [bob].y, detectResult [bob].width, detectResult [bob].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_bob, new Scalar(255, 0, 0, 255), 2);
}
}
// draw face points.
if (displayDebugFacePoints)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2);
}
}
Texture2D texture = new Texture2D(rgbaMat.cols(), rgbaMat.rows(), TextureFormat.RGBA32, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture);
gameObject.transform.GetComponent <Renderer> ().material.mainTexture = texture;
frontalFaceChecker.Dispose();
rgbaMat.Dispose();
}
private void Run()
{
Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
// Convert Unity Texture2D to OpenCV Mat.
OpenCVForUnity.Utils.texture2DToMat(imgTexture, imgMat);
Debug.Log("imgMat dst ToString " + imgMat.ToString());
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = imgMat.width();
float height = imgMat.height();
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
FaceLandmarkDetector faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, imgMat);
//detect face rectdetecton
List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection();
foreach (var result in detectResult)
{
Debug.Log("rect : " + result.rect);
Debug.Log("detection_confidence : " + result.detection_confidence);
Debug.Log("weight_index : " + result.weight_index);
// Debug.Log ("face : " + rect);
Imgproc.putText(imgMat, "" + result.detection_confidence, new Point(result.rect.xMin, result.rect.yMin - 20), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
Imgproc.putText(imgMat, "" + result.weight_index, new Point(result.rect.xMin, result.rect.yMin - 5), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(result.rect);
Debug.Log("face points count : " + points.Count);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(imgMat, result.rect, new Scalar(255, 0, 0, 255), 2);
}
faceLandmarkDetector.Dispose();
Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false);
// Convert OpenCV Mat to Unity Texture2D.
OpenCVForUnity.Utils.matToTexture2D(imgMat, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
}
private void Run()
{
meshOverlay = this.GetComponent <TrackedMeshOverlay> ();
shader_FadeID = Shader.PropertyToID("_Fade");
rectangleTracker = new RectangleTracker();
faceLandmarkDetector = new FaceLandmarkDetector(shape_predictor_68_face_landmarks_dat_filepath);
frontalFaceParam = new FrontalFaceParam();
rgbMat = new Mat();
capture.open(couple_avi_filepath);
if (capture.isOpened())
{
Debug.Log("capture.isOpened() true");
}
else
{
Debug.Log("capture.isOpened() false");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
texture = new Texture2D((int)(frameWidth), (int)(frameHeight), TextureFormat.RGBA32, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
float widthScale = (float)Screen.width / (float)frameWidth;
float heightScale = (float)Screen.height / (float)frameHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = (float)frameHeight / 2;
}
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
meshOverlay.UpdateOverlayTransform();
grayMat = new Mat((int)frameHeight, (int)frameWidth, CvType.CV_8UC1);
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
if (cascade.empty())
{
Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerSample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
isShowingFaceRectsToggle.isOn = isShowingFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
isFilteringNonFrontalFacesToggle.isOn = isFilteringNonFrontalFaces;
isShowingDebugFacePointsToggle.isOn = isShowingDebugFacePoints;
OnChangeFaceMaskButton();
}
private void DetectInRegion(Mat img, Rect region, List <Rect> detectedObjectsInRegions, FaceLandmarkDetector landmarkDetector)
{
Rect r0 = new Rect(new Point(), img.size());
Rect r1 = new Rect(region.x, region.y, region.width, region.height);
Rect.inflate(r1, (int)((r1.width * coeffTrackingWindowSize) - r1.width) / 2,
(int)((r1.height * coeffTrackingWindowSize) - r1.height) / 2);
r1 = Rect.intersect(r0, r1);
if ((r1.width <= 0) || (r1.height <= 0))
{
Debug.Log("detectInRegion: Empty intersection");
return;
}
using (Mat img1_roi = new Mat(img, r1))
using (Mat img1 = new Mat(r1.size(), img.type()))
{
img1_roi.copyTo(img1);
OpenCVForUnityUtils.SetImage(landmarkDetector, img1);
List <UnityEngine.Rect> detectResult = landmarkDetector.Detect();
int len = detectResult.Count;
for (int i = 0; i < len; i++)
{
UnityEngine.Rect tmp = detectResult[i];
Rect r = new Rect((int)(tmp.x + r1.x), (int)(tmp.y + r1.y), (int)tmp.width, (int)tmp.height);
detectedObjectsInRegions.Add(r);
}
}
}
private void Run()
{
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
StartCoroutine(Initialize());
}
// Use this for initialization
void Start()
{
faceLandmarkDetector = new FaceLandmarkDetector(Utils.getFilePath("shape_predictor_68_face_landmarks.dat"));
StartCoroutine(init());
}
private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
// Convert Unity Texture2D to OpenCV Mat.
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(imgTexture, imgMat);
Debug.Log("imgMat dst ToString " + imgMat.ToString());
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = imgMat.width();
float height = imgMat.height();
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
FaceLandmarkDetector faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, imgMat);
//detect face rectdetecton
List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection();
foreach (var result in detectResult)
{
Debug.Log("rect : " + result.rect);
Debug.Log("detection_confidence : " + result.detection_confidence);
Debug.Log("weight_index : " + result.weight_index);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(result.rect);
Debug.Log("face points count : " + points.Count);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2, true);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(imgMat, result, new Scalar(255, 0, 0, 255), 2);
}
faceLandmarkDetector.Dispose();
Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false);
// Convert OpenCV Mat to Unity Texture2D.
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(imgMat, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName);
fpsMonitor.Add("width", width.ToString());
fpsMonitor.Add("height", height.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
}
private void Run()
{
faceLandmarkDetector = new FaceLandmarkDetector(shape_predictor_68_face_landmarks_dat_filepath);
StartCoroutine(init());
}
//struct FaceData
//{
// internal Vector2 leftEye;
// internal Vector2 rightEye;
// internal Image spex;
// internal void ReOrient(Vector2 leftEye, Vector2 rightEye, Image spex)
// {
// leftEye
// }
//}
void SetupFaceDetection()
{
spexMap = new List <Image>();
faceLandmarkDetector = new FaceLandmarkDetector(System.IO.Path.Combine(Application.streamingAssetsPath, LibraryMap[library]));
faceLandmarkDetector.SetImage(sourceImage);
}
private void Run()
{
meshOverlay = this.GetComponent <TrackedMeshOverlay> ();
shader_FadeID = Shader.PropertyToID("_Fade");
shader_ColorCorrectionID = Shader.PropertyToID("_ColorCorrection");
shader_LUTTexID = Shader.PropertyToID("_LUTTex");
rectangleTracker = new RectangleTracker();
frontalFaceChecker = new FrontalFaceChecker((float)frameWidth, (float)frameHeight);
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
lowPassFilterDict = new Dictionary <int, LowPassPointsFilter> ();
opticalFlowFilterDict = new Dictionary <int, OFPointsFilter> ();
faceMaskColorCorrector = new FaceMaskColorCorrector();
rgbMat = new Mat();
capture.open(couple_avi_filepath);
if (capture.isOpened())
{
Debug.Log("capture.isOpened() true");
}
else
{
Debug.Log("capture.isOpened() false");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
texture = new Texture2D((int)(frameWidth), (int)(frameHeight), TextureFormat.RGB24, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
float widthScale = (float)Screen.width / (float)frameWidth;
float heightScale = (float)Screen.height / (float)frameHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = (float)frameHeight / 2;
}
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
meshOverlay.UpdateOverlayTransform(gameObject.transform);
grayMat = new Mat((int)frameHeight, (int)frameWidth, CvType.CV_8UC1);
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
// if (cascade.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
enableNoiseFilterToggle.isOn = enableNoiseFilter;
enableColorCorrectionToggle.isOn = enableColorCorrection;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
OnChangeFaceMaskButtonClick();
}
private void Run()
{
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
Initialize();
}
private void Run()
{
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
rgbMat = new Mat();
capture = new VideoCapture();
capture.open(dance_avi_filepath);
if (capture.isOpened())
{
Debug.Log("capture.isOpened() true");
}
else
{
Debug.Log("capture.isOpened() false");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
capture.grab();
capture.retrieve(rgbMat, 0);
int frameWidth = rgbMat.cols();
int frameHeight = rgbMat.rows();
texture = new Texture2D(frameWidth, frameHeight, TextureFormat.RGB24, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
float widthScale = (float)Screen.width / (float)frameWidth;
float heightScale = (float)Screen.height / (float)frameHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = (float)frameHeight / 2;
}
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName);
fpsMonitor.Add("width", frameWidth.ToString());
fpsMonitor.Add("height", frameHeight.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
}
private void Run()
{
meshOverlay = this.GetComponent <TrackedMeshOverlay> ();
// Customize face mask.
GameObject newBaseObject = Instantiate(meshOverlay.baseObject);
newBaseObject.name = "CustomFaceMaskTrackedMesh";
TrackedMesh tm = newBaseObject.GetComponent <TrackedMesh> ();
if (extendForehead)
{
ExtendForehead(tm.meshFilter.mesh);
}
Texture alphaMask = tm.material.GetTexture("_MaskTex");
Vector2[] uv = tm.meshFilter.sharedMesh.uv2;
Texture2D newAlphaMask = CreateFaceMaskAlphaMaskTexture(alphaMask.width, alphaMask.height, uv, makeBothEyesTransparent, makeMouthTransparent);
tm.material.SetTexture("_MaskTex", newAlphaMask);
meshOverlay.baseObject = newBaseObject;
shader_FadeID = Shader.PropertyToID("_Fade");
shader_ColorCorrectionID = Shader.PropertyToID("_ColorCorrection");
shader_LUTTexID = Shader.PropertyToID("_LUTTex");
rectangleTracker = new RectangleTracker();
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
lowPassFilterDict = new Dictionary <int, LowPassPointsFilter> ();
opticalFlowFilterDict = new Dictionary <int, OFPointsFilter> ();
faceMaskColorCorrector = new FaceMaskColorCorrector();
#if UNITY_ANDROID && !UNITY_EDITOR
// Set the requestedFPS parameter to avoid the problem of the WebCamTexture image becoming low light on some Android devices. (Pixel, pixel 2)
// https://forum.unity.com/threads/android-webcamtexture-in-low-light-only-some-models.520656/
// https://forum.unity.com/threads/released-opencv-for-unity.277080/page-33#post-3445178
rearCameraRequestedFPS = webCamTextureToMatHelper.requestedFPS;
if (webCamTextureToMatHelper.requestedIsFrontFacing)
{
webCamTextureToMatHelper.requestedFPS = 15;
webCamTextureToMatHelper.Initialize();
}
else
{
webCamTextureToMatHelper.Initialize();
}
#else
webCamTextureToMatHelper.Initialize();
#endif
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
enableNoiseFilterToggle.isOn = enableNoiseFilter;
enableColorCorrectionToggle.isOn = enableColorCorrection;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
}
private void Run()
{
rectangleTracker = new RectangleTracker();
frontalFaceChecker = new FrontalFaceChecker((float)frameWidth, (float)frameHeight);
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
faceSwapper = new DlibFaceSwapper();
faceSwapper.useSeamlessCloneForPasteFaces = useSeamlessClone;
faceSwapper.isShowingDebugFacePoints = displayDebugFacePoints;
rgbMat = new Mat();
capture.open(couple_avi_filepath);
if (capture.isOpened())
{
Debug.Log("capture.isOpened() true");
}
else
{
Debug.Log("capture.isOpened() false");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
texture = new Texture2D((int)(frameWidth), (int)(frameHeight), TextureFormat.RGBA32, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
float widthScale = (float)Screen.width / (float)frameWidth;
float heightScale = (float)Screen.height / (float)frameHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = (float)frameHeight / 2;
}
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
grayMat = new Mat((int)frameHeight, (int)frameWidth, CvType.CV_8UC1);
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
// if (cascade.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
useSeamlessCloneToggle.isOn = useSeamlessClone;
enableTrackingToggle.isOn = enableTracking;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
}
// Use this for initialization
void Start()
{
//set 3d face object points.
objectPoints = new MatOfPoint3f(
new Point3(-31, 72, 86), //l eye
new Point3(31, 72, 86), //r eye
new Point3(0, 40, 114), //nose
new Point3(-20, 15, 90), //l mouse
new Point3(20, 15, 90), //r mouse
new Point3(-69, 76, -2), //l ear
new Point3(69, 76, -2) //r ear
);
imagePoints = new MatOfPoint2f();
rvec = new Mat();
tvec = new Mat();
rotM = new Mat(3, 3, CvType.CV_64FC1);
faceLandmarkDetector = new FaceLandmarkDetector(DlibFaceLandmarkDetector.Utils.getFilePath("shape_predictor_68_face_landmarks.dat"));
rgbMat = new Mat();
capture = new VideoCapture();
capture.open(OpenCVForUnity.Utils.getFilePath("dance.avi"));
if (capture.isOpened())
{
Debug.Log("capture.isOpened() true");
}
else
{
Debug.Log("capture.isOpened() false");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
colors = new Color32[(int)(frameWidth) * (int)(frameHeight)];
texture = new Texture2D((int)(frameWidth), (int)(frameHeight), TextureFormat.RGBA32, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = 0;
float height = 0;
width = gameObject.transform.localScale.x;
height = gameObject.transform.localScale.y;
float imageScale = 1.0f;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
imageScale = (float)Screen.height / (float)Screen.width;
}
else
{
Camera.main.orthographicSize = height / 2;
}
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
int max_d = (int)Mathf.Max((float)frameHeight, (float)frameWidth);
camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, max_d);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, (float)frameWidth / 2.0f);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, max_d);
camMatrix.put(1, 2, (float)frameHeight / 2.0f);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
Size imageSize = new Size((float)frameWidth * imageScale, (float)frameHeight * imageScale);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point();
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx [0]);
Debug.Log("fovy " + fovy [0]);
Debug.Log("focalLength " + focalLength [0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio [0]);
if (widthScale < heightScale)
{
ARCamera.fieldOfView = (float)fovx [0];
}
else
{
ARCamera.fieldOfView = (float)fovy [0];
}
Debug.Log("camMatrix " + camMatrix.dump());
distCoeffs = new MatOfDouble(0, 0, 0, 0);
Debug.Log("distCoeffs " + distCoeffs.dump());
invertYM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, -1, 1));
Debug.Log("invertYM " + invertYM.ToString());
invertZM = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
Debug.Log("invertZM " + invertZM.ToString());
axes.SetActive(false);
head.SetActive(false);
rightEye.SetActive(false);
leftEye.SetActive(false);
mouth.SetActive(false);
mouthParticleSystem = mouth.GetComponentsInChildren <ParticleSystem> (true);
}
private void Run()
{
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
didUpdateFaceLanmarkPoints = false;
}
