private void Run()
{
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
#if UNITY_ANDROID && !UNITY_EDITOR
// Set the requestedFPS parameter to avoid the problem of the WebCamTexture image becoming low light on some Android devices. (Pixel, pixel 2)
// https://forum.unity.com/threads/android-webcamtexture-in-low-light-only-some-models.520656/
// https://forum.unity.com/threads/released-opencv-for-unity.277080/page-33#post-3445178
rearCameraRequestedFPS = webCamTextureToMatHelper.requestedFPS;
if (webCamTextureToMatHelper.requestedIsFrontFacing)
{
webCamTextureToMatHelper.requestedFPS = 15;
webCamTextureToMatHelper.Initialize();
}
else
{
webCamTextureToMatHelper.Initialize();
}
#else
webCamTextureToMatHelper.Initialize();
#endif
}
private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
webCamTextureToMatHelper.Initialize();
}
private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
#if UNITY_ANDROID && !UNITY_EDITOR
// Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2).
webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true;
#endif
webCamTextureToMatHelper.Initialize();
}
private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
if (string.IsNullOrEmpty(sourceToMatHelper.requestedVideoFilePath))
{
sourceToMatHelper.requestedVideoFilePath = VIDEO_FILENAME;
}
sourceToMatHelper.outputColorFormat = VideoCaptureToMatHelper.ColorFormat.RGB;
sourceToMatHelper.Initialize();
}
// Use this for initialization
protected override void Start()
{
base.Start();
displayCameraPreviewToggle.isOn = displayCameraPreview;
useSeparateDetectionToggle.isOn = useSeparateDetection;
displayAxesToggle.isOn = displayAxes;
displayHeadToggle.isOn = displayHead;
displayEffectsToggle.isOn = displayEffects;
enableOpticalFlowFilterToggle.isOn = enableOpticalFlowFilter;
enableLowPassFilterToggle.isOn = enableLowPassFilter;
imageOptimizationHelper = gameObject.GetComponent <ImageOptimizationHelper> ();
webCamTextureToMatHelper = gameObject.GetComponent <HololensCameraStreamToMatHelper> ();
#if NETFX_CORE && !DISABLE_HOLOLENSCAMSTREAM_API
webCamTextureToMatHelper.frameMatAcquired += OnFrameMatAcquired;
#endif
webCamTextureToMatHelper.Initialize();
rectangleTracker = new RectangleTracker();
faceLandmarkDetector = new FaceLandmarkDetector(DlibFaceLandmarkDetector.Utils.getFilePath("sp_human_face_68.dat"));
// faceLandmarkDetector = new FaceLandmarkDetector (DlibFaceLandmarkDetector.Utils.getFilePath ("sp_human_face_68_for_mobile.dat"));
// The coordinates of the detection object on the real world space connected with the pixel coordinates.(mm)
objectPoints = new MatOfPoint3f(
new Point3(-34, 90, 83), //l eye (Interpupillary breadth)
new Point3(34, 90, 83), //r eye (Interpupillary breadth)
new Point3(0.0, 50, 120), //nose (Nose top)
new Point3(-26, 15, 83), //l mouse (Mouth breadth)
new Point3(26, 15, 83), //r mouse (Mouth breadth)
new Point3(-79, 90, 0.0), //l ear (Bitragion breadth)
new Point3(79, 90, 0.0) //r ear (Bitragion breadth)
);
imagePoints = new MatOfPoint2f();
rotMat = new Mat(3, 3, CvType.CV_64FC1);
opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter.diffDlib /= imageOptimizationHelper.downscaleRatio;
}
// Update is called once per frame
void Update()
{
// loop play.
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
if (capture.grab())
{
capture.retrieve(rgbMat, 0);
Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB);
//Debug.Log ("Mat toString " + rgbMat.ToString ());
// detect faces.
List <OpenCVForUnity.Rect> detectResult = new List <OpenCVForUnity.Rect> ();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new OpenCVForUnity.Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect()) {
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// adjust to Dilb's result.
foreach (OpenCVForUnity.Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
}
// face tracking.
rectangleTracker.UpdateTrackedObjects(detectResult);
List <TrackedRect> trackedRects = new List <TrackedRect> ();
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict [openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict [openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// create LUT texture.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
faceMaskColorCorrector.CreateLUTTex(openCVRect.id);
}
else if (openCVRect.state == TrackedState.DELETED)
{
faceMaskColorCorrector.DeleteLUTTex(openCVRect.id);
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> > ();
for (int i = 0; i < trackedRects.Count; i++)
{
TrackedRect tr = trackedRects [i];
UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
if (tr.state > TrackedState.NEW && tr.state < TrackedState.DELETED)
{
opticalFlowFilterDict [tr.id].Process(rgbMat, points, points);
lowPassFilterDict [tr.id].Process(rgbMat, points, points);
}
}
landmarkPoints.Add(points);
}
// face masking.
if (faceMaskTexture != null && landmarkPoints.Count >= 1) // Apply face masking between detected faces and a face mask image.
{
float maskImageWidth = faceMaskTexture.width;
float maskImageHeight = faceMaskTexture.height;
TrackedRect tr;
for (int i = 0; i < trackedRects.Count; i++)
{
tr = trackedRects [i];
if (tr.state == TrackedState.NEW)
{
meshOverlay.CreateObject(tr.id, faceMaskTexture);
}
if (tr.state < TrackedState.DELETED)
{
MaskFace(meshOverlay, tr, landmarkPoints [i], faceLandmarkPointsInMask, maskImageWidth, maskImageHeight);
if (enableColorCorrection)
{
CorrectFaceMaskColor(tr.id, faceMaskMat, rgbMat, faceLandmarkPointsInMask, landmarkPoints [i]);
}
}
else if (tr.state == TrackedState.DELETED)
{
meshOverlay.DeleteObject(tr.id);
}
}
}
else if (landmarkPoints.Count >= 1) // Apply face masking between detected faces.
{
float maskImageWidth = texture.width;
float maskImageHeight = texture.height;
TrackedRect tr;
for (int i = 0; i < trackedRects.Count; i++)
{
tr = trackedRects [i];
if (tr.state == TrackedState.NEW)
{
meshOverlay.CreateObject(tr.id, texture);
}
if (tr.state < TrackedState.DELETED)
{
MaskFace(meshOverlay, tr, landmarkPoints [i], landmarkPoints [0], maskImageWidth, maskImageHeight);
if (enableColorCorrection)
{
CorrectFaceMaskColor(tr.id, rgbMat, rgbMat, landmarkPoints [0], landmarkPoints [i]);
}
}
else if (tr.state == TrackedState.DELETED)
{
meshOverlay.DeleteObject(tr.id);
}
}
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < detectResult.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(detectResult [i].x, detectResult [i].y, detectResult [i].width, detectResult [i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2);
}
for (int i = 0; i < trackedRects.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects [i].x, trackedRects [i].y, trackedRects [i].width, trackedRects [i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 255, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
// draw face points.
if (displayDebugFacePoints)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2);
}
}
// display face mask image.
if (faceMaskTexture != null && faceMaskMat != null)
{
if (displayFaceRects)
{
OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2);
}
if (displayDebugFacePoints)
{
OpenCVForUnityUtils.DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2);
}
float scale = (rgbMat.width() / 4f) / faceMaskMat.width();
float tx = rgbMat.width() - faceMaskMat.width() * scale;
float ty = 0.0f;
Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty);
trans.put(0, 0, scale);
trans.put(0, 1, 0.0f);
trans.put(0, 2, tx);
trans.put(1, 0, 0.0f);
trans.put(1, 1, scale);
trans.put(1, 2, ty);
Imgproc.warpAffine(faceMaskMat, rgbMat, trans, rgbMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0));
if (displayFaceRects || displayDebugFacePointsToggle)
{
OpenCVForUnity.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
}
}
// Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.fastMatToTexture2D(rgbMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (sourceToMatHelper.IsPlaying() && sourceToMatHelper.DidUpdateThisFrame())
{
Mat rgbMat = sourceToMatHelper.GetMat();
// detect faces.
List <Rect> detectResult = new List <Rect>();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
foreach (Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
}
// face tracking.
List <TrackedRect> trackedRects = new List <TrackedRect>();
rectangleTracker.UpdateTrackedObjects(detectResult);
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict[openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict[openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
opticalFlowFilterDict[openCVRect.id].Process(rgbMat, points, points);
lowPassFilterDict[openCVRect.id].Process(rgbMat, points, points);
}
landmarkPoints.Add(points);
}
}
// filter non frontal faces.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit)
{
trackedRects.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// face swapping.
if (landmarkPoints.Count >= 2)
{
int ann = 0, bob = 1;
for (int i = 0; i < landmarkPoints.Count - 1; i += 2)
{
ann = i;
bob = i + 1;
faceSwapper.SwapFaces(rgbMat, landmarkPoints[ann], landmarkPoints[bob], 1);
}
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < trackedRects.Count; i++)
{
Rect openCVRect = trackedRects[i];
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2);
//Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
//Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// detect faces.
List <Rect> detectResult;
DetectFaces(rgbaMat, out detectResult, useDlibFaceDetecter);
// face tracking.
List <TrackedRect> trackedRects = new List <TrackedRect>();
rectangleTracker.UpdateTrackedObjects(detectResult);
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict[openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict[openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
opticalFlowFilterDict[openCVRect.id].Process(rgbaMat, points, points);
lowPassFilterDict[openCVRect.id].Process(rgbaMat, points, points);
}
landmarkPoints.Add(points);
}
}
// filter non frontal faces.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit)
{
trackedRects.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// face changing.
if (faceMaskTexture != null && landmarkPoints.Count >= 1)
{ // Apply face changing between detected faces and a mask image.
if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f)
{
DetectFaces(faceMaskMat, out detectResult, useDlibFaceDetecter);
if (detectResult.Count >= 1)
{
detectedFaceRect = new UnityEngine.Rect(detectResult[0].x, detectResult[0].y, detectResult[0].width, detectResult[0].height);
}
}
if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat);
List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect);
faceChanger.SetTargetImage(rgbaMat);
for (int i = 0; i < landmarkPoints.Count; i++)
{
faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints[i], 1);
}
faceChanger.ChangeFace();
if (displayFaceRects)
{
OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2);
}
}
}
else if (landmarkPoints.Count >= 2)
{ // Apply face changing between detected faces.
faceChanger.SetTargetImage(rgbaMat);
for (int i = 1; i < landmarkPoints.Count; i++)
{
faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints[0], landmarkPoints[i], 1);
}
faceChanger.ChangeFace();
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < trackedRects.Count; i++)
{
Rect openCVRect = trackedRects[i];
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
// display face mask image.
if (faceMaskMat != null)
{
float scale = (rgbaMat.width() / 4f) / faceMaskMat.width();
float tx = rgbaMat.width() - faceMaskMat.width() * scale;
float ty = 0.0f;
Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty);
trans.put(0, 0, scale);
trans.put(0, 1, 0.0f);
trans.put(0, 2, tx);
trans.put(1, 0, 0.0f);
trans.put(1, 1, scale);
trans.put(1, 2, ty);
Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0));
if (displayFaceRects || displayDebugFacePoints)
{
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
private void Run()
{
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
rgbMat = new Mat();
capture = new VideoCapture();
capture.open(video_filepath);
if (!capture.isOpened())
{
Debug.LogError("capture.isOpened() is false. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
double ext = capture.get(Videoio.CAP_PROP_FOURCC);
Debug.Log("CAP_PROP_FOURCC: " + (char)((int)ext & 0XFF) + (char)(((int)ext & 0XFF00) >> 8) + (char)(((int)ext & 0XFF0000) >> 16) + (char)(((int)ext & 0XFF000000) >> 24));
capture.grab();
capture.retrieve(rgbMat, 0);
int frameWidth = rgbMat.cols();
int frameHeight = rgbMat.rows();
texture = new Texture2D(frameWidth, frameHeight, TextureFormat.RGB24, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
float widthScale = (float)Screen.width / (float)frameWidth;
float heightScale = (float)Screen.height / (float)frameHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = (float)frameHeight / 2;
}
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName);
fpsMonitor.Add("width", frameWidth.ToString());
fpsMonitor.Add("height", frameHeight.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
}
private void Run()
{
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
lowPassFilter = new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
kalmanFilter = new KFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
rgbMat = new Mat();
capture = new VideoCapture();
capture.open(dance_avi_filepath);
if (capture.isOpened())
{
Debug.Log("capture.isOpened() true");
}
else
{
Debug.Log("capture.isOpened() false");
}
Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT));
Debug.Log("CV_CAP_PROP_PREVIEW_FORMAT: " + capture.get(Videoio.CV_CAP_PROP_PREVIEW_FORMAT));
Debug.Log("CAP_PROP_POS_MSEC: " + capture.get(Videoio.CAP_PROP_POS_MSEC));
Debug.Log("CAP_PROP_POS_FRAMES: " + capture.get(Videoio.CAP_PROP_POS_FRAMES));
Debug.Log("CAP_PROP_POS_AVI_RATIO: " + capture.get(Videoio.CAP_PROP_POS_AVI_RATIO));
Debug.Log("CAP_PROP_FRAME_COUNT: " + capture.get(Videoio.CAP_PROP_FRAME_COUNT));
Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS));
Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
capture.grab();
capture.retrieve(rgbMat, 0);
int frameWidth = rgbMat.cols();
int frameHeight = rgbMat.rows();
texture = new Texture2D(frameWidth, frameHeight, TextureFormat.RGB24, false);
gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1);
float widthScale = (float)Screen.width / (float)frameWidth;
float heightScale = (float)Screen.height / (float)frameHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = (float)frameHeight / 2;
}
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
if (fpsMonitor != null)
{
fpsMonitor.Add("dlib shape predictor", dlibShapePredictorFileName);
fpsMonitor.Add("width", frameWidth.ToString());
fpsMonitor.Add("height", frameHeight.ToString());
fpsMonitor.Add("orientation", Screen.orientation.ToString());
}
}
